Repository: buraksezer/olricdb Branch: master Commit: 792bb41fc287 Files: 202 Total size: 981.6 KB Directory structure: gitextract_bkk5kycd/ ├── .github/ │ ├── FUNDING.yml │ └── workflows/ │ ├── ci.yml │ ├── codeql-analysis.yml │ └── golangci-lint.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── auth.go ├── auth_test.go ├── client.go ├── cluster.go ├── cluster_client.go ├── cluster_client_test.go ├── cluster_iterator.go ├── cluster_iterator_test.go ├── cluster_test.go ├── cmd/ │ └── olric-server/ │ ├── main.go │ ├── olric-server-local.yaml │ └── server/ │ └── server.go ├── config/ │ ├── authentication.go │ ├── client.go │ ├── config.go │ ├── config_test.go │ ├── dmap.go │ ├── dmap_test.go │ ├── dmaps.go │ ├── engine.go │ ├── engine_test.go │ ├── internal/ │ │ └── loader/ │ │ └── loader.go │ ├── load.go │ ├── memberlist.go │ ├── network.go │ └── network_test.go ├── docker/ │ ├── README.md │ ├── docker-compose.yml │ ├── nginx.conf │ └── olric-server-consul.yaml ├── embedded_client.go ├── embedded_client_test.go ├── embedded_iterator.go ├── embedded_iterator_test.go ├── events/ │ ├── cluster_events.go │ └── cluster_events_test.go ├── get_response.go ├── get_response_test.go ├── go.mod ├── go.sum ├── hasher/ │ └── hasher.go ├── integration_test.go ├── internal/ │ ├── bufpool/ │ │ ├── bufpool.go │ │ └── bufpool_test.go │ ├── checkpoint/ │ │ ├── checkpoint.go │ │ └── checkpoint_test.go │ ├── cluster/ │ │ ├── balancer/ │ │ │ ├── balancer.go │ │ │ └── balancer_test.go │ │ ├── partitions/ │ │ │ ├── fragment.go │ │ │ ├── hkey.go │ │ │ ├── hkey_test.go │ │ │ ├── partition.go │ │ │ ├── partition_test.go │ │ │ ├── partitions.go │ │ │ └── partitions_test.go │ │ └── routingtable/ │ │ ├── callback.go │ │ ├── callback_test.go │ │ ├── discovery.go │ │ ├── discovery_test.go │ │ ├── distribute.go │ │ ├── distribute_test.go │ │ ├── events.go │ │ ├── events_test.go │ │ ├── handlers.go │ │ ├── left_over_data.go │ │ ├── left_over_data_test.go │ │ ├── members.go │ │ ├── members_test.go │ │ ├── operations.go │ │ ├── routingtable.go │ │ ├── routingtable_test.go │ │ └── update.go │ ├── discovery/ │ │ ├── delegate.go │ │ ├── discovery.go │ │ ├── discovery_test.go │ │ ├── events.go │ │ ├── member.go │ │ └── member_test.go │ ├── dmap/ │ │ ├── atomic.go │ │ ├── atomic_handlers.go │ │ ├── atomic_test.go │ │ ├── balance.go │ │ ├── balance_test.go │ │ ├── compaction.go │ │ ├── compaction_test.go │ │ ├── config.go │ │ ├── config_test.go │ │ ├── delete.go │ │ ├── delete_handlers.go │ │ ├── delete_test.go │ │ ├── destroy.go │ │ ├── destroy_handlers.go │ │ ├── destroy_test.go │ │ ├── dmap.go │ │ ├── dmap_test.go │ │ ├── env.go │ │ ├── eviction.go │ │ ├── eviction_test.go │ │ ├── expire.go │ │ ├── expire_handlers.go │ │ ├── expire_test.go │ │ ├── fragment.go │ │ ├── fragment_test.go │ │ ├── get.go │ │ ├── get_handlers.go │ │ ├── get_test.go │ │ ├── handlers.go │ │ ├── janitor.go │ │ ├── lock.go │ │ ├── lock_handlers.go │ │ ├── lock_test.go │ │ ├── put.go │ │ ├── put_handlers.go │ │ ├── put_test.go │ │ ├── scan_handlers.go │ │ ├── scan_test.go │ │ ├── service.go │ │ ├── service_test.go │ │ └── stats_test.go │ ├── environment/ │ │ ├── environment.go │ │ └── environment_test.go │ ├── locker/ │ │ ├── locker.go │ │ └── locker_test.go │ ├── protocol/ │ │ ├── cluster.go │ │ ├── cluster_test.go │ │ ├── commands.go │ │ ├── dmap.go │ │ ├── dmap_test.go │ │ ├── errors.go │ │ ├── errors_test.go │ │ ├── pubsub.go │ │ ├── pubsub_test.go │ │ ├── system.go │ │ └── system_test.go │ ├── pubsub/ │ │ ├── handlers.go │ │ ├── handlers_test.go │ │ ├── pubsub.go │ │ ├── pubsub_test.go │ │ └── service.go │ ├── ramblock/ │ │ ├── compaction.go │ │ ├── compaction_test.go │ │ ├── entry/ │ │ │ ├── entry.go │ │ │ └── entry_test.go │ │ ├── ramblock.go │ │ ├── ramblock_test.go │ │ ├── table/ │ │ │ ├── pack.go │ │ │ ├── pack_test.go │ │ │ ├── table.go │ │ │ └── table_test.go │ │ └── transport.go │ ├── resp/ │ │ ├── encoder.go │ │ ├── encoder_test.go │ │ └── scan.go │ ├── roundrobin/ │ │ ├── round_robin.go │ │ └── round_robin_test.go │ ├── server/ │ │ ├── client.go │ │ ├── client_test.go │ │ ├── handler.go │ │ ├── handler_test.go │ │ ├── mux.go │ │ ├── mux_test.go │ │ ├── server.go │ │ └── server_test.go │ ├── service/ │ │ └── service.go │ ├── stats/ │ │ ├── stats.go │ │ └── stats_test.go │ ├── testcluster/ │ │ └── testcluster.go │ ├── testutil/ │ │ ├── mockfragment/ │ │ │ └── mockfragment.go │ │ └── testutil.go │ └── util/ │ ├── safe.go │ ├── strconv.go │ └── unsafe.go ├── olric-server-docker.yaml ├── olric.go ├── olric_test.go ├── ping.go ├── ping_test.go ├── pipeline.go ├── pipeline_test.go ├── pkg/ │ ├── flog/ │ │ └── flog.go │ ├── neterrors/ │ │ └── errors.go │ ├── service_discovery/ │ │ └── service_discovery.go │ └── storage/ │ ├── config.go │ ├── config_test.go │ ├── engine.go │ ├── entry.go │ └── stats.go ├── pubsub.go ├── pubsub_test.go ├── stats/ │ ├── stats.go │ └── stats_test.go ├── stats.go └── stats_test.go ================================================ FILE CONTENTS ================================================ ================================================ FILE: .github/FUNDING.yml ================================================ # These are supported funding model platforms github: buraksezer patreon: # not used anymore open_collective: # Replace with a single Open Collective username ko_fi: # Replace with a single Ko-fi username tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry liberapay: # Replace with a single Liberapay username issuehunt: # Replace with a single IssueHunt username otechie: # Replace with a single Otechie username custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] ================================================ FILE: .github/workflows/ci.yml ================================================ name: Unit & Integration tests on: push: branches: [ "master" ] pull_request: branches: [ "master" ] jobs: test: strategy: # Default is true, cancels jobs for other platforms in the matrix if one fails fail-fast: false matrix: os: [ ubuntu-latest ] go: [ '1.23', '1.24' ] runs-on: ${{ matrix.os }} steps: - name: Install Go uses: actions/setup-go@v4 with: go-version: ${{ matrix.go }} - name: Checkout code uses: actions/checkout@v3 - name: Print Go version and environment id: vars run: | printf "Using go at: $(which go)\n" printf "Go version: $(go version)\n" printf "\n\nGo environment:\n\n" go env printf "\n\nSystem environment:\n\n" env # Calculate the short SHA1 hash of the git commit echo "::set-output name=short_sha::$(git rev-parse --short HEAD)" echo "::set-output name=go_cache::$(go env GOCACHE)" - name: Cache the build cache uses: actions/cache@v4 with: path: ${{ steps.vars.outputs.go_cache }} key: ${{ runner.os }}-${{ matrix.go }}-go-ci-${{ hashFiles('**/go.sum') }} restore-keys: | ${{ runner.os }}-${{ matrix.go }}-go-ci - name: Install dependencies run: | go mod download - name: Run tests run: make test ================================================ FILE: .github/workflows/codeql-analysis.yml ================================================ # For most projects, this workflow file will not need changing; you simply need # to commit it to your repository. # # You may wish to alter this file to override the set of languages analyzed, # or to provide custom queries or build logic. # # ******** NOTE ******** # We have attempted to detect the languages in your repository. Please check # the `language` matrix defined below to confirm you have the correct set of # supported CodeQL languages. # name: "CodeQL" on: push: branches: [ master ] pull_request: # The branches below must be a subset of the branches above branches: [ master ] schedule: - cron: '28 20 * * 0' jobs: analyze: name: Analyze runs-on: ubuntu-latest permissions: actions: read contents: read security-events: write strategy: fail-fast: false matrix: language: [ 'go' ] # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] # Learn more about CodeQL language support at https://git.io/codeql-language-support steps: - name: Checkout repository uses: actions/checkout@v2 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. # queries: ./path/to/local/query, your-org/your-repo/queries@main # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild uses: github/codeql-action/autobuild@v3 # ℹ️ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines # and modify them (or add more) to build your code if your project # uses a compiled language #- run: | # make bootstrap # make release - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v3 ================================================ FILE: .github/workflows/golangci-lint.yml ================================================ name: golangci-lint on: push: branches: - master pull_request: permissions: contents: read # Optional: allow read access to pull request. Use with `only-new-issues` option. # pull-requests: read jobs: golangci: name: lint runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: stable - name: golangci-lint uses: golangci/golangci-lint-action@v7 with: version: v2.0 ================================================ FILE: .gitignore ================================================ # Binaries for programs and plugins *.exe *.dll *.so *.dylib # Vim creates this *.swp # Test binary, build with `go test -c` *.test # Output of the go coverage tool, specifically when used with LiteIDE *.out # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 .glide/ # GoLand creates this .idea/ # OSX creates this .DS_Store .claude/ CLAUDE.md ================================================ FILE: Dockerfile ================================================ FROM golang:latest as build WORKDIR /src/ COPY . /src/ RUN go mod download RUN CGO_ENABLED=1 go build -ldflags="-s -w" -o /usr/bin/olric-server /src/cmd/olric-server FROM gcr.io/distroless/base-debian12 COPY --from=build /usr/bin/olric-server /usr/bin/olric-server COPY --from=build /src/olric-server-docker.yaml /etc/olric-server.yaml EXPOSE 3320 3322 ENTRYPOINT ["/usr/bin/olric-server", "-c", "/etc/olric-server.yaml"] ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 https://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS Copyright 2018-2025 The Olric Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: Makefile ================================================ .PHONY: test test: go test -p 1 ./... .PHONY: test-quick test-quick: go test -p 1 -count=1 ./... .PHONY: test-race test-race: go test -p 1 -race ./... .PHONY: format format: go fmt ./... .PHONY: prepare-merge prepare-merge: format test .PHONY: ci ci: test .PHONY: ci-quick ci-full: test-quick .PHONY: install install: go install -ldflags="-s -w" -v ./cmd/* ================================================ FILE: README.md ================================================ # Olric [![Tweet](https://img.shields.io/twitter/url/http/shields.io.svg?style=social)](https://twitter.com/intent/tweet?text=Olric%3A+Distributed+and+in-memory+key%2Fvalue+database.+It+can+be+used+both+as+an+embedded+Go+library+and+as+a+language-independent+service.+&url=https://github.com/olric-data/olric/&hashtags=golang,distributed,database) [![Go Reference](https://pkg.go.dev/badge/github.com/olric-data/olric/.svg)](https://pkg.go.dev/github.com/olric-data/olric/) [![Go Report Card](https://goreportcard.com/badge/olric-data/olric)](https://goreportcard.com/report/github.com/olric-data/olric/) [![Discord](https://img.shields.io/discord/721708998021087273.svg?label=&logo=discord&logoColor=ffffff&color=7389D8&labelColor=6A7EC2)](https://discord.gg/ahK7Vjr8We) [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) Distributed In-Memory Cache & Key/Value Store Olric provides a simple way to create a **fast, scalable, and shared pool of RAM** across a cluster of machines. It's a distributed, in-memory key/value store and cache, written entirely in Go and designed specifically for distributed environments. **Flexible Deployment:** * **Embedded Go Library:** Integrate Olric directly into your Go applications. * **Standalone Service:** Run Olric as a language-independent service. **Key Features:** * **Effortless Scalability:** Designed to handle hundreds of members and thousands of clients. New nodes auto-discover the cluster and linearly increase capacity. * **Automatic Distribution:** Provides partitioning (sharding) and data re-balancing out-of-the-box, requiring no external coordination services. Data and backups are automatically balanced when capacity is added. * **Wide Client Support:** Uses the standard **Redis Serialization Protocol (RESP)**, ensuring client libraries are available in nearly all major programming languages. * **Common Use Cases:** Ideal for distributed caching, managing application cluster state, and implementing publish-subscribe messaging. See [Docker](#docker) and [Samples](#samples) sections to get started! Join our [Discord server!](https://discord.gg/ahK7Vjr8We) The current production version is [v0.7.0](https://github.com/olric-data/olric/tree/release/v0.7) ### About renaming the module `github.com/buraksezer/olric` module has been renamed to `github.com/olric-data/olric`. This change has been effective since **v0.6.0**. Importing previous versions should redirect you to the new repository, but you should change the import paths in your codebase as soon as possible. There is no other difference between v0.5.7 and v0.6.0. ## At a glance * Designed to share some transient, approximate, fast-changing data between servers, * Uses Redis serialization protocol, * Implements a distributed hash table, * Provides a drop-in replacement for Redis Publish/Subscribe messaging system, * Supports both programmatic and declarative configuration, * Embeddable but can be used as a language-independent service with *olric-server*, * Supports different eviction algorithms (including LRU and TTL), * Highly available and horizontally scalable, * Provides best-effort consistency guarantees without being a complete CP (indeed PA/EC) solution, * Supports replication by default (with sync and async options), * Quorum-based voting for replica control (Read/Write quorums), * Supports atomic operations, * Provides an iterator on distributed maps, * Provides a plugin interface for service discovery daemons, * Provides a locking primitive which inspired by [SETNX of Redis](https://redis.io/commands/setnx#design-pattern-locking-with-codesetnxcode), ## Possible Use Cases Olric is an eventually consistent, unordered key/value data store. It supports various eviction mechanisms for distributed caching implementations. Olric also provides publish-subscribe messaging, data replication, failure detection and simple anti-entropy services. It's good at distributed caching and publish/subscribe messaging. ## Table of Contents * [Features](#features) * [Support](#support) * [Installing](#installing) * [Docker](#docker) * [Getting Started](#getting-started) * [Operation Modes](#operation-modes) * [Embedded Member](#embedded-member) * [Client-Server](#client-server) * [Golang Client](#golang-client) * [Cluster Events](#cluster-events) * [Authentication](#authentication) * [Commands](#commands) * [Distributed Map](#distributed-map) * [DM.PUT](#dmput) * [DM.GET](#dmget) * [DM.DEL](#dmdel) * [DM.EXPIRE](#dmexpire) * [DM.PEXPIRE](#dmpexpire) * [DM.DESTROY](#dmdestroy) * [Atomic Operations](#atomic-operations) * [DM.INCR](#dmincr) * [DM.DECR](#dmdecr) * [DM.GETPUT](#dmgetput) * [DM.INCRBYFLOAT](#dmincrbyfloat) * [Locking](#locking) * [DM.LOCK](#dmlock) * [DM.UNLOCK](#dmunlock) * [DM.LOCKLEASE](#dmlocklease) * [DM.PLOCKLEASE](#dmplocklease) * [DM.SCAN](#dmscan) * [Publish-Subscribe](#publish-subscribe) * [SUBSCRIBE](#subscribe) * [PSUBSCRIBE](#psubscribe) * [UNSUBSCRIBE](#unsubscribe) * [PUNSUBSCRIBE](#punsubscribe) * [PUBSUB CHANNELS](#pubsub-channels) * [PUBSUB NUMPAT](#pubsub-numpat) * [PUBSUB NUMSUB](#pubsub-numsub) * [QUIT](#quit) * [PING](#ping) * [Cluster](#cluster) * [CLUSTER.ROUTINGTABLE](#clusterroutingtable) * [CLUSTER.MEMBERS](#clustermembers) * [Others](#others) * [PING](#ping) * [STATS](#stats) * [AUTH](#auth) * [Configuration](#configuration) * [Embedded Member Mode](#embedded-member-mode) * [Manage the configuration in YAML format](#manage-the-configuration-in-yaml-format) * [Client-Server Mode](#client-server-mode) * [Network Configuration](#network-configuration) * [Service discovery](#service-discovery) * [Timeouts](#timeouts) * [Architecture](#architecture) * [Overview](#overview) * [Consistency and Replication Model](#consistency-and-replication-model) * [Last-write-wins conflict resolution](#last-write-wins-conflict-resolution) * [PACELC Theorem](#pacelc-theorem) * [Read-Repair on DMaps](#read-repair-on-dmaps) * [Quorum-based Replica Control](#quorum-based-replica-control) * [Simple Split-Brain Protection](#simple-split-brain-protection) * [Eviction](#eviction) * [Expire with TTL](#expire-with-ttl) * [Expire with MaxIdleDuration](#expire-with-maxidleduration) * [Expire with LRU](#expire-with-lru) * [Lock Implementation](#lock-implementation) * [Storage Engine](#storage-engine) * [Samples](#samples) * [Contributions](#contributions) * [License](#license) * [About the name](#about-the-name) ## Features * Designed to share some transient, approximate, fast-changing data between servers, * Accepts arbitrary types as value, * Only in-memory, * Uses Redis protocol, * Compatible with existing Redis clients, * Embeddable but can be used as a language-independent service with olric-server, * GC-friendly storage engine, * O(1) running time for lookups, * Supports atomic operations, * Provides a lock implementation which can be used for non-critical purposes, * Different eviction policies: LRU, MaxIdleDuration and Time-To-Live (TTL), * Highly available, * Horizontally scalable, * Provides best-effort consistency guarantees without being a complete CP (indeed PA/EC) solution, * Distributes load fairly among cluster members with a [consistent hash function](https://github.com/buraksezer/consistent), * Supports replication by default (with sync and async options), * Quorum-based voting for replica control, * Thread-safe by default, * Provides an iterator on distributed maps, * Provides a plugin interface for service discovery daemons and cloud providers, * Provides a locking primitive which inspired by [SETNX of Redis](https://redis.io/commands/setnx#design-pattern-locking-with-codesetnxcode), * Provides a drop-in replacement of Redis' Publish-Subscribe messaging feature. See the [Architecture](#architecture) section to see details. ## Support We have a few communication channels: * [Issue Tracker](https://github.com/olric-data/olric/issues) * [Discord server](https://discord.gg/ahK7Vjr8We) You should know that the issue tracker is only intended for bug reports and feature requests. Software doesn't maintain itself. If you need support on complex topics or request new features, please consider [sponsoring Olric](https://github.com/sponsors/buraksezer). ## Installing With a correctly configured Golang environment: ``` go install github.com/olric-data/olric/cmd/olric-server@v0.7.0 ``` Now you can start using Olric: ``` olric-server -c cmd/olric-server/olric-server-local.yaml ``` See the [Configuration](#configuration) section to create your cluster properly. ### Docker You can launch `olric-server` Docker container by running the following command. ```bash docker pull ghcr.io/olric-data/olric:latest ``` This command will pull olric-server Docker image and run a new Olric Instance. You should know that the container exposes `3320` and `3322` ports. Now, you can access an Olric cluster using any Redis client including `redis-cli`: ```bash redis-cli -p 3320 127.0.0.1:3320> DM.PUT my-dmap my-key "Olric Rocks!" OK 127.0.0.1:3320> DM.GET my-dmap my-key "Olric Rocks!" 127.0.0.1:3320> ``` ## Getting Started With olric-server, you can create an Olric cluster with a few commands. This is how to install olric-server: ```bash go install github.com/olric-data/olric/cmd/olric-server@v0.7.0 ``` Let's create a cluster with the following: ``` olric-server -c ``` You can find the sample configuration file under `cmd/olric-server/olric-server-local.yaml`. It can perfectly run with single node. olric-server also supports `OLRIC_SERVER_CONFIG` environment variable to set configuration. Just like that: ``` OLRIC_SERVER_CONFIG= olric-server ``` Olric uses [hashicorp/memberlist](https://github.com/hashicorp/memberlist) for failure detection and cluster membership. Currently, there are different ways to discover peers in a cluster. You can use a static list of nodes in your configuration. It's ideal for development and test environments. Olric also supports Consul, Kubernetes and all well-known cloud providers for service discovery. Please take a look at [Service Discovery](#service-discovery) section for further information. See [Client-Server](#client-server) section to get more information about this deployment scenario. #### Maintaining a list of peers manually Basically, there is a list of nodes under `memberlist` block in the configuration file. In order to create an Olric cluster, you just need to add `Host:Port` pairs of the other nodes. Please note that the `Port` is the memberlist port of the peer. It is `3322` by default. ```yaml memberlist: peers: - "localhost:3322" ``` Thanks to [hashicorp/memberlist](https://github.com/hashicorp/memberlist), Olric nodes can share the full list of members with each other. So an Olric node can discover the whole cluster by using a single member address. #### Embedding into your Go application. See [Samples](#samples) section to learn how to embed Olric into your existing Golang application. ### Operation Modes Olric has two different operation modes. #### Embedded Member In Embedded Member Mode, members include both the application and Olric data and services. The advantage of the Embedded Member Mode is having a low-latency data access and locality. #### Client-Server In Client-Server Mode, Olric data and services are centralized in one or more servers, and they are accessed by the application through clients. You can have a cluster of servers that can be independently created and scaled. Your clients communicate with these members to reach to Olric data and services on them. Client-Server deployment has advantages including more predictable and reliable performance, easier identification of problem causes and, most importantly, better scalability. When you need to scale in this deployment type, just add more Olric server members. You can address client and server scalability concerns separately. ## Golang Client The official Golang client is defined by the `Client` interface. There are two different implementations of that interface in this repository. `EmbeddedClient` provides a client implementation for [embedded-member](#embedded-member) scenario, `ClusterClient` provides an implementation of the same interface for [client-server](#client-server) deployment scenario. Obviously, you can use `ClusterClient` for your embedded-member deployments. But it's good to use `EmbeddedClient` provides a better performance due to localization of the queries. See the client documentation on [pkg.go.dev](https://pkg.go.dev/github.com/olric-data/olric/@v0.7.0) ## Cluster Events Olric can send push cluster events to `cluster.events` channel. Available cluster events: * node-join-event * node-left-event * fragment-migration-event * fragment-received-even If you want to receive these events, set `true` to `EnableClusterEventsChannel` and subscribe to `cluster.events` channel. The default is `false`. See the [events/cluster_events.go](events/cluster_events.go) file to get more information about events. ## Authentication Olric supports simple password-based authentication to restrict access to the data store. This mechanism is similar to the `requirepass` directive in Redis and is intended to provide a basic level of protection in trusted environments (e.g., internal networks or local development). > **Important**: This authentication method **does not provide transport-layer encryption or full access control**. For secure > deployments over untrusted networks (e.g., Internet), it's strongly recommended to place Olric behind a reverse proxy with TLS > support or use a secure network overlay (e.g., WireGuard, VPN). ### YAML-based Configuration You can enable password-based authentication by adding the `authentication` block to your configuration file: ```yaml authentication: password: "your-password" ``` When this is set, all clients must authenticate using the provided password before performing any operations. ### Programmatic Configuration (Go API) For applications embedding Olric or configuring it dynamically in Go, you can enable authentication as follows: ```go c := config.New("local") c.Authentication = &config.Authentication{ Password: "your-password", } ``` This sets the password required for any client to interact with the Olric node. ### Client-Side Usage Clients must send the password using the [AUTH](#auth) command. If the password is incorrect or not provided, the connection will be denied or commands will be rejected. With the cluster client, you can use `WithPassword` cluster client option. ```go client, err := NewClusterClient([]string{db.name}, WithPassword("test-password")) ``` **Important:** The embedded client has not been covered by the authentication implementation. ## Commands Olric uses Redis protocol and supports Redis-style commands to query the database. You can use any Redis client, including `redis-cli`. The official Go client is a thin layer around [go-redis/redis](https://github.com/go-redis/redis) package. See [Golang Client](#golang-client) section for the documentation. ### Distributed Map #### DM.PUT DM.PUT sets the value for the given key. It overwrites any previous value for that key. ``` DM.PUT dmap key value [ EX seconds | PX milliseconds | EXAT unix-time-seconds | PXAT unix-time-milliseconds ] [ NX | XX] ``` **Example:** ``` 127.0.0.1:3320> DM.PUT my-dmap my-key value OK ``` **Options:** The DM.PUT command supports a set of options that modify its behavior: * **EX** *seconds* -- Set the specified expire time, in seconds. * **PX** *milliseconds* -- Set the specified expire time, in milliseconds. * **EXAT** *timestamp-seconds* -- Set the specified Unix time at which the key will expire, in seconds. * **PXAT** *timestamp-milliseconds* -- Set the specified Unix time at which the key will expire, in milliseconds. * **NX** -- Only set the key if it does not already exist. * **XX** -- Only set the key if it already exist. **Return:** * **Simple string reply:** OK if DM.PUT was executed correctly. * **KEYFOUND:** (error) if the DM.PUT operation was not performed because the user specified the NX option but the condition was not met. * **KEYNOTFOUND:** (error) if the DM.PUT operation was not performed because the user specified the XX option but the condition was not met. #### DM.GET DM.GET gets the value for the given key. It returns (error)`KEYNOTFOUND` if the key doesn't exist. ``` DM.GET dmap key ``` **Example:** ``` 127.0.0.1:3320> DM.GET dmap key "value" ``` **Return:** **Bulk string reply**: the value of key, or (error)`KEYNOTFOUND` when key does not exist. #### DM.DEL DM.DEL deletes values for the given keys. It doesn't return any error if the key does not exist. ``` DM.DEL dmap key [key...] ``` **Example:** ``` 127.0.0.1:3320> DM.DEL dmap key1 key2 (integer) 2 ``` **Return:** * **Integer reply**: The number of keys that were removed. #### DM.EXPIRE DM.EXPIRE updates or sets the timeout for the given key. It returns `KEYNOTFOUND` if the key doesn't exist. After the timeout has expired, the key will automatically be deleted. The timeout will only be cleared by commands that delete or overwrite the contents of the key, including DM.DEL, DM.PUT, DM.GETPUT. ``` DM.EXPIRE dmap key seconds ``` **Example:** ``` 127.0.0.1:3320> DM.EXPIRE dmap key 1 OK ``` **Return:** * **Simple string reply:** OK if DM.EXPIRE was executed correctly. * **KEYNOTFOUND:** (error) when key does not exist. #### DM.PEXPIRE DM.PEXPIRE updates or sets the timeout for the given key. It returns `KEYNOTFOUND` if the key doesn't exist. After the timeout has expired, the key will automatically be deleted. The timeout will only be cleared by commands that delete or overwrite the contents of the key, including DM.DEL, DM.PUT, DM.GETPUT. ``` DM.PEXPIRE dmap key milliseconds ``` **Example:** ``` 127.0.0.1:3320> DM.PEXPIRE dmap key 1000 OK ``` **Return:** * **Simple string reply:** OK if DM.EXPIRE was executed correctly. * **KEYNOTFOUND:** (error) when key does not exist. #### DM.DESTROY DM.DESTROY flushes the given DMap on the cluster. You should know that there is no global lock on DMaps. DM.PUT and DM.DESTROY commands may run concurrently on the same DMap. ``` DM.DESTROY dmap ``` **Example:** ``` 127.0.0.1:3320> DM.DESTROY dmap OK ``` **Return:** * **Simple string reply:** OK, if DM.DESTROY was executed correctly. ### Atomic Operations Operations on key/value pairs are performed by the partition owner. In addition, atomic operations are guarded by a lock implementation which can be found under `internal/locker`. It means that Olric guaranties consistency of atomic operations, if there is no network partition. Basic flow for `DM.INCR`: * Acquire the lock for the given key, * Call `DM.GET` to retrieve the current value, * Calculate the new value, * Call `DM.PUT` to set the new value, * Release the lock. It's important to know that if you call `DM.PUT` and `DM.GETPUT` concurrently on the same key, this will break the atomicity. `internal/locker` package is provided by [Docker](https://github.com/moby/moby). **Important note about consistency:** You should know that Olric is a PA/EC (see [Consistency and Replication Model](#consistency-and-replication-model)) product. So if your network is stable, all the operations on key/value pairs are performed by a single cluster member. It means that you can be sure about the consistency when the cluster is stable. It's important to know that computer networks fail occasionally, processes crash and random GC pauses may happen. Many factors can lead a network partitioning. If you cannot tolerate losing strong consistency under network partitioning, you need to use a different tool for atomic operations. See [Hazelcast and the Mythical PA/EC System](https://dbmsmusings.blogspot.com/2017/10/hazelcast-and-mythical-paec-system.html) and [Jepsen Analysis on Hazelcast 3.8.3](https://hazelcast.com/blog/jepsen-analysis-hazelcast-3-8-3/) for more insight on this topic. #### DM.INCR DM.INCR atomically increments the number stored at key by delta. The return value is the new value after being incremented or an error. ``` DM.INCR dmap key delta ``` **Example:** ``` 127.0.0.1:3320> DM.INCR dmap key 10 (integer) 10 ``` **Return:** * **Integer reply:** the value of key after the increment. #### DM.DECR DM.DECR atomically decrements the number stored at key by delta. The return value is the new value after being incremented or an error. ``` DM.DECR dmap key delta ``` **Example:** ``` 127.0.0.1:3320> DM.DECR dmap key 10 (integer) 0 ``` **Return:** * **Integer reply:** the value of key after the increment. #### DM.GETPUT DM.GETPUT atomically sets key to value and returns the old value stored at the key. ``` DM.GETPUT dmap key value ``` **Example:** ``` 127.0.0.1:3320> DM.GETPUT dmap key value-1 (nil) 127.0.0.1:3320> DM.GETPUT dmap key value-2 "value-1" ``` **Return:** * **Bulk string reply**: the old value stored at the key. #### DM.INCRBYFLOAT DM.INCRBYFLOAT atomically increments the number stored at key by delta. The return value is the new value after being incremented or an error. ``` DM.INCRBYFLOAT dmap key delta ``` **Example:** ``` 127.0.0.1:3320> DM.PUT dmap key 10.50 OK 127.0.0.1:3320> DM.INCRBYFLOAT dmap key 0.1 "10.6" 127.0.0.1:3320> DM.PUT dmap key 5.0e3 OK 127.0.0.1:3320> DM.INCRBYFLOAT dmap key 2.0e2 "5200" ``` **Return:** * **Bulk string reply**: the value of key after the increment. ### Locking **Important:** The lock provided by DMap implementation is approximate and only to be used for non-critical purposes. The DMap implementation is already thread-safe to meet your thread safety requirements. When you want to have more control on the concurrency, you can use **DM.LOCK** command. Olric borrows the locking algorithm from Redis. Redis authors propose the following algorithm: > The command is a simple way to implement a locking system with Redis. > > A client can acquire the lock if the above command returns OK (or retry after some time if the command returns Nil), and remove the lock just using DEL. > > The lock will be auto-released after the expire time is reached. > > It is possible to make this system more robust modifying the unlock schema as follows: > > Instead of setting a fixed string, set a non-guessable large random string, called token. > Instead of releasing the lock with DEL, send a script that only removes the key if the value matches. > This avoids that a client will try to release the lock after the expire time deleting the key created by another client that acquired the lock later. Equivalent of `SETNX` command in Olric is `DM.PUT dmap key value NX`. DM.LOCK command are properly implements the algorithm which is proposed above. You should know that this implementation is subject to the clustering algorithm. So there is no guarantee about reliability in the case of network partitioning. I recommend the lock implementation to be used for efficiency purposes in general, instead of correctness. **Important note about consistency:** You should know that Olric is a PA/EC (see [Consistency and Replication Model](#consistency-and-replication-model)) product. So if your network is stable, all the operations on key/value pairs are performed by a single cluster member. It means that you can be sure about the consistency when the cluster is stable. It's important to know that computer networks fail occasionally, processes crash and random GC pauses may happen. Many factors can lead a network partitioning. If you cannot tolerate losing strong consistency under network partitioning, you need to use a different tool for locking. See [Hazelcast and the Mythical PA/EC System](https://dbmsmusings.blogspot.com/2017/10/hazelcast-and-mythical-paec-system.html) and [Jepsen Analysis on Hazelcast 3.8.3](https://hazelcast.com/blog/jepsen-analysis-hazelcast-3-8-3/) for more insight on this topic. #### DM.LOCK DM.LOCK sets a lock for the given key. The acquired lock is only valid for the key in this DMap. It returns immediately if it acquires the lock for the given key. Otherwise, it waits until deadline. DM.LOCK returns a token. You must keep that token to unlock the key. Using prefixed keys is highly recommended. If the key does already exist in the DMap, DM.LOCK will wait until the deadline is exceeded. ``` DM.LOCK dmap key seconds [ EX seconds | PX milliseconds ] ``` **Options:** * **EX** *seconds* -- Set the specified expire time, in seconds. * **PX** *milliseconds* -- Set the specified expire time, in milliseconds. **Example:** ``` 127.0.0.1:3320> DM.LOCK dmap lock.key 10 2363ec600be286cb10fbb35181efb029 ``` **Return:** * **Simple string reply:** a token to unlock or lease the lock. * **NOSUCHLOCK**: (error) returned when the requested lock does not exist. * **LOCKNOTACQUIRED**: (error) returned when the requested lock could not be acquired. #### DM.UNLOCK DM.UNLOCK releases an acquired lock for the given key. It returns `NOSUCHLOCK` if there is no lock for the given key. ``` DM.UNLOCK dmap key token ``` **Example:** ``` 127.0.0.1:3320> DM.UNLOCK dmap key 2363ec600be286cb10fbb35181efb029 OK ``` **Return:** * **Simple string reply:** OK if DM.UNLOCK was executed correctly. * **NOSUCHLOCK**: (error) returned when the lock does not exist. #### DM.LOCKLEASE DM.LOCKLEASE sets or updates the timeout of the acquired lock for the given key. It returns `NOSUCHLOCK` if there is no lock for the given key. DM.LOCKLEASE accepts seconds as timeout. ``` DM.LOCKLEASE dmap key token seconds ``` **Example:** ``` 127.0.0.1:3320> DM.LOCKLEASE dmap key 2363ec600be286cb10fbb35181efb029 100 OK ``` **Return:** * **Simple string reply:** OK if DM.UNLOCK was executed correctly. * **NOSUCHLOCK**: (error) returned when the lock does not exist. #### DM.PLOCKLEASE DM.PLOCKLEASE sets or updates the timeout of the acquired lock for the given key. It returns `NOSUCHLOCK` if there is no lock for the given key. DM.PLOCKLEASE accepts milliseconds as timeout. ``` DM.LOCKLEASE dmap key token milliseconds ``` **Example:** ``` 127.0.0.1:3320> DM.PLOCKLEASE dmap key 2363ec600be286cb10fbb35181efb029 1000 OK ``` **Return:** * **Simple string reply:** OK if DM.PLOCKLEASE was executed correctly. * **NOSUCHLOCK**: (error) returned when the lock does not exist. #### DM.SCAN DM.SCAN is a cursor based iterator. This means that at every call of the command, the server returns an updated cursor that the user needs to use as the cursor argument in the next call. An iteration starts when the cursor is set to 0, and terminates when the cursor returned by the server is 0. The iterator runs locally on every partition. So you need to know the partition count. If the returned cursor is 0 for a particular partition, you have to start scanning the next partition. ``` DM.SCAN partID dmap cursor [ MATCH pattern | COUNT count ] ``` **Example:** ``` 127.0.0.1:3320> DM.SCAN 3 bench 0 1) "96990" 2) 1) "memtier-2794837" 2) "memtier-8630933" 3) "memtier-6415429" 4) "memtier-7808686" 5) "memtier-3347072" 6) "memtier-4247791" 7) "memtier-3931982" 8) "memtier-7164719" 9) "memtier-4710441" 10) "memtier-8892916" 127.0.0.1:3320> DM.SCAN 3 bench 96990 1) "193499" 2) 1) "memtier-429905" 2) "memtier-1271812" 3) "memtier-7835776" 4) "memtier-2717575" 5) "memtier-95312" 6) "memtier-2155214" 7) "memtier-123931" 8) "memtier-2902510" 9) "memtier-2632291" 10) "memtier-1938450" ``` ### Publish-Subscribe **SUBSCRIBE**, **UNSUBSCRIBE** and **PUBLISH** implement the Publish/Subscribe messaging paradigm where senders are not programmed to send their messages to specific receivers. Rather, published messages are characterized into channels, without knowledge of what (if any) subscribers there may be. Subscribers express interest in one or more channels, and only receive messages that are of interest, without knowledge of what (if any) publishers there are. This decoupling of publishers and subscribers can allow for greater scalability and a more dynamic network topology. **Important note:** In an Olric cluster, clients can subscribe to every node, and can also publish to every other node. The cluster will make sure that published messages are forwarded as needed. *Source of this section: [https://redis.io/commands/?group=pubsub](https://redis.io/commands/?group=pubsub)* #### SUBSCRIBE Subscribes the client to the specified channels. ``` SUBSCRIBE channel [channel...] ``` Once the client enters the subscribed state it is not supposed to issue any other commands, except for additional **SUBSCRIBE**, **PSUBSCRIBE**, **UNSUBSCRIBE**, **PUNSUBSCRIBE**, **PING**, and **QUIT** commands. #### PSUBSCRIBE Subscribes the client to the given patterns. ``` PSUBSCRIBE pattern [ pattern ...] ``` Supported glob-style patterns: * `h?llo` subscribes to hello, hallo and hxllo * `h*llo` subscribes to hllo and heeeello * `h[ae]llo` subscribes to hello and hallo, but not hillo * Use **\\** to escape special characters if you want to match them verbatim. #### UNSUBSCRIBE Unsubscribes the client from the given channels, or from all of them if none is given. ``` UNSUBSCRIBE [channel [channel ...]] ``` When no channels are specified, the client is unsubscribed from all the previously subscribed channels. In this case, a message for every unsubscribed channel will be sent to the client. #### PUNSUBSCRIBE Unsubscribes the client from the given patterns, or from all of them if none is given. ``` PUNSUBSCRIBE [pattern [pattern ...]] ``` When no patterns are specified, the client is unsubscribed from all the previously subscribed patterns. In this case, a message for every unsubscribed pattern will be sent to the client. #### PUBSUB CHANNELS Lists the currently active channels. ``` PUBSUB CHANNELS [pattern] ``` An active channel is a Pub/Sub channel with one or more subscribers (excluding clients subscribed to patterns). If no pattern is specified, all the channels are listed, otherwise if pattern is specified only channels matching the specified glob-style pattern are listed. #### PUBSUB NUMPAT Returns the number of unique patterns that are subscribed to by clients (that are performed using the PSUBSCRIBE command). ``` PUBSUB NUMPAT ``` Note that this isn't the count of clients subscribed to patterns, but the total number of unique patterns all the clients are subscribed to. **Important note**: In an Olric cluster, clients can subscribe to every node, and can also publish to every other node. The cluster will make sure that published messages are forwarded as needed. That said, PUBSUB's replies in a cluster only report information from the node's Pub/Sub context, rather than the entire cluster. #### PUBSUB NUMSUB Returns the number of subscribers (exclusive of clients subscribed to patterns) for the specified channels. ``` PUBSUB NUMSUB [channel [channel ...]] ``` Note that it is valid to call this command without channels. In this case it will just return an empty list. **Important note**: In an Olric cluster, clients can subscribe to every node, and can also publish to every other node. The cluster will make sure that published messages are forwarded as needed. That said, PUBSUB's replies in a cluster only report information from the node's Pub/Sub context, rather than the entire cluster. #### QUIT Ask the server to close the connection. The connection is closed as soon as all pending replies have been written to the client. ``` QUIT ``` ### Cluster #### CLUSTER.ROUTINGTABLE CLUSTER.ROUTINGTABLE returns the latest view of the routing table. Simply, it's a data structure that maps partitions to members. ``` CLUSTER.ROUTINGTABLE ``` **Example:** ``` 127.0.0.1:3320> CLUSTER.ROUTINGTABLE 1) 1) (integer) 0 2) 1) "127.0.0.1:3320" 3) (empty array) 2) 1) (integer) 1 2) 1) "127.0.0.1:3320" 3) (empty array) 3) 1) (integer) 2 2) 1) "127.0.0.1:3320" 3) (empty array) ``` It returns an array of arrays. **Fields:** ``` 1) (integer) 0 <- Partition ID 2) 1) "127.0.0.1:3320" <- Array of the current and previous primary owners 3) (empty array) <- Array of backup owners. ``` #### CLUSTER.MEMBERS CLUSTER.MEMBERS returns an array of known members by the server. ``` CLUSTER.MEMBERS ``` **Example:** ``` 127.0.0.1:3320> CLUSTER.MEMBERS 1) 1) "127.0.0.1:3320" 2) (integer) 1652619388427137000 3) "true" ``` **Fields:** ``` 1) 1) "127.0.0.1:3320" <- Member's name in the cluster 2) (integer) 1652619388427137000 <-Member's birthedate 3) "true" <- Is cluster coordinator (the oldest node) ``` ### Others #### PING Returns PONG if no argument is provided, otherwise return a copy of the argument as a bulk. This command is often used to test if a connection is still alive, or to measure latency. ``` PING ``` #### STATS The STATS command returns information and statistics about the server in JSON format. See `stats/stats.go` file. ``` 127.0.0.1:3320> STATS ``` #### AUTH `AUTH` authenticates the client using the given password: ``` 127.0.0.1:3320> AUTH your-password OK ``` Unauthenticated clients get `NOAUTH` error: ``` 127.0.0.1:3320> DMAP.PUT dmap key value (error) NOAUTH Authentication required. ``` If you try to authenticate the client but the server is not configured, Olric returns the following error: ``` 127.0.0.1:3320> AUTH your-password (error) ERR AUTH called without any password configured for the default user. Are you sure your configuration is correct? ``` ## Configuration Olric supports both declarative and programmatic configurations. You can choose one of them depending on your needs. You should feel free to ask any questions about configuration and integration. Please see [Support](#support) section. ### Embedded-Member Mode #### Programmatic Configuration Olric provides a function to generate default configuration to use in embedded-member mode: ```go import "github.com/olric-data/olric/config" ... c := config.New("local") ``` The `New` function takes a parameter called `env`. It denotes the network environment and consumed by [hashicorp/memberlist](https://github.com/hashicorp/memberlist). Default configuration is good enough for distributed caching scenario. In order to see all configuration parameters, please take a look at [this](https://godoc.org/github.com/olric-data/olric/config). See [Sample Code](#sample-code) section for an introduction. #### Declarative configuration with YAML format You can also import configuration from a YAML file by using the `Load` function: ```go c, err := config.Load(path/to/olric.yaml) ``` A sample configuration file in YAML format can be found [here](https://github.com/olric-data/olric/blob/master/cmd/olric-server/olric-server.yaml). This may be the most appropriate way to manage the Olric configuration. ### Client-Server Mode Olric provides **olric-server** to implement client-server mode. olric-server gets a YAML file for the configuration. The most basic functionality of olric-server is that translating YAML configuration into Olric's configuration struct. A sample `olric-server.yaml` file is being provided [here](https://github.com/olric-data/olric/blob/master/cmd/olric-server/olric-server.yaml). ### Network Configuration In an Olric instance, there are two different TCP servers. One for Olric, and the other one is for memberlist. `BindAddr` is very critical to deploy a healthy Olric node. There are different scenarios: * You can freely set a domain name or IP address as `BindAddr` for both Olric and memberlist. Olric will resolve and use it to bind. * You can freely set `localhost`, `127.0.0.1` or `::1` as `BindAddr` in development environment for both Olric and memberlist. * You can freely set `0.0.0.0` as `BindAddr` for both Olric and memberlist. Olric will pick an IP address, if there is any. * If you don't set `BindAddr`, hostname will be used, and it will be resolved to get a valid IP address. * You can set a network interface by using `Config.Interface` and `Config.MemberlistInterface` fields. Olric will find an appropriate IP address for the given interfaces, if there is any. * You can set both `BindAddr` and interface parameters. In this case Olric will ensure that `BindAddr` is available on the given interface. You should know that Olric needs a single and stable IP address to function properly. If you don't know the IP address of the host at the deployment time, you can set `BindAddr` as `0.0.0.0`. Olric will very likely to find an IP address for you. ### Service Discovery Olric provides a service discovery interface which can be used to implement plugins. We currently have a bunch of service discovery plugins for automatic peer discovery on cloud environments: * [olric-data/olric-consul-plugin](https://github.com/olric-data/olric-consul-plugin) provides a plugin using Consul. * [olric-data/olric-cloud-plugin](https://github.com/olric-data/olric-cloud-plugin) provides a plugin for well-known cloud providers. Including Kubernetes. * [justinfx/olric-nats-plugin](https://github.com/justinfx/olric-nats-plugin) provides a plugin using nats.io In order to get more info about installation and configuration of the plugins, see their GitHub page. ### Timeouts Olric nodes supports setting `KeepAlivePeriod` on TCP sockets. **Server-side:** ##### config.KeepAlivePeriod KeepAlivePeriod denotes whether the operating system should send keep-alive messages on the connection. **Client-side:** ##### config.DialTimeout Timeout for TCP dial. The timeout includes name resolution, if required. When using TCP, and the host in the address parameter resolves to multiple IP addresses, the timeout is spread over each consecutive dial, such that each is given an appropriate fraction of the time to connect. ##### config.ReadTimeout Timeout for socket reads. If reached, commands will fail with a timeout instead of blocking. Use value -1 for no timeout and 0 for default. The default is config.DefaultReadTimeout ##### config.WriteTimeout Timeout for socket writes. If reached, commands will fail with a timeout instead of blocking. The default is config.DefaultWriteTimeout ## Architecture ### Overview Olric uses: * [hashicorp/memberlist](https://github.com/hashicorp/memberlist) for cluster membership and failure detection, * [buraksezer/consistent](https://github.com/buraksezer/consistent) for consistent hashing and load balancing, * [Redis Serialization Protocol](https://github.com/tidwall/redcon) for communication. Olric distributes data among partitions. Every partition is being owned by a cluster member and may have one or more backups for redundancy. When you read or write a DMap entry, you transparently talk to the partition owner. Each request hits the most up-to-date version of a particular data entry in a stable cluster. In order to find the partition which the key belongs to, Olric hashes the key and mod it with the number of partitions: ``` partID = MOD(hash result, partition count) ``` The partitions are being distributed among cluster members by using a consistent hashing algorithm. In order to get details, please see [buraksezer/consistent](https://github.com/buraksezer/consistent). When a new cluster is created, one of the instances is elected as the **cluster coordinator**. It manages the partition table: * When a node joins or leaves, it distributes the partitions and their backups among the members again, * Removes empty previous owners from the partition owners list, * Pushes the new partition table to all the members, * Pushes the partition table to the cluster periodically. Members propagate their birthdate(POSIX time in nanoseconds) to the cluster. The coordinator is the oldest member in the cluster. If the coordinator leaves the cluster, the second oldest member gets elected as the coordinator. Olric has a component called **rebalancer** which is responsible for keeping underlying data structures consistent: * Works on every node, * When a node joins or leaves, the cluster coordinator pushes the new partition table. Then, the **rebalancer** runs immediately and moves the partitions and backups to their new hosts, * Merges fragmented partitions. Partitions have a concept called **owners list**. When a node joins or leaves the cluster, a new primary owner may be assigned by the coordinator. At any time, a partition may have one or more partition owners. If a partition has two or more owners, this is called **fragmented partition**. The last added owner is called **primary owner**. Write operation is only done by the primary owner. The previous owners are only used for read and delete. When you read a key, the primary owner tries to find the key on itself, first. Then, queries the previous owners and backups, respectively. The delete operation works the same way. The data(distributed map objects) in the fragmented partition is moved slowly to the primary owner by the **rebalancer**. Until the move is done, the data remains available on the previous owners. The DMap methods use this list to query data on the cluster. *Please note that, 'multiple partition owners' is an undesirable situation and the **rebalancer** component is designed to fix that in a short time.* ### Consistency and Replication Model **Olric is an AP product** in the context of [CAP theorem](https://en.wikipedia.org/wiki/CAP_theorem), which employs the combination of primary-copy and [optimistic replication](https://en.wikipedia.org/wiki/Optimistic_replication) techniques. With optimistic replication, when the partition owner receives a write or delete operation for a key, applies it locally, and propagates it to the backup owners. This technique enables Olric clusters to offer high throughput. However, due to temporary situations in the system, such as network failure, backup owners can miss some updates and diverge from the primary owner. If a partition owner crashes while there is an inconsistency between itself and the backups, strong consistency of the data can be lost. Two types of backup replication are available: **sync** and **async**. Both types are still implementations of the optimistic replication model. * **sync**: Blocks until write/delete operation is applied by backup owners. * **async**: Just fire & forget. #### Last-write-wins conflict resolution Every time a piece of data is written to Olric, a timestamp is attached by the client. Then, when Olric has to deal with conflict data in the case of network partitioning, it simply chooses the data with the most recent timestamp. This called LWW conflict resolution policy. #### PACELC Theorem From Wikipedia: > In theoretical computer science, the [PACELC theorem](https://en.wikipedia.org/wiki/PACELC_theorem) is an extension to the [CAP theorem](https://en.wikipedia.org/wiki/CAP_theorem). It states that in case of network partitioning (P) in a > distributed computer system, one has to choose between availability (A) and consistency (C) (as per the CAP theorem), but else (E), even when the system is > running normally in the absence of partitions, one has to choose between latency (L) and consistency (C). In the context of PACELC theorem, Olric is a **PA/EC** product. It means that Olric is considered to be **consistent** data store if the network is stable. Because the key space is divided between partitions and every partition is controlled by its primary owner. All operations on DMaps are redirected to the partition owner. In the case of network partitioning, Olric chooses **availability** over consistency. So that you can still access some parts of the cluster when the network is unreliable, but the cluster may return inconsistent results. Olric implements read-repair and quorum based voting system to deal with inconsistencies in the DMaps. Readings on PACELC theorem: * [Please stop calling databases CP or AP](https://martin.kleppmann.com/2015/05/11/please-stop-calling-databases-cp-or-ap.html) * [Problems with CAP, and Yahoo’s little known NoSQL system](https://dbmsmusings.blogspot.com/2010/04/problems-with-cap-and-yahoos-little.html) * [A Critique of the CAP Theorem](https://arxiv.org/abs/1509.05393) * [Hazelcast and the Mythical PA/EC System](https://dbmsmusings.blogspot.com/2017/10/hazelcast-and-mythical-paec-system.html) #### Read-Repair on DMaps Read repair is a feature that allows for inconsistent data to be fixed at query time. Olric tracks every write operation with a timestamp value and assumes that the latest write operation is the valid one. When you want to access a key/value pair, the partition owner retrieves all available copies for that pair and compares the timestamp values. The latest one is the winner. If there is some outdated version of the requested pair, the primary owner propagates the latest version of the pair. Read-repair is disabled by default for the sake of performance. If you have a use case that requires a more strict consistency control than a distributed caching scenario, you can enable read-repair via the configuration. #### Quorum-based replica control Olric implements Read/Write quorum to keep the data in a consistent state. When you start a write operation on the cluster and write quorum (W) is 2, the partition owner tries to write the given key/value pair on its own data storage and on the replica nodes. If the number of successful write operations is below W, the primary owner returns `ErrWriteQuorum`. The read flow is the same: if you have R=2 and the owner only access one of the replicas, it returns `ErrReadQuorum`. #### Simple Split-Brain Protection Olric implements a technique called *majority quorum* to manage split-brain conditions. If a network partitioning occurs, and some members lost the connection to rest of the cluster, they immediately stops functioning and return an error to incoming requests. This behaviour is controlled by `MemberCountQuorum` parameter. It's default `1`. When the network healed, the stopped nodes joins again the cluster and fragmented partitions is merged by their primary owners in accordance with *LWW policy*. Olric also implements an *ownership report* mechanism to fix inconsistencies in partition distribution after a partitioning event. ### Eviction Olric supports different policies to evict keys from distributed maps. #### Expire with TTL Olric implements TTL eviction policy. It shares the same algorithm with [Redis](https://redis.io/commands/expire#appendix-redis-expires): > Periodically Redis tests a few keys at random among keys with an expire set. All the keys that are already expired are deleted from the keyspace. > > Specifically this is what Redis does 10 times per second: > > * Test 20 random keys from the set of keys with an associated expire. > * Delete all the keys found expired. > * If more than 25% of keys were expired, start again from step 1. > > This is a trivial probabilistic algorithm, basically the assumption is that our sample is representative of the whole key space, and we continue to expire until the percentage of keys that are likely to be expired is under 25% When a client tries to access a key, Olric returns `ErrKeyNotFound` if the key is found to be timed out. A background task evicts keys with the algorithm described above. #### Expire with MaxIdleDuration Maximum time for each entry to stay idle in the DMap. It limits the lifetime of the entries relative to the time of the last read or write access performed on them. The entries whose idle period exceeds this limit are expired and evicted automatically. An entry is idle if no Get, Put, PutEx, Expire, PutIf, PutIfEx on it. Configuration of MaxIdleDuration feature varies by preferred deployment method. #### Expire with LRU Olric implements LRU eviction method on DMaps. Approximated LRU algorithm is borrowed from Redis. The Redis authors proposes the following algorithm: > It is important to understand that the eviction process works like this: > > * A client runs a new command, resulting in more data added. > * Redis checks the memory usage, and if it is greater than the maxmemory limit , it evicts keys according to the policy. > * A new command is executed, and so forth. > > So we continuously cross the boundaries of the memory limit, by going over it, and then by evicting keys to return back under the limits. > > If a command results in a lot of memory being used (like a big set intersection stored into a new key) for some time the memory > limit can be surpassed by a noticeable amount. > > **Approximated LRU algorithm** > > Redis LRU algorithm is not an exact implementation. This means that Redis is not able to pick the best candidate for eviction, > that is, the access that was accessed the most in the past. Instead it will try to run an approximation of the LRU algorithm, > by sampling a small number of keys, and evicting the one that is the best (with the oldest access time) among the sampled keys. Olric tracks access time for every DMap instance. Then it picks and sorts some configurable amount of keys to select keys for eviction. Every node runs this algorithm independently. The access log is moved along with the partition when a network partition is occured. #### Configuration of eviction mechanisms Here is a simple configuration block for `olric-server.yaml`: ``` cache: numEvictionWorkers: 1 maxIdleDuration: "" ttlDuration: "100s" maxKeys: 100000 maxInuse: 1000000 # in bytes lRUSamples: 10 evictionPolicy: "LRU" # NONE/LRU ``` You can also set cache configuration per DMap. Here is a simple configuration for a DMap named `mydmap`: ``` dmaps: mydmap: maxIdleDuration: "60s" ttlDuration: "300s" maxKeys: 500000 # in-bytes lRUSamples: 20 evictionPolicy: "NONE" # NONE/LRU ``` If you prefer embedded-member deployment scenario, please take a look at [config#CacheConfig](https://godoc.org/github.com/olric-data/olric/config#CacheConfig) and [config#DMapCacheConfig](https://godoc.org/github.com/olric-data/olric/config#DMapCacheConfig) for the configuration. ### Lock Implementation The DMap implementation is already thread-safe to meet your thread safety requirements. When you want to have more control on the concurrency, you can use **LockWithTimeout** and **Lock** methods. Olric borrows the locking algorithm from Redis. Redis authors propose the following algorithm: > The command is a simple way to implement a locking system with Redis. > > A client can acquire the lock if the above command returns OK (or retry after some time if the command returns Nil), and remove the lock just using DEL. > > The lock will be auto-released after the expire time is reached. > > It is possible to make this system more robust modifying the unlock schema as follows: > > Instead of setting a fixed string, set a non-guessable large random string, called token. > Instead of releasing the lock with DEL, send a script that only removes the key if the value matches. > This avoids that a client will try to release the lock after the expire time deleting the key created by another client that acquired the lock later. Equivalent of`SETNX` command in Olric is `PutIf(key, value, IfNotFound)`. Lock and LockWithTimeout commands are properly implements the algorithm which is proposed above. You should know that this implementation is subject to the clustering algorithm. So there is no guarantee about reliability in the case of network partitioning. I recommend the lock implementation to be used for efficiency purposes in general, instead of correctness. **Important note about consistency:** You should know that Olric is a PA/EC (see [Consistency and Replication Model](#consistency-and-replication-model)) product. So if your network is stable, all the operations on key/value pairs are performed by a single cluster member. It means that you can be sure about the consistency when the cluster is stable. It's important to know that computer networks fail occasionally, processes crash and random GC pauses may happen. Many factors can lead a network partitioning. If you cannot tolerate losing strong consistency under network partitioning, you need to use a different tool for locking. See [Hazelcast and the Mythical PA/EC System](https://dbmsmusings.blogspot.com/2017/10/hazelcast-and-mythical-paec-system.html) and [Jepsen Analysis on Hazelcast 3.8.3](https://hazelcast.com/blog/jepsen-analysis-hazelcast-3-8-3/) for more insight on this topic. ### Storage Engine Olric implements a GC-friendly storage engine to store large amounts of data on RAM. Basically, it applies an append-only log file approach with indexes. Olric inserts key/value pairs into pre-allocated byte slices (table in Olric terminology) and indexes that memory region by using Golang's built-in map. The data type of this map is `map[uint64]uint64`. When a pre-allocated byte slice is full Olric allocates a new one and continues inserting the new data into it. This design greatly reduces the write latency. When you want to read a key/value pair from the Olric cluster, it scans the related DMap fragment by iterating over the indexes(implemented by the built-in map). The number of allocated byte slices should be small. So Olric would find the key immediately but technically, the read performance depends on the number of keys in the fragment. The effect of this design on the read performance is negligible. The size of the pre-allocated byte slices is configurable. ## Samples In this section, you can find code snippets for various scenarios. ### Embedded-member scenario #### Distributed map ```go package main import ( "context" "fmt" "log" "time" "github.com/olric-data/olric" "github.com/olric-data/olric/config" ) func main() { // Sample for Olric v0.7.x // Deployment scenario: embedded-member // This creates a single-node Olric cluster. It's good enough for experimenting. // config.New returns a new config.Config with sane defaults. Available values for env: // local, lan, wan c := config.New("local") // Callback function. It's called when this node is ready to accept connections. ctx, cancel := context.WithCancel(context.Background()) c.Started = func() { defer cancel() log.Println("[INFO] Olric is ready to accept connections") } // Create a new Olric instance. db, err := olric.New(c) if err != nil { log.Fatalf("Failed to create Olric instance: %v", err) } // Start the instance. It will form a single-node cluster. go func() { // Call Start at background. It's a blocker call. err = db.Start() if err != nil { log.Fatalf("olric.Start returned an error: %v", err) } }() <-ctx.Done() // In embedded-member scenario, you can use the EmbeddedClient. It implements // the Client interface. e := db.NewEmbeddedClient() dm, err := e.NewDMap("bucket-of-arbitrary-items") if err != nil { log.Fatalf("olric.NewDMap returned an error: %v", err) } ctx, cancel = context.WithCancel(context.Background()) // Magic starts here! fmt.Println("##") fmt.Println("Simple Put/Get on a DMap instance:") err = dm.Put(ctx, "my-key", "Olric Rocks!") if err != nil { log.Fatalf("Failed to call Put: %v", err) } gr, err := dm.Get(ctx, "my-key") if err != nil { log.Fatalf("Failed to call Get: %v", err) } // Olric uses the Redis serialization format. value, err := gr.String() if err != nil { log.Fatalf("Failed to read Get response: %v", err) } fmt.Println("Response for my-key:", value) fmt.Println("##") // Don't forget the call Shutdown when you want to leave the cluster. ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second) defer cancel() err = db.Shutdown(ctx) if err != nil { log.Printf("Failed to shutdown Olric: %v", err) } } ``` #### Publish-Subscribe ```go package main import ( "context" "fmt" "log" "time" "github.com/olric-data/olric" "github.com/olric-data/olric/config" ) func main() { // Sample for Olric v0.7.x // Deployment scenario: embedded-member // This creates a single-node Olric cluster. It's good enough for experimenting. // config.New returns a new config.Config with sane defaults. Available values for env: // local, lan, wan c := config.New("local") // Callback function. It's called when this node is ready to accept connections. ctx, cancel := context.WithCancel(context.Background()) c.Started = func() { defer cancel() log.Println("[INFO] Olric is ready to accept connections") } // Create a new Olric instance. db, err := olric.New(c) if err != nil { log.Fatalf("Failed to create Olric instance: %v", err) } // Start the instance. It will form a single-node cluster. go func() { // Call Start at background. It's a blocker call. err = db.Start() if err != nil { log.Fatalf("olric.Start returned an error: %v", err) } }() <-ctx.Done() // In embedded-member scenario, you can use the EmbeddedClient. It implements // the Client interface. e := db.NewEmbeddedClient() ps, err := e.NewPubSub() if err != nil { log.Fatalf("olric.NewPubSub returned an error: %v", err) } ctx, cancel = context.WithCancel(context.Background()) // Olric implements a drop-in replacement of Redis Publish-Subscribe messaging // system. PubSub client is just a thin layer around go-redis/redis. rps := ps.Subscribe(ctx, "my-channel") // Get a message to read messages from my-channel msg := rps.Channel() go func() { // Publish a message here. _, err := ps.Publish(ctx, "my-channel", "Olric Rocks!") if err != nil { log.Fatalf("PubSub.Publish returned an error: %v", err) } }() // Consume messages rm := <-msg fmt.Printf("Received message: \"%s\" from \"%s\"", rm.Channel, rm.Payload) // Don't forget the call Shutdown when you want to leave the cluster. ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second) defer cancel() err = e.Close(ctx) if err != nil { log.Printf("Failed to close EmbeddedClient: %v", err) } } ``` ### Client-Server scenario #### Distributed map ```go package main import ( "context" "fmt" "log" "time" "github.com/olric-data/olric" ) func main() { // Sample for Olric v0.7.x // Deployment scenario: client-server // NewClusterClient takes a list of the nodes. This list may only contain a // load balancer address. Please note that Olric nodes will calculate the partition owner // and proxy the incoming requests. c, err := olric.NewClusterClient([]string{"localhost:3320"}) if err != nil { log.Fatalf("olric.NewClusterClient returned an error: %v", err) } // In client-server scenario, you can use the ClusterClient. It implements // the Client interface. dm, err := c.NewDMap("bucket-of-arbitrary-items") if err != nil { log.Fatalf("olric.NewDMap returned an error: %v", err) } ctx, cancel := context.WithCancel(context.Background()) // Magic starts here! fmt.Println("##") fmt.Println("Simple Put/Get on a DMap instance:") err = dm.Put(ctx, "my-key", "Olric Rocks!") if err != nil { log.Fatalf("Failed to call Put: %v", err) } gr, err := dm.Get(ctx, "my-key") if err != nil { log.Fatalf("Failed to call Get: %v", err) } // Olric uses the Redis serialization format. value, err := gr.String() if err != nil { log.Fatalf("Failed to read Get response: %v", err) } fmt.Println("Response for my-key:", value) fmt.Println("##") // Don't forget the call Shutdown when you want to leave the cluster. ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second) defer cancel() err = c.Close(ctx) if err != nil { log.Printf("Failed to close ClusterClient: %v", err) } } ``` ### SCAN on DMaps ```go package main import ( "context" "fmt" "log" "time" "github.com/olric-data/olric" "github.com/olric-data/olric/config" ) func main() { // Sample for Olric v0.7.x // Deployment scenario: embedded-member // This creates a single-node Olric cluster. It's good enough for experimenting. // config.New returns a new config.Config with sane defaults. Available values for env: // local, lan, wan c := config.New("local") // Callback function. It's called when this node is ready to accept connections. ctx, cancel := context.WithCancel(context.Background()) c.Started = func() { defer cancel() log.Println("[INFO] Olric is ready to accept connections") } // Create a new Olric instance. db, err := olric.New(c) if err != nil { log.Fatalf("Failed to create Olric instance: %v", err) } // Start the instance. It will form a single-node cluster. go func() { // Call Start at background. It's a blocker call. err = db.Start() if err != nil { log.Fatalf("olric.Start returned an error: %v", err) } }() <-ctx.Done() // In embedded-member scenario, you can use the EmbeddedClient. It implements // the Client interface. e := db.NewEmbeddedClient() dm, err := e.NewDMap("bucket-of-arbitrary-items") if err != nil { log.Fatalf("olric.NewDMap returned an error: %v", err) } ctx, cancel = context.WithCancel(context.Background()) // Magic starts here! fmt.Println("##") fmt.Println("Insert 10 keys") var key string for i := 0; i < 10; i++ { if i%2 == 0 { key = fmt.Sprintf("even:%d", i) } else { key = fmt.Sprintf("odd:%d", i) } err = dm.Put(ctx, key, nil) if err != nil { log.Fatalf("Failed to call Put: %v", err) } } i, err := dm.Scan(ctx) if err != nil { log.Fatalf("Failed to call Scan: %v", err) } fmt.Println("Iterate over all the keys") for i.Next() { fmt.Println(">> Key", i.Key()) } i.Close() i, err = dm.Scan(ctx, olric.Match("^even:")) if err != nil { log.Fatalf("Failed to call Scan: %v", err) } fmt.Println("\n\nScan with regex: ^even:") for i.Next() { fmt.Println(">> Key", i.Key()) } i.Close() // Don't forget the call Shutdown when you want to leave the cluster. ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second) defer cancel() err = db.Shutdown(ctx) if err != nil { log.Printf("Failed to shutdown Olric: %v", err) } } ``` #### Publish-Subscribe ```go package main import ( "context" "fmt" "log" "time" "github.com/olric-data/olric" ) func main() { // Sample for Olric v0.7.x // Deployment scenario: client-server // NewClusterClient takes a list of the nodes. This list may only contain a // load balancer address. Please note that Olric nodes will calculate the partition owner // and proxy the incoming requests. c, err := olric.NewClusterClient([]string{"localhost:3320"}) if err != nil { log.Fatalf("olric.NewClusterClient returned an error: %v", err) } // In client-server scenario, you can use the ClusterClient. It implements // the Client interface. ps, err := c.NewPubSub() if err != nil { log.Fatalf("olric.NewPubSub returned an error: %v", err) } ctx, cancel := context.WithCancel(context.Background()) // Olric implements a drop-in replacement of Redis Publish-Subscribe messaging // system. PubSub client is just a thin layer around go-redis/redis. rps := ps.Subscribe(ctx, "my-channel") // Get a message to read messages from my-channel msg := rps.Channel() go func() { // Publish a message here. _, err := ps.Publish(ctx, "my-channel", "Olric Rocks!") if err != nil { log.Fatalf("PubSub.Publish returned an error: %v", err) } }() // Consume messages rm := <-msg fmt.Printf("Received message: \"%s\" from \"%s\"", rm.Channel, rm.Payload) // Don't forget the call Shutdown when you want to leave the cluster. ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second) defer cancel() err = c.Close(ctx) if err != nil { log.Printf("Failed to close ClusterClient: %v", err) } } ``` ## Contributions Please don't hesitate to fork the project and send a pull request or just e-mail me to ask questions and share ideas. ## License The Apache License, Version 2.0 - see LICENSE for more details. ## About the name The inner voice of Turgut Özben who is the main character of [Oğuz Atay's masterpiece -The Disconnected-](https://www.themodernnovel.org/asia/other-asia/turkey/oguz-atay/the-disconnected/). ================================================ FILE: auth.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package olric import ( "errors" "github.com/olric-data/olric/internal/protocol" "github.com/olric-data/olric/internal/server" "github.com/tidwall/redcon" ) // authCommandHandler handles authentication requests sent by clients and verifies the provided password for access. func (db *Olric) authCommandHandler(conn redcon.Conn, cmd redcon.Command) { authCmd, err := protocol.ParseAuthCommand(cmd) if err != nil { protocol.WriteError(conn, err) return } if !db.config.Authentication.Enabled() { protocol.WriteError(conn, errors.New("AUTH called without any password configured for the default user. Are you sure your configuration is correct?")) return } if authCmd.Password == db.config.Authentication.Password { ctx := conn.Context().(*server.ConnContext) ctx.SetAuthenticated(true) conn.WriteString(protocol.StatusOK) return } protocol.WriteError(conn, ErrWrongPass) } ================================================ FILE: auth_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package olric import ( "context" "testing" "github.com/olric-data/olric/config" "github.com/olric-data/olric/internal/testutil" "github.com/stretchr/testify/require" ) func TestAuthCommandHandler_WithPassword(t *testing.T) { cluster := newTestOlricCluster(t) testConfig := testutil.NewConfig() testConfig.Authentication = &config.Authentication{ Password: "test-password", } db := cluster.addMemberWithConfig(t, testConfig) expectedMessage := "error while discovering the cluster members: wrong password" ctx := context.Background() t.Run("With correct credentials", func(t *testing.T) { c, err := NewClusterClient([]string{db.name}, WithPassword("test-password")) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() response, err := c.Ping(ctx, db.rt.This().String(), "") require.NoError(t, err) require.Equal(t, DefaultPingResponse, response) }) t.Run("With wrong credentials", func(t *testing.T) { _, err := NewClusterClient([]string{db.name}, WithPassword("wrong")) require.ErrorContains(t, err, expectedMessage) }) t.Run("Without credentials", func(t *testing.T) { _, err := NewClusterClient([]string{db.name}, WithPassword("wrong")) require.ErrorContains(t, err, expectedMessage) }) } func TestAuthCommandHandler_Auth_Disabled(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) _, err := NewClusterClient([]string{db.name}, WithPassword("test-password")) require.ErrorContains(t, err, "error while discovering the cluster members: AUTH called without any password configured for the default user. Are you sure your configuration is correct?") } ================================================ FILE: client.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package olric import ( "context" "time" "github.com/olric-data/olric/internal/dmap" "github.com/olric-data/olric/pkg/storage" "github.com/olric-data/olric/stats" ) const DefaultScanCount = 10 // Member denotes a member of the Olric cluster. type Member struct { // Member name in the cluster. It's also host:port of the node. Name string // ID of the Member in the cluster. Hash of Name and Birthdate of the member ID uint64 // Birthdate of the member in nanoseconds. Birthdate int64 // Role of the member in the cluster. There is only one coordinator member // in a healthy cluster. Coordinator bool } // Iterator defines an interface to implement iterators on the distributed maps. type Iterator interface { // Next returns true if there is more key in the iterator implementation. // Otherwise, it returns false. Next() bool // Key returns a key name from the distributed map. Key() string // Close stops the iteration and releases allocated resources. Close() } // LockContext interface defines methods to manage locks on distributed maps. type LockContext interface { // Unlock releases an acquired lock for the given key. It returns ErrNoSuchLock // if there is no lock for the given key. Unlock(ctx context.Context) error // Lease sets or updates the timeout of the acquired lock for the given key. // It returns ErrNoSuchLock if there is no lock for the given key. Lease(ctx context.Context, duration time.Duration) error } // PutOption is a function for define options to control behavior of the Put command. type PutOption func(*dmap.PutConfig) // EX sets the specified expire time, in seconds. func EX(ex time.Duration) PutOption { return func(cfg *dmap.PutConfig) { cfg.HasEX = true cfg.EX = ex } } // PX sets the specified expire time, in milliseconds. func PX(px time.Duration) PutOption { return func(cfg *dmap.PutConfig) { cfg.HasPX = true cfg.PX = px } } // EXAT sets the specified Unix time at which the key will expire, in seconds. func EXAT(exat time.Duration) PutOption { return func(cfg *dmap.PutConfig) { cfg.HasEXAT = true cfg.EXAT = exat } } // PXAT sets the specified Unix time at which the key will expire, in milliseconds. func PXAT(pxat time.Duration) PutOption { return func(cfg *dmap.PutConfig) { cfg.HasPXAT = true cfg.PXAT = pxat } } // NX only sets the key if it does not already exist. func NX() PutOption { return func(cfg *dmap.PutConfig) { cfg.HasNX = true } } // XX only sets the key if it already exists. func XX() PutOption { return func(cfg *dmap.PutConfig) { cfg.HasXX = true } } type dmapConfig struct { storageEntryImplementation func() storage.Entry } // DMapOption is a function for defining options to control behavior of distributed map instances. type DMapOption func(*dmapConfig) // StorageEntryImplementation sets and encoder/decoder implementation for your choice of storage engine. func StorageEntryImplementation(e func() storage.Entry) DMapOption { return func(cfg *dmapConfig) { cfg.storageEntryImplementation = e } } // ScanOption is a function for defining options to control behavior of the SCAN command. type ScanOption func(*dmap.ScanConfig) // Count is the user specified the amount of work that should be done at every call in order to // retrieve elements from the distributed map. This is just a hint for the implementation, // however generally speaking this is what you could expect most of the time from the implementation. // The default value is 10. func Count(c int) ScanOption { return func(cfg *dmap.ScanConfig) { cfg.HasCount = true cfg.Count = c } } // Match is used for using regular expressions on keys. See https://pkg.go.dev/regexp func Match(s string) ScanOption { return func(cfg *dmap.ScanConfig) { cfg.HasMatch = true cfg.Match = s } } // DMap defines methods to access and manipulate distributed maps. type DMap interface { // Name exposes name of the DMap. Name() string // Put sets the value for the given key. It overwrites any previous value for // that key, and it's thread-safe. The key has to be a string. value type is arbitrary. // It is safe to modify the contents of the arguments after Put returns but not before. Put(ctx context.Context, key string, value interface{}, options ...PutOption) error // Get gets the value for the given key. It returns ErrKeyNotFound if the DB // does not contain the key. It's thread-safe. It is safe to modify the contents // of the returned value. See GetResponse for the details. Get(ctx context.Context, key string) (*GetResponse, error) // Delete deletes values for the given keys. Delete will not return error // if key doesn't exist. It's thread-safe. It is safe to modify the contents // of the argument after Delete returns. Delete(ctx context.Context, keys ...string) (int, error) // Incr atomically increments the key by delta. The return value is the new value // after being incremented or an error. Incr(ctx context.Context, key string, delta int) (int, error) // Decr atomically decrements the key by delta. The return value is the new value // after being decremented or an error. Decr(ctx context.Context, key string, delta int) (int, error) // GetPut atomically sets the key to value and returns the old value stored at key. It returns nil if there is no // previous value. GetPut(ctx context.Context, key string, value interface{}) (*GetResponse, error) // IncrByFloat atomically increments the key by delta. The return value is the new value // after being incremented or an error. IncrByFloat(ctx context.Context, key string, delta float64) (float64, error) // Expire updates the expiry for the given key. It returns ErrKeyNotFound if // the DB does not contain the key. It's thread-safe. Expire(ctx context.Context, key string, timeout time.Duration) error // Lock sets a lock for the given key. Acquired lock is only for the key in // this dmap. // // It returns immediately if it acquires the lock for the given key. Otherwise, // it waits until deadline. // // You should know that the locks are approximate, and only to be used for // non-critical purposes. Lock(ctx context.Context, key string, deadline time.Duration) (LockContext, error) // LockWithTimeout sets a lock for the given key. If the lock is still unreleased // the end of given period of time, // it automatically releases the lock. Acquired lock is only for the key in // this dmap. // // It returns immediately if it acquires the lock for the given key. Otherwise, // it waits until deadline. // // You should know that the locks are approximate, and only to be used for // non-critical purposes. LockWithTimeout(ctx context.Context, key string, timeout, deadline time.Duration) (LockContext, error) // Scan returns an iterator to loop over the keys. // // Available scan options: // // * Count // * Match Scan(ctx context.Context, options ...ScanOption) (Iterator, error) // Destroy flushes the given DMap on the cluster. You should know that there // is no global lock on DMaps. So if you call Put/PutEx and Destroy methods // concurrently on the cluster, Put call may set new values to the DMap. Destroy(ctx context.Context) error // Pipeline is a mechanism to realise Redis Pipeline technique. // // Pipelining is a technique to extremely speed up processing by packing // operations to batches, send them at once to Redis and read a replies in a // singe step. // See https://redis.io/topics/pipelining // // Pay attention, that Pipeline is not a transaction, so you can get unexpected // results in case of big pipelines and small read/write timeouts. // Redis client has retransmission logic in case of timeouts, pipeline // can be retransmitted and commands can be executed more than once. Pipeline(opts ...PipelineOption) (*DMapPipeline, error) // Close stops background routines and frees allocated resources. Close(ctx context.Context) error } // PipelineOption is a function for defining options to control behavior of the Pipeline command. type PipelineOption func(pipeline *DMapPipeline) // PipelineConcurrency is a PipelineOption controlling the number of concurrent goroutines. func PipelineConcurrency(concurrency int) PipelineOption { return func(dp *DMapPipeline) { dp.concurrency = concurrency } } type statsConfig struct { CollectRuntime bool } // StatsOption is a function for defining options to control behavior of the STATS command. type StatsOption func(*statsConfig) // CollectRuntime is a StatsOption for collecting Go runtime statistics from a cluster member. func CollectRuntime() StatsOption { return func(cfg *statsConfig) { cfg.CollectRuntime = true } } type pubsubConfig struct { Address string } // ToAddress is a PubSubOption for using a specific cluster member to publish messages to a channel. func ToAddress(addr string) PubSubOption { return func(cfg *pubsubConfig) { cfg.Address = addr } } // PubSubOption is a function for defining options to control behavior of the Publish-Subscribe service. type PubSubOption func(option *pubsubConfig) // Client is an interface that denotes an Olric client. type Client interface { // NewDMap returns a new DMap client with the given options. NewDMap(name string, options ...DMapOption) (DMap, error) // NewPubSub returns a new PubSub client with the given options. NewPubSub(options ...PubSubOption) (*PubSub, error) // Stats returns stats.Stats with the given options. Stats(ctx context.Context, address string, options ...StatsOption) (stats.Stats, error) // Ping sends a ping message to an Olric node. Returns PONG if message is empty, // otherwise return a copy of the message as a bulk. This command is often used to test // if a connection is still alive, or to measure latency. Ping(ctx context.Context, address, message string) (string, error) // RoutingTable returns the latest version of the routing table. RoutingTable(ctx context.Context) (RoutingTable, error) // Members returns a thread-safe list of cluster members. Members(ctx context.Context) ([]Member, error) // RefreshMetadata fetches a list of available members and the latest routing // table version. It also closes stale clients, if there are any. RefreshMetadata(ctx context.Context) error // Close stops background routines and frees allocated resources. Close(ctx context.Context) error } ================================================ FILE: cluster.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package olric import ( "context" "fmt" "strconv" "github.com/olric-data/olric/internal/protocol" "github.com/tidwall/redcon" ) type Route struct { PrimaryOwners []string ReplicaOwners []string } type RoutingTable map[uint64]Route func mapToRoutingTable(slice []interface{}) (RoutingTable, error) { rt := make(RoutingTable) for _, raw := range slice { item := raw.([]interface{}) rawPartID, rawPrimaryOwners, rawReplicaOwners := item[0], item[1], item[2] var partID uint64 switch rawPartID.(type) { case int64: partID = uint64(rawPartID.(int64)) case string: raw, err := strconv.ParseUint(rawPartID.(string), 10, 64) if err != nil { return nil, fmt.Errorf("invalid partition id: %v: %w", rawPartID, err) } partID = raw default: return nil, fmt.Errorf("invalid partition id: %v", rawPartID) } r := Route{} primaryOwners, ok := rawPrimaryOwners.([]interface{}) if !ok { return nil, fmt.Errorf("invalid primary owners: %v", rawPrimaryOwners) } for _, rawOwner := range primaryOwners { owner, ok := rawOwner.(string) if !ok { return nil, fmt.Errorf("invalid owner: %v", owner) } r.PrimaryOwners = append(r.PrimaryOwners, owner) } replicaOwners, ok := rawReplicaOwners.([]interface{}) if !ok { return nil, fmt.Errorf("invalid replica owners: %v", rawPrimaryOwners) } for _, rawOwner := range replicaOwners { owner, ok := rawOwner.(string) if !ok { return nil, fmt.Errorf("invalid owner: %v", owner) } r.ReplicaOwners = append(r.ReplicaOwners, owner) } rt[partID] = r } return rt, nil } func (db *Olric) clusterRoutingTableCommandHandler(conn redcon.Conn, cmd redcon.Command) { _, err := protocol.ParseClusterRoutingTable(cmd) if err != nil { protocol.WriteError(conn, err) return } coordinator := db.rt.Discovery().GetCoordinator() if coordinator.CompareByID(db.rt.This()) { conn.WriteArray(int(db.config.PartitionCount)) rt := db.fillRoutingTable() for partID := uint64(0); partID < db.config.PartitionCount; partID++ { conn.WriteArray(3) conn.WriteUint64(partID) r := rt[partID] primaryOwners := r.PrimaryOwners conn.WriteArray(len(primaryOwners)) for _, owner := range primaryOwners { conn.WriteBulkString(owner) } replicaOwners := r.ReplicaOwners conn.WriteArray(len(replicaOwners)) for _, owner := range replicaOwners { conn.WriteBulkString(owner) } } return } // Redirect to the cluster coordinator rtCmd := protocol.NewClusterRoutingTable().Command(db.ctx) rc := db.client.Get(coordinator.String()) err = rc.Process(db.ctx, rtCmd) if err != nil { protocol.WriteError(conn, err) return } slice, err := rtCmd.Slice() if err != nil { protocol.WriteError(conn, err) return } conn.WriteAny(slice) } func (db *Olric) fillRoutingTable() RoutingTable { rt := make(RoutingTable) for partID := uint64(0); partID < db.config.PartitionCount; partID++ { r := Route{} primaryOwners := db.primary.PartitionOwnersByID(partID) for _, owner := range primaryOwners { r.PrimaryOwners = append(r.PrimaryOwners, owner.String()) } replicaOwners := db.backup.PartitionOwnersByID(partID) for _, owner := range replicaOwners { r.ReplicaOwners = append(r.ReplicaOwners, owner.String()) } rt[partID] = r } return rt } func (db *Olric) routingTable(ctx context.Context) (RoutingTable, error) { coordinator := db.rt.Discovery().GetCoordinator() if coordinator.CompareByID(db.rt.This()) { return db.fillRoutingTable(), nil } rtCmd := protocol.NewClusterRoutingTable().Command(ctx) rc := db.client.Get(coordinator.String()) err := rc.Process(ctx, rtCmd) if err != nil { return nil, err } slice, err := rtCmd.Slice() if err != nil { return nil, err } return mapToRoutingTable(slice) } func (db *Olric) clusterMembersCommandHandler(conn redcon.Conn, cmd redcon.Command) { _, err := protocol.ParseClusterMembers(cmd) if err != nil { protocol.WriteError(conn, err) return } coordinator := db.rt.Discovery().GetCoordinator() members := db.rt.Discovery().GetMembers() conn.WriteArray(len(members)) for _, member := range members { conn.WriteArray(3) conn.WriteBulkString(member.Name) // go-redis/redis package cannot handle uint64. At the time of this writing, // there is no solution for this, and I don't want to use a soft fork to repair it. //conn.WriteUint64(member.ID) conn.WriteInt64(member.Birthdate) if coordinator.CompareByID(member) { conn.WriteBulkString("true") } else { conn.WriteBulkString("false") } } } ================================================ FILE: cluster_client.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package olric import ( "context" "encoding/json" "errors" "fmt" "log" "net" "os" "sync" "sync/atomic" "syscall" "time" "github.com/olric-data/olric/config" "github.com/olric-data/olric/hasher" "github.com/olric-data/olric/internal/bufpool" "github.com/olric-data/olric/internal/cluster/partitions" "github.com/olric-data/olric/internal/discovery" "github.com/olric-data/olric/internal/dmap" "github.com/olric-data/olric/internal/protocol" "github.com/olric-data/olric/internal/ramblock/entry" "github.com/olric-data/olric/internal/resp" "github.com/olric-data/olric/internal/server" "github.com/olric-data/olric/pkg/storage" "github.com/olric-data/olric/stats" "github.com/redis/go-redis/v9" ) var pool = bufpool.New() // DefaultRoutingTableFetchInterval is the default value of RoutingTableFetchInterval. ClusterClient implementation // fetches the routing table from the cluster to route requests to the right partition. const DefaultRoutingTableFetchInterval = time.Minute type ClusterLockContext struct { key string token string dm *ClusterDMap } // ClusterDMap implements a client for DMaps. type ClusterDMap struct { name string newEntry func() storage.Entry config *dmapConfig client *server.Client clusterClient *ClusterClient } // Name exposes name of the DMap. func (dm *ClusterDMap) Name() string { return dm.name } // processProtocolError processes protocol-related errors and translates them into defined application-level errors. func processProtocolError(err error) error { if err == nil { return nil } if errors.Is(err, redis.Nil) { return ErrKeyNotFound } if errors.Is(err, syscall.ECONNREFUSED) { opErr := err.(*net.OpError) return fmt.Errorf("%s %s %s: %w", opErr.Op, opErr.Net, opErr.Addr, ErrConnRefused) } return convertDMapError(protocol.ConvertError(err)) } // writePutCommand constructs and returns a new protocol.Put command based on the provided key, value, and configuration options. func (dm *ClusterDMap) writePutCommand(c *dmap.PutConfig, key string, value []byte) *protocol.Put { cmd := protocol.NewPut(dm.name, key, value) switch { case c.HasEX: cmd.SetEX(c.EX.Seconds()) case c.HasPX: cmd.SetPX(c.PX.Milliseconds()) case c.HasEXAT: cmd.SetEXAT(c.EXAT.Seconds()) case c.HasPXAT: cmd.SetPXAT(c.PXAT.Milliseconds()) } switch { case c.HasNX: cmd.SetNX() case c.HasXX: cmd.SetXX() } return cmd } func (cl *ClusterClient) clientByPartID(partID uint64) (*redis.Client, error) { raw := cl.routingTable.Load() if raw == nil { return nil, fmt.Errorf("routing table is empty") } routingTable, ok := raw.(RoutingTable) if !ok { return nil, fmt.Errorf("routing table is corrupt") } route := routingTable[partID] if len(route.PrimaryOwners) == 0 { return nil, fmt.Errorf("primary owners list for %d is empty", partID) } primaryOwner := route.PrimaryOwners[len(route.PrimaryOwners)-1] return cl.client.Get(primaryOwner), nil } func (cl *ClusterClient) smartPick(dmap, key string) (*redis.Client, error) { hkey := partitions.HKey(dmap, key) partID := hkey % cl.partitionCount return cl.clientByPartID(partID) } // Put sets the value for the given key. It overwrites any previous value for // that key, and it's thread-safe. The key has to be a string. value type is arbitrary. // It is safe to modify the contents of the arguments after Put returns but not before. func (dm *ClusterDMap) Put(ctx context.Context, key string, value interface{}, options ...PutOption) error { rc, err := dm.clusterClient.smartPick(dm.name, key) if err != nil { return err } valueBuf := pool.Get() defer pool.Put(valueBuf) enc := resp.New(valueBuf) err = enc.Encode(value) if err != nil { return err } var pc dmap.PutConfig for _, opt := range options { opt(&pc) } putCmd := dm.writePutCommand(&pc, key, valueBuf.Bytes()) cmd := putCmd.Command(ctx) err = rc.Process(ctx, cmd) if err != nil { return processProtocolError(err) } return processProtocolError(cmd.Err()) } func (dm *ClusterDMap) makeGetResponse(cmd *redis.StringCmd) (*GetResponse, error) { raw, err := cmd.Bytes() if err != nil { return nil, processProtocolError(err) } e := dm.newEntry() e.Decode(raw) return &GetResponse{ entry: e, }, nil } // Get gets the value for the given key. It returns ErrKeyNotFound if the DB // does not contain the key. It's thread-safe. It is safe to modify the contents // of the returned value. See GetResponse for the details. func (dm *ClusterDMap) Get(ctx context.Context, key string) (*GetResponse, error) { cmd := protocol.NewGet(dm.name, key).SetRaw().Command(ctx) rc, err := dm.clusterClient.smartPick(dm.name, key) if err != nil { return nil, err } err = rc.Process(ctx, cmd) if err != nil { return nil, processProtocolError(err) } return dm.makeGetResponse(cmd) } // Delete deletes values for the given keys. Delete will not return an error if the key doesn't exist. // It's thread-safe. It is safe to modify the contents of the argument after Delete returns. func (dm *ClusterDMap) Delete(ctx context.Context, keys ...string) (int, error) { rc, err := dm.client.Pick() if err != nil { return 0, err } cmd := protocol.NewDel(dm.name, keys...).Command(ctx) err = rc.Process(ctx, cmd) if err != nil { return 0, processProtocolError(err) } res, err := cmd.Uint64() if err != nil { return 0, processProtocolError(cmd.Err()) } return int(res), nil } // Incr atomically increments the key by delta. The return value is the new value // after being incremented or an error. func (dm *ClusterDMap) Incr(ctx context.Context, key string, delta int) (int, error) { rc, err := dm.clusterClient.smartPick(dm.name, key) if err != nil { return 0, err } cmd := protocol.NewIncr(dm.name, key, delta).Command(ctx) err = rc.Process(ctx, cmd) if err != nil { return 0, processProtocolError(err) } res, err := cmd.Uint64() if err != nil { return 0, processProtocolError(cmd.Err()) } return int(res), nil } // Decr atomically decrements the key by delta. The return value is the new value // after being decremented or an error. func (dm *ClusterDMap) Decr(ctx context.Context, key string, delta int) (int, error) { rc, err := dm.clusterClient.smartPick(dm.name, key) if err != nil { return 0, err } cmd := protocol.NewDecr(dm.name, key, delta).Command(ctx) err = rc.Process(ctx, cmd) if err != nil { return 0, processProtocolError(err) } res, err := cmd.Uint64() if err != nil { return 0, processProtocolError(cmd.Err()) } return int(res), nil } // GetPut atomically sets the key to value and returns the old value stored at a key. It returns nil if there is no // previous value. func (dm *ClusterDMap) GetPut(ctx context.Context, key string, value interface{}) (*GetResponse, error) { rc, err := dm.clusterClient.smartPick(dm.name, key) if err != nil { return nil, err } valueBuf := pool.Get() defer pool.Put(valueBuf) enc := resp.New(valueBuf) err = enc.Encode(value) if err != nil { return nil, err } cmd := protocol.NewGetPut(dm.name, key, valueBuf.Bytes()).SetRaw().Command(ctx) err = rc.Process(ctx, cmd) err = processProtocolError(err) if err != nil { // First try to set a key/value with GetPut if errors.Is(err, ErrKeyNotFound) { return nil, nil } return nil, err } raw, err := cmd.Bytes() if err != nil { return nil, processProtocolError(err) } e := dm.newEntry() e.Decode(raw) return &GetResponse{ entry: e, }, nil } // IncrByFloat atomically increments the key by delta. The return value is the new value // after being incremented or an error. func (dm *ClusterDMap) IncrByFloat(ctx context.Context, key string, delta float64) (float64, error) { rc, err := dm.clusterClient.smartPick(dm.name, key) if err != nil { return 0, err } cmd := protocol.NewIncrByFloat(dm.name, key, delta).Command(ctx) err = rc.Process(ctx, cmd) if err != nil { return 0, processProtocolError(err) } res, err := cmd.Result() if err != nil { return 0, processProtocolError(cmd.Err()) } return res, nil } // Expire updates the expiry for the given key. It returns ErrKeyNotFound if // the DB does not contain the key. It's thread-safe. func (dm *ClusterDMap) Expire(ctx context.Context, key string, timeout time.Duration) error { rc, err := dm.clusterClient.smartPick(dm.name, key) if err != nil { return err } cmd := protocol.NewExpire(dm.name, key, timeout).Command(ctx) err = rc.Process(ctx, cmd) if err != nil { return processProtocolError(err) } return processProtocolError(cmd.Err()) } // Lock sets a lock for the given key. Acquired lock is only for the key in // this dmap. // // It returns immediately if it acquires the lock for the given key. Otherwise, // it waits until deadline. // // You should know that the locks are approximate and only to be used for // non-critical purposes. func (dm *ClusterDMap) Lock(ctx context.Context, key string, deadline time.Duration) (LockContext, error) { rc, err := dm.clusterClient.smartPick(dm.name, key) if err != nil { return nil, err } cmd := protocol.NewLock(dm.name, key, deadline.Seconds()).Command(ctx) err = rc.Process(ctx, cmd) if err != nil { return nil, processProtocolError(err) } token, err := cmd.Bytes() if err != nil { return nil, processProtocolError(err) } return &ClusterLockContext{ key: key, token: string(token), dm: dm, }, nil } // LockWithTimeout sets a lock for the given key. If the lock is still unreleased // the end of a given period of time, it automatically releases the lock. // Acquired lock is only for the key in this DMap. // // It returns immediately if it acquires the lock for the given key. Otherwise, // it waits until deadline. // // You should know that the locks are approximate and only to be used for // non-critical purposes. func (dm *ClusterDMap) LockWithTimeout(ctx context.Context, key string, timeout, deadline time.Duration) (LockContext, error) { rc, err := dm.clusterClient.smartPick(dm.name, key) if err != nil { return nil, err } cmd := protocol.NewLock(dm.name, key, deadline.Seconds()).SetPX(timeout.Milliseconds()).Command(ctx) err = rc.Process(ctx, cmd) if err != nil { return nil, processProtocolError(err) } token, err := cmd.Bytes() if err != nil { return nil, processProtocolError(err) } return &ClusterLockContext{ key: key, token: string(token), dm: dm, }, nil } // Close stops background routines and frees allocated resources. func (dm *ClusterDMap) Close(_ context.Context) error { return nil } // Unlock releases the distributed lock associated with the current context by using the provided context for execution. func (c *ClusterLockContext) Unlock(ctx context.Context) error { rc, err := c.dm.clusterClient.smartPick(c.dm.name, c.key) if err != nil { return err } cmd := protocol.NewUnlock(c.dm.name, c.key, c.token).Command(ctx) err = rc.Process(ctx, cmd) if err != nil { return processProtocolError(err) } return processProtocolError(cmd.Err()) } // Lease extends the lease of the distributed lock associated with the context for the specified duration. func (c *ClusterLockContext) Lease(ctx context.Context, duration time.Duration) error { rc, err := c.dm.clusterClient.smartPick(c.dm.name, c.key) if err != nil { return err } cmd := protocol.NewLockLease(c.dm.name, c.key, c.token, duration.Seconds()).Command(ctx) err = rc.Process(ctx, cmd) if err != nil { return processProtocolError(err) } return processProtocolError(cmd.Err()) } // Scan returns an iterator to loop over the keys. // // Available scan options: // // * Count // * Match func (dm *ClusterDMap) Scan(ctx context.Context, options ...ScanOption) (Iterator, error) { var sc dmap.ScanConfig for _, opt := range options { opt(&sc) } if sc.Count == 0 { sc.Count = DefaultScanCount } ictx, cancel := context.WithCancel(ctx) i := &ClusterIterator{ dm: dm, clusterClient: dm.clusterClient, config: &sc, logger: dm.clusterClient.logger, partitionKeys: make(map[string]struct{}), cursors: make(map[uint64]map[string]*currentCursor), ctx: ictx, cancel: cancel, } // Embedded iterator uses a slightly different scan function. i.scanner = i.scanOnOwners if err := i.fetchRoutingTable(); err != nil { return nil, err } // Load the route for the first partition (0) to scan. i.loadRoute() i.wg.Add(1) go i.fetchRoutingTablePeriodically() return i, nil } // Destroy flushes the given DMap on the cluster. You should know that there // is no global lock on DMaps. So if you call Put/PutEx and Destroy methods // concurrently on the cluster, Put call may set new values to the DMap. func (dm *ClusterDMap) Destroy(ctx context.Context) error { rc, err := dm.client.Pick() if err != nil { return err } cmd := protocol.NewDestroy(dm.name).Command(ctx) err = rc.Process(ctx, cmd) if err != nil { return processProtocolError(err) } return processProtocolError(cmd.Err()) } // ClusterClient is a client for managing and interacting with a distributed cluster of nodes. type ClusterClient struct { client *server.Client config *clusterClientConfig logger *log.Logger routingTable atomic.Value partitionCount uint64 wg sync.WaitGroup ctx context.Context cancel context.CancelFunc } // Ping sends a ping message to an Olric node. Returns PONG if a message is empty, // otherwise return a copy of the message as bulk. This command is often used to test // if a connection is still alive or to measure latency. func (cl *ClusterClient) Ping(ctx context.Context, addr, message string) (string, error) { pingCmd := protocol.NewPing() if message != "" { pingCmd.SetMessage(message) } cmd := pingCmd.Command(ctx) rc := cl.client.Get(addr) err := rc.Process(ctx, cmd) if err != nil { return "", processProtocolError(err) } err = processProtocolError(cmd.Err()) if err != nil { return "", nil } return cmd.Result() } // RoutingTable returns the latest version of the routing table. func (cl *ClusterClient) RoutingTable(ctx context.Context) (RoutingTable, error) { cmd := protocol.NewClusterRoutingTable().Command(ctx) rc, err := cl.client.Pick() if err != nil { return RoutingTable{}, err } err = rc.Process(ctx, cmd) if err != nil { return RoutingTable{}, processProtocolError(err) } if err = cmd.Err(); err != nil { return RoutingTable{}, processProtocolError(err) } result, err := cmd.Slice() if err != nil { return RoutingTable{}, processProtocolError(err) } return mapToRoutingTable(result) } // Stats returns stats.Stats with the given options. func (cl *ClusterClient) Stats(ctx context.Context, address string, options ...StatsOption) (stats.Stats, error) { var cfg statsConfig for _, opt := range options { opt(&cfg) } statsCmd := protocol.NewStats() if cfg.CollectRuntime { statsCmd.SetCollectRuntime() } cmd := statsCmd.Command(ctx) rc := cl.client.Get(address) err := rc.Process(ctx, cmd) if err != nil { return stats.Stats{}, processProtocolError(err) } if err = cmd.Err(); err != nil { return stats.Stats{}, processProtocolError(err) } data, err := cmd.Bytes() if err != nil { return stats.Stats{}, processProtocolError(err) } var s stats.Stats err = json.Unmarshal(data, &s) if err != nil { return stats.Stats{}, processProtocolError(err) } return s, nil } // Members returns a thread-safe list of cluster members. func (cl *ClusterClient) Members(ctx context.Context) ([]Member, error) { rc, err := cl.client.Pick() if err != nil { return []Member{}, err } cmd := protocol.NewClusterMembers().Command(ctx) err = rc.Process(ctx, cmd) if err != nil { return []Member{}, processProtocolError(err) } if err = cmd.Err(); err != nil { return []Member{}, processProtocolError(err) } items, err := cmd.Slice() if err != nil { return []Member{}, processProtocolError(err) } var members []Member for _, rawItem := range items { m := Member{} item := rawItem.([]interface{}) m.Name = item[0].(string) m.Birthdate = item[1].(int64) // go-redis/redis package cannot handle uint64 type. At the time of this writing, // there is no solution for this, and I don't want to use a soft fork to repair it. m.ID = discovery.MemberID(m.Name, m.Birthdate) if item[2] == "true" { m.Coordinator = true } members = append(members, m) } return members, nil } // RefreshMetadata fetches a list of available members and the latest routing // table version. It also closes stale clients if there are any. func (cl *ClusterClient) RefreshMetadata(ctx context.Context) error { // Fetch a list of currently available cluster members. var members []Member var err error for { members, err = cl.Members(ctx) if errors.Is(err, ErrConnRefused) { continue } if err != nil { return err } break } // Use a map for fast access. addresses := make(map[string]struct{}) for _, member := range members { addresses[member.Name] = struct{}{} } // Clean stale client connections for addr := range cl.client.Addresses() { if _, ok := addresses[addr]; !ok { // Gone if err := cl.client.Close(addr); err != nil { return err } } } // Re-fetch the routing table, we should use the latest routing table version. return cl.fetchRoutingTable() } // Close stops background routines and frees allocated resources. func (cl *ClusterClient) Close(ctx context.Context) error { select { case <-cl.ctx.Done(): return nil default: } cl.cancel() // Wait for the background workers: // * fetchRoutingTablePeriodically cl.wg.Wait() // Close the underlying TCP sockets gracefully. return cl.client.Shutdown(ctx) } // NewPubSub returns a new PubSub client with the given options. func (cl *ClusterClient) NewPubSub(options ...PubSubOption) (*PubSub, error) { return newPubSub(cl.client, options...) } // NewDMap returns a new DMap client with the given options. func (cl *ClusterClient) NewDMap(name string, options ...DMapOption) (DMap, error) { var dc dmapConfig for _, opt := range options { opt(&dc) } if dc.storageEntryImplementation == nil { dc.storageEntryImplementation = func() storage.Entry { return entry.New() } } return &ClusterDMap{name: name, config: &dc, newEntry: dc.storageEntryImplementation, client: cl.client, clusterClient: cl, }, nil } // ClusterClientOption is a functional option for configuring a clusterClientConfig instance. type ClusterClientOption func(c *clusterClientConfig) // clusterClientConfig holds the configuration required to initialize and manage a cluster client instance. type clusterClientConfig struct { logger *log.Logger config *config.Client authentication *config.Authentication hasher hasher.Hasher routingTableFetchInterval time.Duration } // WithHasher sets a custom hasher implementation to the cluster client configuration. func WithHasher(h hasher.Hasher) ClusterClientOption { return func(cfg *clusterClientConfig) { cfg.hasher = h } } // WithLogger sets a custom logger for the cluster client configuration. func WithLogger(l *log.Logger) ClusterClientOption { return func(cfg *clusterClientConfig) { cfg.logger = l } } // WithConfig applies a specified config.Client to the clusterClientConfig. func WithConfig(c *config.Client) ClusterClientOption { return func(cfg *clusterClientConfig) { cfg.config = c } } // WithPassword configures a cluster client with the specified password for authentication. func WithPassword(password string) ClusterClientOption { return func(cfg *clusterClientConfig) { cfg.authentication = &config.Authentication{ Password: password, } } } // WithRoutingTableFetchInterval sets the interval for periodic fetching of the routing table in a cluster client configuration. func WithRoutingTableFetchInterval(interval time.Duration) ClusterClientOption { return func(cfg *clusterClientConfig) { cfg.routingTableFetchInterval = interval } } // fetchRoutingTable updates the cluster routing table by fetching the latest version from the cluster. // It initializes the partition count if it's the first invocation. Returns an error if fetching fails. func (cl *ClusterClient) fetchRoutingTable() error { ctx, cancel := context.WithCancel(cl.ctx) defer cancel() routingTable, err := cl.RoutingTable(ctx) if err != nil { return fmt.Errorf("error while loading the routing table: %w", err) } previous := cl.routingTable.Load() if previous == nil { // First run. Partition count is a constant, actually. It has to be greater than zero. cl.partitionCount = uint64(len(routingTable)) } cl.routingTable.Store(routingTable) return nil } // fetchRoutingTablePeriodically periodically updates the routing table by invoking fetchRoutingTable at configured intervals. // It stops gracefully when the context is canceled or an error occurs. func (cl *ClusterClient) fetchRoutingTablePeriodically() { defer cl.wg.Done() ticker := time.NewTicker(cl.config.routingTableFetchInterval) defer ticker.Stop() for { select { case <-cl.ctx.Done(): return case <-ticker.C: err := cl.fetchRoutingTable() if err != nil { cl.logger.Printf("[ERROR] Failed to fetch the latest version of the routing table: %s", err) } } } } // NewClusterClient creates a new Client instance. It needs one node address at least to discover the whole cluster. func NewClusterClient(addresses []string, options ...ClusterClientOption) (*ClusterClient, error) { if len(addresses) == 0 { return nil, fmt.Errorf("addresses cannot be empty") } var cc clusterClientConfig for _, opt := range options { opt(&cc) } if cc.hasher == nil { cc.hasher = hasher.NewDefaultHasher() } if cc.logger == nil { cc.logger = log.New(os.Stderr, "logger: ", log.Lshortfile) } if cc.config == nil { cc.config = config.NewClient() } if cc.authentication != nil { cc.config.Authentication = cc.authentication } if cc.routingTableFetchInterval <= 0 { cc.routingTableFetchInterval = DefaultRoutingTableFetchInterval } if err := cc.config.Sanitize(); err != nil { return nil, err } if err := cc.config.Validate(); err != nil { return nil, err } ctx, cancel := context.WithCancel(context.Background()) cl := &ClusterClient{ client: server.NewClient(cc.config), config: &cc, logger: cc.logger, ctx: ctx, cancel: cancel, } // Initialize clients for the given cluster members. for _, address := range addresses { cl.client.Get(address) } // Discover all cluster members members, err := cl.Members(ctx) if err != nil { return nil, fmt.Errorf("error while discovering the cluster members: %w", err) } for _, member := range members { cl.client.Get(member.Name) } // Hash function is required to target primary owners instead of random cluster members. partitions.SetHashFunc(cc.hasher) // Initial fetch. ClusterClient targets the primary owners for a smooth and quick operation. if err := cl.fetchRoutingTable(); err != nil { return nil, err } // Refresh the routing table in every 15 seconds. cl.wg.Add(1) go cl.fetchRoutingTablePeriodically() return cl, nil } var ( _ Client = (*ClusterClient)(nil) _ DMap = (*ClusterDMap)(nil) ) ================================================ FILE: cluster_client_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package olric import ( "context" "log" "os" "testing" "time" "github.com/olric-data/olric/config" "github.com/olric-data/olric/hasher" "github.com/olric-data/olric/internal/testutil" "github.com/olric-data/olric/stats" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" ) func TestClusterClient_Ping(t *testing.T) { cluster := newTestOlricCluster(t) cluster.addMember(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() response, err := c.Ping(ctx, db.rt.This().String(), "") require.NoError(t, err) require.Equal(t, DefaultPingResponse, response) } func TestClusterClient_Ping_WithMessage(t *testing.T) { cluster := newTestOlricCluster(t) cluster.addMember(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() message := "Olric is the best!" result, err := c.Ping(ctx, db.rt.This().String(), message) require.NoError(t, err) require.Equal(t, message, result) } func TestClusterClient_RoutingTable(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() rt, err := c.RoutingTable(ctx) require.NoError(t, err) require.Len(t, rt, int(db.config.PartitionCount)) } func TestClusterClient_RoutingTable_Cluster(t *testing.T) { cluster := newTestOlricCluster(t) cluster.addMember(t) // Cluster coordinator <-time.After(250 * time.Millisecond) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() rt, err := c.RoutingTable(ctx) require.NoError(t, err) require.Len(t, rt, int(db.config.PartitionCount)) } func TestClusterClient_Put(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) err = dm.Put(ctx, "mykey", "myvalue") require.NoError(t, err) } func TestClusterClient_Get(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) err = dm.Put(ctx, "mykey", "myvalue") require.NoError(t, err) gr, err := dm.Get(ctx, "mykey") require.NoError(t, err) res, err := gr.String() require.NoError(t, err) require.Equal(t, res, "myvalue") } func TestClusterClient_Delete(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) err = dm.Put(ctx, "mykey", "myvalue") require.NoError(t, err) count, err := dm.Delete(ctx, "mykey") require.NoError(t, err) require.Equal(t, 1, count) _, err = dm.Get(ctx, "mykey") require.ErrorIs(t, err, ErrKeyNotFound) } func TestClusterClient_Delete_Many_Keys(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) var keys []string for i := 0; i < 10; i++ { key := testutil.ToKey(i) err = dm.Put(context.Background(), key, "myvalue") require.NoError(t, err) keys = append(keys, key) } count, err := dm.Delete(context.Background(), keys...) require.NoError(t, err) require.Equal(t, 10, count) } func TestClusterClient_Destroy(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) err = dm.Put(ctx, "mykey", "myvalue") require.NoError(t, err) err = dm.Destroy(ctx) require.NoError(t, err) _, err = dm.Get(ctx, "mykey") require.ErrorIs(t, err, ErrKeyNotFound) } func TestClusterClient_Incr(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) var errGr errgroup.Group for i := 0; i < 10; i++ { errGr.Go(func() error { _, err = dm.Incr(ctx, "mykey", 1) return err }) } require.NoError(t, errGr.Wait()) result, err := dm.Incr(ctx, "mykey", 1) require.NoError(t, err) require.Equal(t, 11, result) } func TestClusterClient_IncrByFloat(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) var errGr errgroup.Group for i := 0; i < 10; i++ { errGr.Go(func() error { _, err = dm.IncrByFloat(ctx, "mykey", 1.2) return err }) } require.NoError(t, errGr.Wait()) result, err := dm.IncrByFloat(ctx, "mykey", 1.2) require.NoError(t, err) require.Equal(t, 13.199999999999998, result) } func TestClusterClient_Decr(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) err = dm.Put(ctx, "mykey", 11) require.NoError(t, err) var errGr errgroup.Group for i := 0; i < 10; i++ { errGr.Go(func() error { _, err = dm.Decr(ctx, "mykey", 1) return err }) } require.NoError(t, errGr.Wait()) result, err := dm.Decr(ctx, "mykey", 1) require.NoError(t, err) require.Equal(t, 0, result) } func TestClusterClient_GetPut(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) gr, err := dm.GetPut(ctx, "mykey", "myvalue") require.NoError(t, err) require.Nil(t, gr) gr, err = dm.GetPut(ctx, "mykey", "myvalue-2") require.NoError(t, err) value, err := gr.String() require.NoError(t, err) require.Equal(t, "myvalue", value) } func TestClusterClient_Expire(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) err = dm.Put(ctx, "mykey", "myvalue") require.NoError(t, err) err = dm.Expire(ctx, "mykey", time.Millisecond) require.NoError(t, err) <-time.After(time.Millisecond) _, err = dm.Get(ctx, "mykey") require.ErrorIs(t, err, ErrKeyNotFound) } func TestClusterClient_Lock_Unlock(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) lx, err := dm.Lock(ctx, "lock.foo.key", time.Second) require.NoError(t, err) err = lx.Unlock(ctx) require.NoError(t, err) } func TestClusterClient_Lock_Lease(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) lx, err := dm.Lock(ctx, "lock.foo.key", time.Second) require.NoError(t, err) err = lx.Lease(ctx, time.Millisecond) require.NoError(t, err) <-time.After(time.Millisecond) err = lx.Unlock(ctx) require.ErrorIs(t, err, ErrNoSuchLock) } func TestClusterClient_Lock_ErrLockNotAcquired(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) _, err = dm.Lock(ctx, "lock.foo.key", time.Second) require.NoError(t, err) _, err = dm.Lock(ctx, "lock.foo.key", time.Millisecond) require.ErrorIs(t, err, ErrLockNotAcquired) } func TestClusterClient_LockWithTimeout(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) lx, err := dm.LockWithTimeout(ctx, "lock.foo.key", time.Hour, time.Second) require.NoError(t, err) err = lx.Unlock(ctx) require.NoError(t, err) } func TestClusterClient_LockWithTimeout_ErrNoSuchLock(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) lx, err := dm.LockWithTimeout(ctx, "lock.foo.key", time.Millisecond, time.Second) require.NoError(t, err) <-time.After(time.Millisecond) err = lx.Unlock(ctx) require.ErrorIs(t, err, ErrNoSuchLock) } func TestClusterClient_LockWithTimeout_Then_Lease(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) lx, err := dm.LockWithTimeout(ctx, "lock.foo.key", 50*time.Millisecond, time.Second) require.NoError(t, err) // Expand its timeout value err = lx.Lease(ctx, time.Hour) require.NoError(t, err) <-time.After(100 * time.Millisecond) _, err = dm.Lock(ctx, "lock.foo.key", time.Millisecond) require.ErrorIs(t, err, ErrLockNotAcquired) } func TestClusterClient_LockWithTimeout_ErrLockNotAcquired(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) _, err = dm.LockWithTimeout(ctx, "lock.foo.key", time.Hour, time.Second) require.NoError(t, err) _, err = dm.Lock(ctx, "lock.foo.key", time.Millisecond) require.Equal(t, err, ErrLockNotAcquired) } func TestClusterClient_Put_Ex(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) err = dm.Put(ctx, "mykey", "myvalue", EX(time.Second)) require.NoError(t, err) <-time.After(time.Second) _, err = dm.Get(ctx, "mykey") require.ErrorIs(t, err, ErrKeyNotFound) } func TestClusterClient_Put_PX(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) err = dm.Put(ctx, "mykey", "myvalue", PX(time.Millisecond)) require.NoError(t, err) <-time.After(time.Millisecond) _, err = dm.Get(ctx, "mykey") require.ErrorIs(t, err, ErrKeyNotFound) } func TestClusterClient_Put_EXAT(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) err = dm.Put(ctx, "mykey", "myvalue", EXAT(time.Duration(time.Now().Add(time.Second).UnixNano()))) require.NoError(t, err) <-time.After(time.Second) _, err = dm.Get(ctx, "mykey") require.ErrorIs(t, err, ErrKeyNotFound) } func TestClusterClient_Put_PXAT(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) err = dm.Put(ctx, "mykey", "myvalue", PXAT(time.Duration(time.Now().Add(time.Millisecond).UnixNano()))) require.NoError(t, err) <-time.After(time.Millisecond) _, err = dm.Get(ctx, "mykey") require.ErrorIs(t, err, ErrKeyNotFound) } func TestClusterClient_Put_NX(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) err = dm.Put(ctx, "mykey", "myvalue") require.NoError(t, err) err = dm.Put(ctx, "mykey", "myvalue-2", NX()) require.ErrorIs(t, err, ErrKeyFound) gr, err := dm.Get(ctx, "mykey") require.NoError(t, err) value, err := gr.String() require.NoError(t, err) require.Equal(t, "myvalue", value) } func TestClusterClient_Put_XX(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) err = dm.Put(ctx, "mykey", "myvalue-2", XX()) require.ErrorIs(t, err, ErrKeyNotFound) } func TestClusterClient_Stats(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() var empty stats.Stats s, err := c.Stats(ctx, db.rt.This().String()) require.NoError(t, err) require.Nil(t, s.Runtime) require.NotEqual(t, empty, s) } func TestClusterClient_Stats_Cluster(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) db2 := cluster.addMember(t) <-time.After(250 * time.Millisecond) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() var empty stats.Stats s, err := c.Stats(ctx, db2.rt.This().String()) require.NoError(t, err) require.Nil(t, s.Runtime) require.NotEqual(t, empty, s) require.Equal(t, db2.rt.This().String(), s.Member.String()) } func TestClusterClient_Stats_CollectRuntime(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() var empty stats.Stats s, err := c.Stats(ctx, db.rt.This().String(), CollectRuntime()) require.NoError(t, err) require.NotNil(t, s.Runtime) require.NotEqual(t, empty, s) } func TestClusterClient_Set_Options(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() lg := log.New(os.Stderr, "logger: ", log.Lshortfile) cfg := config.NewClient() c, err := NewClusterClient([]string{db.name}, WithConfig(cfg), WithLogger(lg)) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() require.Equal(t, cfg, c.config.config) require.Equal(t, lg, c.config.logger) } func TestClusterClient_Members(t *testing.T) { cluster := newTestOlricCluster(t) cluster.addMember(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() members, err := c.Members(ctx) require.NoError(t, err) require.Len(t, members, 2) coordinator := db.rt.Discovery().GetCoordinator() for _, member := range members { require.NotEqual(t, "", member.Name) require.NotEqual(t, 0, member.ID) require.NotEqual(t, 0, member.Birthdate) if coordinator.ID == member.ID { require.True(t, member.Coordinator) } else { require.False(t, member.Coordinator) } } } func TestClusterClient_smartPick(t *testing.T) { cluster := newTestOlricCluster(t) db1 := cluster.addMember(t) db2 := cluster.addMember(t) db3 := cluster.addMember(t) db4 := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient( []string{db1.name, db2.name, db3.name, db4.name}, WithHasher(hasher.NewDefaultHasher()), ) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() clients := make(map[string]struct{}) for i := 0; i < 1000; i++ { rc, err := c.smartPick("mydmap", testutil.ToKey(i)) require.NoError(t, err) clients[rc.String()] = struct{}{} } require.Len(t, clients, 4) } ================================================ FILE: cluster_iterator.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package olric import ( "context" "log" "sync" "time" "github.com/olric-data/olric/internal/dmap" "github.com/olric-data/olric/internal/protocol" ) type currentCursor struct { primary uint64 replica uint64 } // ClusterIterator implements distributed query on DMaps. type ClusterIterator struct { mtx sync.Mutex // protects pos and page routingTableMtx sync.Mutex // protects routingTable and partitionCount logger *log.Logger dm *ClusterDMap clusterClient *ClusterClient pos int page []string route *Route partitionKeys map[string]struct{} cursors map[uint64]map[string]*currentCursor partID uint64 // current partition id routingTable RoutingTable partitionCount uint64 config *dmap.ScanConfig scanner func() error wg sync.WaitGroup ctx context.Context cancel context.CancelFunc } func (i *ClusterIterator) loadRoute() { i.routingTableMtx.Lock() defer i.routingTableMtx.Unlock() route, ok := i.routingTable[i.partID] if !ok { panic("partID: could not be found in the routing table") } i.route = &route } func (i *ClusterIterator) updateCursor(owner string, cursor uint64) { if _, ok := i.cursors[i.partID]; !ok { i.cursors[i.partID] = make(map[string]*currentCursor) } cc, ok := i.cursors[i.partID][owner] if !ok { cc = ¤tCursor{} if i.config.Replica { cc.replica = cursor } else { cc.primary = cursor } i.cursors[i.partID][owner] = cc return } if i.config.Replica { cc.replica = cursor } else { cc.primary = cursor } i.cursors[i.partID][owner] = cc } func (i *ClusterIterator) loadCursor(owner string) uint64 { if _, ok := i.cursors[i.partID]; !ok { return 0 } cc, ok := i.cursors[i.partID][owner] if !ok { return 0 } if i.config.Replica { return cc.replica } return cc.primary } func (i *ClusterIterator) updateIterator(keys []string, cursor uint64, owner string) { for _, key := range keys { if _, ok := i.partitionKeys[key]; !ok { i.page = append(i.page, key) i.partitionKeys[key] = struct{}{} } } i.updateCursor(owner, cursor) } func (i *ClusterIterator) getOwners() []string { var raw []string if i.config.Replica { raw = i.routingTable[i.partID].ReplicaOwners } else { raw = i.routingTable[i.partID].PrimaryOwners } var owners []string // Make a safe copy of the raw. for _, owner := range raw { owners = append(owners, owner) } return owners } func (i *ClusterIterator) removeScannedOwner(idx int) { if i.config.Replica { if len(i.route.ReplicaOwners) > 0 && len(i.route.ReplicaOwners) > idx { i.route.ReplicaOwners = append(i.route.ReplicaOwners[:idx], i.route.ReplicaOwners[idx+1:]...) } } else { if len(i.route.PrimaryOwners) > 0 && len(i.route.PrimaryOwners) > idx { i.route.PrimaryOwners = append(i.route.PrimaryOwners[:idx], i.route.PrimaryOwners[idx+1:]...) } } } func (i *ClusterIterator) scanOnOwners() error { owners := i.getOwners() for idx, owner := range owners { cursor := i.loadCursor(owner) // Build a scan command here s := protocol.NewScan(i.partID, i.dm.Name(), cursor) if i.config.HasCount { s.SetCount(i.config.Count) } if i.config.HasMatch { s.SetMatch(i.config.Match) } if i.config.Replica { s.SetReplica() } scanCmd := s.Command(i.ctx) // Fetch a Redis client for the given owner. rc := i.clusterClient.client.Get(owner) err := rc.Process(i.ctx, scanCmd) if err != nil { return err } keys, newCursor, err := scanCmd.Result() if err != nil { return err } i.updateIterator(keys, newCursor, owner) if newCursor == 0 { i.removeScannedOwner(idx) } } return nil } func (i *ClusterIterator) resetPage() { if len(i.page) != 0 { i.page = []string{} } i.pos = 0 } func (i *ClusterIterator) fetchData() error { i.config.Replica = false if err := i.scanner(); err != nil { return err } i.config.Replica = true return i.scanner() } func (i *ClusterIterator) reset() { i.partitionKeys = make(map[string]struct{}) i.resetPage() i.loadRoute() } func (i *ClusterIterator) next() bool { if len(i.page) != 0 { i.pos++ if i.pos <= len(i.page) { return true } } i.resetPage() for { if err := i.fetchData(); err != nil { i.logger.Printf("[ERROR] Failed to fetch data: %s", err) return false } if len(i.page) != 0 { // We have data on the page to read. Stop the iteration. break } if len(i.route.PrimaryOwners) == 0 && len(i.route.ReplicaOwners) == 0 { // We completed scanning all the owners. Stop the iteration. break } } if len(i.page) == 0 && len(i.route.PrimaryOwners) == 0 && len(i.route.ReplicaOwners) == 0 { i.partID++ if i.partID >= i.partitionCount { return false } i.reset() return i.next() } i.pos = 1 return true } // Next returns true if there is more key in the iterator implementation. // Otherwise, it returns false func (i *ClusterIterator) Next() bool { i.mtx.Lock() defer i.mtx.Unlock() select { case <-i.ctx.Done(): return false default: } return i.next() } // Key returns a key name from the distributed map. func (i *ClusterIterator) Key() string { i.mtx.Lock() defer i.mtx.Unlock() var key string if i.pos > 0 && i.pos <= len(i.page) { key = i.page[i.pos-1] } return key } func (i *ClusterIterator) fetchRoutingTablePeriodically() { defer i.wg.Done() for { select { case <-i.ctx.Done(): return case <-time.After(time.Second): if err := i.fetchRoutingTable(); err != nil { i.logger.Printf("[ERROR] Failed to fetch the latest version of the routing table: %s", err) } } } } func (i *ClusterIterator) fetchRoutingTable() error { routingTable, err := i.clusterClient.RoutingTable(i.ctx) if err != nil { return err } i.routingTableMtx.Lock() defer i.routingTableMtx.Unlock() // Partition count is a constant, actually. It has to be greater than zero. i.partitionCount = uint64(len(routingTable)) i.routingTable = routingTable return nil } // Close stops the iteration and releases allocated resources. func (i *ClusterIterator) Close() { select { case <-i.ctx.Done(): return default: } i.cancel() // await for routing table updater i.wg.Wait() } ================================================ FILE: cluster_iterator_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package olric import ( "context" "fmt" "testing" "github.com/olric-data/olric/internal/testutil" "github.com/stretchr/testify/require" ) func TestClusterClient_ScanMatch(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) evenKeys := make(map[string]bool) for i := 0; i < 100; i++ { var key string if i%2 == 0 { key = fmt.Sprintf("even:%s", testutil.ToKey(i)) evenKeys[key] = false } else { key = fmt.Sprintf("odd:%s", testutil.ToKey(i)) } err = dm.Put(ctx, key, i) require.NoError(t, err) } i, err := dm.Scan(ctx, Match("^even:")) require.NoError(t, err) var count int defer i.Close() for i.Next() { count++ require.Contains(t, evenKeys, i.Key()) } require.Equal(t, 50, count) } func TestClusterClient_Scan(t *testing.T) { cl := newTestOlricCluster(t) db := cl.addMember(t) cl.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) allKeys := make(map[string]bool) for i := 0; i < 100; i++ { err = dm.Put(ctx, testutil.ToKey(i), i) require.NoError(t, err) allKeys[testutil.ToKey(i)] = false } i, err := dm.Scan(ctx) require.NoError(t, err) var count int defer i.Close() for i.Next() { count++ require.Contains(t, allKeys, i.Key()) } require.Equal(t, 100, count) } ================================================ FILE: cluster_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package olric import ( "context" "testing" "github.com/olric-data/olric/internal/protocol" "github.com/stretchr/testify/require" ) func TestOlric_ClusterRoutingTable_clusterRoutingTableCommandHandler(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) rtCmd := protocol.NewClusterRoutingTable().Command(db.ctx) rc := db.client.Get(db.rt.This().String()) err := rc.Process(db.ctx, rtCmd) require.NoError(t, err) slice, err := rtCmd.Slice() require.NoError(t, err) rt, err := mapToRoutingTable(slice) require.NoError(t, err) require.Len(t, rt, int(db.config.PartitionCount)) for _, route := range rt { require.Len(t, route.PrimaryOwners, 1) require.Equal(t, db.rt.This().String(), route.PrimaryOwners[0]) require.Len(t, route.ReplicaOwners, 0) } } func TestOlric_RoutingTable_Standalone(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) rt, err := db.routingTable(context.Background()) require.NoError(t, err) require.Len(t, rt, int(db.config.PartitionCount)) for _, route := range rt { require.Len(t, route.PrimaryOwners, 1) require.Equal(t, db.rt.This().String(), route.PrimaryOwners[0]) require.Len(t, route.ReplicaOwners, 0) } } ================================================ FILE: cmd/olric-server/main.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Server implementation for Olric. Olric Server basically manages configuration for you. package main import ( "context" "flag" "fmt" "io/ioutil" "os" "runtime" "github.com/olric-data/olric" "github.com/olric-data/olric/cmd/olric-server/server" "github.com/olric-data/olric/config" "github.com/sean-/seed" ) func usage() { var msg = `Usage: olric-server [options] ... Distributed key-value store and cache Options: -h, --help Print this message and exit. -v, --version Print the version number and exit. -c, --config Sets configuration file path. Default is olric-server-local.yaml in the current folder. Set OLRIC_SERVER_CONFIG to overwrite it. The Go runtime version %s Report bugs to https://github.com/olric-data/olric/issues ` _, err := fmt.Fprintf(os.Stdout, msg, runtime.Version()) if err != nil { panic(err) } } type arguments struct { config string help bool version bool } const ( // DefaultConfigFile is the default configuration file path on a Unix-based operating system. DefaultConfigFile = "olric-server-local.yaml" // EnvConfigFile is the name of environment variable which can be used to override default configuration file path. EnvConfigFile = "OLRIC_SERVER_CONFIG" ) func main() { args := &arguments{} // Parse command line parameters f := flag.NewFlagSet(os.Args[0], flag.ContinueOnError) f.SetOutput(ioutil.Discard) f.BoolVar(&args.help, "h", false, "") f.BoolVar(&args.help, "help", false, "") f.BoolVar(&args.version, "version", false, "") f.BoolVar(&args.version, "v", false, "") f.StringVar(&args.config, "config", DefaultConfigFile, "") f.StringVar(&args.config, "c", DefaultConfigFile, "") if err := f.Parse(os.Args[1:]); err != nil { _, _ = fmt.Fprintf(os.Stderr, fmt.Sprintf("parsing error: %v\n", err)) usage() os.Exit(1) } if args.version { _, _ = fmt.Fprintf(os.Stderr, "olric-server version %s %s %s/%s\n", olric.ReleaseVersion, runtime.Version(), runtime.GOOS, runtime.GOARCH, ) return } else if args.help { usage() return } // MustInit provides guaranteed secure seeding. If `/dev/urandom` is not // available, MustInit will panic() with an error indicating why reading from // `/dev/urandom` failed. MustInit() will upgrade the seed if for some reason a // call to Init() failed in the past. seed.MustInit() envPath := os.Getenv(EnvConfigFile) if envPath != "" { args.config = envPath } c, err := config.Load(args.config) if err != nil { _, _ = fmt.Fprintf(os.Stderr, "failed to load the configuration file: %s: %v\n", args.config, err) os.Exit(1) } s, err := server.New(c) if err != nil { c.Logger.Fatalf("[ERROR] Failed to create a new Olric instance: %v", err) } if err = s.Start(); err != nil { c.Logger.Printf("[ERROR] Failed to start Olric: %v", err) ctx, cancel := context.WithCancel(context.Background()) defer cancel() if err := s.Shutdown(ctx); err != nil { c.Logger.Printf("[ERROR] Failed to shutdown Olric: %v", err) } c.Logger.Fatal("[ERROR] Quit unexpectedly!") } c.Logger.Print("[INFO] Quit!") } ================================================ FILE: cmd/olric-server/olric-server-local.yaml ================================================ # # IMPORTANT NOTE: This configuration file is intended for testing and local development. # server: # BindAddr denotes the address that Olric will bind to for communication # with other Olric nodes. bindAddr: localhost # BindPort denotes the address that Olric will bind to for communication # with other Olric nodes. bindPort: 3320 # KeepAlivePeriod denotes whether the operating system should send # keep-alive messages on the connection. keepAlivePeriod: 300s # IdleClose will automatically close idle connections after the specified duration. # Use zero to disable this feature. # idleClose: 300s # Timeout for bootstrap control # # An Olric node checks operation status before taking any action for the # cluster events, responding incoming requests and running API functions. # Bootstrapping status is one of the most important checkpoints for an # "operable" Olric node. BootstrapTimeout sets a deadline to check # bootstrapping status without blocking indefinitely. bootstrapTimeout: 5s # PartitionCount is 271, by default. partitionCount: 271 # ReplicaCount is 1, by default. replicaCount: 1 # Minimum number of successful writes to return a response for a write request. writeQuorum: 1 # Minimum number of successful reads to return a response for a read request. readQuorum: 1 # Switch to control read-repair algorithm which helps to reduce entropy. readRepair: false # Default value is SyncReplicationMode. replicationMode: 0 # sync mode. for async, set 1 # Minimum number of members to form a cluster and run any query on the cluster. memberCountQuorum: 1 # Coordinator member pushes the routing table to cluster members in the case of # node join or left events. It also pushes the table periodically. routingTablePushInterval # is the interval between subsequent calls. Default is 1 minute. routingTablePushInterval: 1m # Olric can send push cluster events to cluster.events channel. Available cluster events: # # * node-join-event # * node-left-event # * fragment-migration-event # * fragment-received-event # # If you want to receive these events, set true to EnableClusterEventsChannel and subscribe to # cluster.events channel. Default is false. enableClusterEventsChannel: true #authentication: #password: "your-password" client: # Timeout for TCP dial. # # The timeout includes name resolution, if required. When using TCP, and the host in the address parameter # resolves to multiple IP addresses, the timeout is spread over each consecutive dial, such that each is # given an appropriate fraction of the time to connect. dialTimeout: 5s # Timeout for socket reads. If reached, commands will fail # with a timeout instead of blocking. Use value -1 for no timeout and 0 for default. # Default is DefaultReadTimeout readTimeout: 3s # Timeout for socket writes. If reached, commands will fail # with a timeout instead of blocking. # Default is DefaultWriteTimeout writeTimeout: 3s # Maximum number of retries before giving up. # Default is 3 retries; -1 (not 0) disables retries. #maxRetries: 3 # Minimum backoff between each retry. # Default is 8 milliseconds; -1 disables backoff. #minRetryBackoff: 8ms # Maximum backoff between each retry. # Default is 512 milliseconds; -1 disables backoff. #maxRetryBackoff: 512ms # Type of connection pool. # true for FIFO pool, false for LIFO pool. # Note that fifo has higher overhead compared to lifo. #poolFIFO: false # Maximum number of socket connections. # Default is 10 connections per every available CPU as reported by runtime.GOMAXPROCS. #poolSize: 0 # Minimum number of idle connections which is useful when establishing # new connection is slow. #minIdleConns: # Connection age at which client retires (closes) the connection. # Default is to not close aged connections. #maxConnAge: # Amount of time client waits for connection if all connections are busy before # returning an error. Default is ReadTimeout + 1 second. #poolTimeout: 3s # Amount of time after which client closes idle connections. # Should be less than server's timeout. # Default is 5 minutes. -1 disables idle timeout check. #idleTimeout: 5m # Frequency of idle checks made by idle connections reaper. # Default is 1 minute. -1 disables idle connections reaper, # but idle connections are still discarded by the client # if IdleTimeout is set. #idleCheckFrequency: 1m logging: # DefaultLogVerbosity denotes default log verbosity level. # # * 1 - Generally useful for this to ALWAYS be visible to an operator # * Programmer errors # * Logging extra info about a panic # * CLI argument handling # * 2 - A reasonable default log level if you don't want verbosity. # * Information about config (listening on X, watching Y) # * Errors that repeat frequently that relate to conditions that can be # corrected # * 3 - Useful steady state information about the service and # important log messages that may correlate to # significant changes in the system. This is the recommended default log # level for most systems. # * Logging HTTP requests and their exit code # * System state changing # * Controller state change events # * Scheduler log messages # * 4 - Extended information about changes # * More info about system state changes # * 5 - Debug level verbosity # * Logging in particularly thorny parts of code where you may want to come # back later and check it # * 6 - Trace level verbosity # * Context to understand the steps leading up to neterrors and warnings # * More information for troubleshooting reported issues verbosity: 3 # Default LogLevel is DEBUG. Available levels: "DEBUG", "WARN", "ERROR", "INFO" level: WARN output: stderr memberlist: environment: local # Configuration related to what address to bind to and ports to # listen on. The port is used for both UDP and TCP gossip. It is # assumed other nodes are running on this port, but they do not need # to. bindAddr: localhost bindPort: 3322 # EnableCompression is used to control message compression. This can # be used to reduce bandwidth usage at the cost of slightly more CPU # utilization. This is only available starting at protocol version 1. enableCompression: false # JoinRetryInterval is the time gap between attempts to join an existing # cluster. joinRetryInterval: 1ms # MaxJoinAttempts denotes the maximum number of attemps to join an existing # cluster before forming a new one. maxJoinAttempts: 1 # See service discovery plugins #peers: # - "localhost:3325" #advertiseAddr: "" #advertisePort: 3322 #suspicionMaxTimeoutMult: 6 #disableTCPPings: false #awarenessMaxMultiplier: 8 #gossipNodes: 3 #gossipVerifyIncoming: true #gossipVerifyOutgoing: true #dnsConfigPath: "/etc/resolv.conf" #handoffQueueDepth: 1024 #udpBufferSize: 1400 dmaps: engine: name: ramblock config: tableSize: 524288 # bytes # checkEmptyFragmentsInterval: 1m # triggerCompactionInterval: 10m # numEvictionWorkers: 1 # maxIdleDuration: "" # ttlDuration: "100s" # maxKeys: 100000 # maxInuse: 1000000 # lRUSamples: 10 # evictionPolicy: "LRU" # custom: # foobar: # maxIdleDuration: "60s" # ttlDuration: "300s" # maxKeys: 500000 # lRUSamples: 20 # evictionPolicy: "NONE" #serviceDiscovery: # # path is a required property and used by Olric. It has to be a full path. # path: "/home/burak/go/src/github.com/olric-data/olric-consul-plugin/consul.so" # # # provider is just informal, # provider: "consul" # # # Plugin specific configuration # # Consul server, used by the plugin. It's required # address: "http://127.0.0.1:8500" # # # Specifies that the server should return only nodes with all checks in the passing state. # passingOnly: true # # # Missing health checks from the request will be deleted from the agent. Using this parameter # # allows to idempotently register a service and its checks without having to manually deregister # # checks. # replaceExistingChecks: true # # # InsecureSkipVerify controls whether a client verifies the # # server's certificate chain and host name. # # If InsecureSkipVerify is true, TLS accepts any certificate # # presented by the server and any host name in that certificate. # # In this mode, TLS is susceptible to man-in-the-middle attacks. # # This should be used only for testing. # insecureSkipVerify: true # # # service record # payload: ' # { # "Name": "olric-cluster", # "ID": "olric-node-1", # "Tags": [ # "primary", # "v1" # ], # "Address": "localhost", # "Port": 3322, # "EnableTagOverride": false, # "check": { # "name": "Olric node on 3322", # "tcp": "0.0.0.0:3322", # "interval": "10s", # "timeout": "1s" # } # } #' # # #serviceDiscovery: # provider: "k8s" # path: "/Users/buraksezer/go/src/github.com/olric-data/olric-cloud-plugin/olric-cloud-plugin.so" # args: 'label_selector="app = olric-server"' ================================================ FILE: cmd/olric-server/server/server.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. /*Package server provides a standalone server implementation for Olric*/ package server import ( "context" "log" "os" "os/signal" "syscall" "github.com/olric-data/olric" "github.com/olric-data/olric/config" "golang.org/x/sync/errgroup" ) // OlricServer represents an instance of the Olric distributed in-memory data structure store. // It encapsulates logging, configuration, the Olric database instance, and an error group for // concurrency management. type OlricServer struct { log *log.Logger config *config.Config db *olric.Olric errGr errgroup.Group } // New initializes a new OlricServer instance using the provided configuration and returns it or an error. func New(c *config.Config) (*OlricServer, error) { db, err := olric.New(c) if err != nil { return nil, err } return &OlricServer{ config: c, log: c.Logger, db: db, }, nil } // waitForInterrupt waits for termination signals (SIGTERM, SIGINT) to gracefully shut down the Olric server instance. func (s *OlricServer) waitForInterrupt() { shutDownChan := make(chan os.Signal, 1) signal.Notify(shutDownChan, syscall.SIGTERM, syscall.SIGINT) ch := <-shutDownChan s.log.Printf("[INFO] Signal catched: %s", ch.String()) // Awaits for shutdown s.errGr.Go(func() error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() if err := s.db.Shutdown(ctx); err != nil { s.log.Printf("[ERROR] Failed to shutdown Olric: %v", err) return err } return nil }) // This is not a goroutine leak. The process will quit. go func() { s.log.Printf("[INFO] Awaiting for background tasks") s.log.Printf("[INFO] Press CTRL+C or send SIGTERM/SIGINT to quit immediately") forceQuitCh := make(chan os.Signal, 1) signal.Notify(forceQuitCh, syscall.SIGTERM, syscall.SIGINT) ch := <-forceQuitCh s.log.Printf("[INFO] Signal caught: %s", ch.String()) s.log.Printf("[INFO] Quits with exit code 1") os.Exit(1) }() } // Start launches the Olric server instance and begins listening for incoming requests and termination signals. func (s *OlricServer) Start() error { s.log.Printf("[INFO] pid: %d has been started", os.Getpid()) // Wait for SIGTERM or SIGINT go s.waitForInterrupt() s.errGr.Go(func() error { return s.db.Start() }) return s.errGr.Wait() } // Shutdown gracefully stops the Olric server instance, releasing resources and ensuring a clean termination. func (s *OlricServer) Shutdown(ctx context.Context) error { return s.db.Shutdown(ctx) } ================================================ FILE: config/authentication.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package config import "strings" type Authentication struct { Password string } // Sanitize ensures the Authentication configuration is pre-processed and prepared for use, with no changes currently applied. func (a *Authentication) Sanitize() error { a.Password = strings.TrimSpace(a.Password) return nil } // Validate checks the current Authentication configuration for validity and returns an error if issues are found. func (a *Authentication) Validate() error { // Nothing to do return nil } // Enabled checks if authentication is enabled by verifying if the password is set and returns true if it is configured. func (a *Authentication) Enabled() bool { return len(a.Password) > 0 } // Interface guard var _ IConfig = (*Authentication)(nil) ================================================ FILE: config/client.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package config import ( "context" "crypto/tls" "fmt" "net" "runtime" "time" "github.com/redis/go-redis/v9" ) const ( DefaultDialTimeout = 5 * time.Second DefaultKeepalive = 5 * time.Minute DefaultReadTimeout = 3 * time.Second DefaultIdleTimeout = 5 * time.Minute DefaultMinRetryBackoff = 8 * time.Millisecond DefaultMaxRetryBackoff = 512 * time.Millisecond DefaultMaxRetries = 3 ) // Client denotes configuration for TCP clients in Olric and the official Golang client. type Client struct { Authentication *Authentication // Dial timeout for establishing new connections. // Default is 5 seconds. DialTimeout time.Duration // Timeout for socket reads. If reached, commands will fail // with a timeout instead of blocking. Use value -1 for no timeout and 0 for default. // Default is 3 seconds. ReadTimeout time.Duration // Timeout for socket writes. If reached, commands will fail // with a timeout instead of blocking. // Default is ReadTimeout. WriteTimeout time.Duration // Dialer creates new network connection and has priority over // Network and Addr options. Dialer func(ctx context.Context, network, addr string) (net.Conn, error) // Hook that is called when new connection is established. OnConnect func(ctx context.Context, cn *redis.Conn) error // Maximum number of retries before giving up. // Default is 3 retries; -1 (not 0) disables retries. MaxRetries int // Minimum backoff between each retry. // Default is 8 milliseconds; -1 disables backoff. MinRetryBackoff time.Duration // Maximum backoff between each retry. // Default is 512 milliseconds; -1 disables backoff. MaxRetryBackoff time.Duration // Type of connection pool. // true for FIFO pool, false for LIFO pool. // Note that fifo has higher overhead compared to lifo. PoolFIFO bool // Maximum number of socket connections. // Default is 10 connections per every available CPU as reported by runtime.GOMAXPROCS. PoolSize int // Minimum number of idle connections which is useful when establishing // new connection is slow. MinIdleConns int // Connection age at which client retires (closes) the connection. // Default is to not close aged connections. MaxConnAge time.Duration // Amount of time client waits for connection if all connections // are busy before returning an error. // Default is ReadTimeout + 1 second. PoolTimeout time.Duration // Amount of time after which client closes idle connections. // Should be less than server's timeout. // Default is 5 minutes. -1 disables idle timeout check. IdleTimeout time.Duration // TLS Config to use. When set TLS will be negotiated. TLSConfig *tls.Config // Limiter interface used to implemented circuit breaker or rate limiter. Limiter redis.Limiter } // NewClient returns a new configuration object for clients. func NewClient() *Client { c := &Client{ Authentication: &Authentication{}, } err := c.Sanitize() if err != nil { panic(fmt.Sprintf("failed to create a new client configuration: %v", err)) } return c } // Sanitize sets default values to empty configuration variables, if it's possible. func (c *Client) Sanitize() error { if err := c.Authentication.Sanitize(); err != nil { return fmt.Errorf("failed to sanitize authentication configuration: %w", err) } if c.DialTimeout == 0 { c.DialTimeout = DefaultDialTimeout } if c.Dialer == nil { c.Dialer = func(ctx context.Context, network, addr string) (net.Conn, error) { netDialer := &net.Dialer{ Timeout: c.DialTimeout, KeepAlive: DefaultKeepalive, } if c.TLSConfig == nil { return netDialer.DialContext(ctx, network, addr) } return tls.DialWithDialer(netDialer, network, addr, c.TLSConfig) } } if c.PoolSize == 0 { c.PoolSize = 10 * runtime.GOMAXPROCS(0) } switch c.ReadTimeout { case -1: c.ReadTimeout = 0 case 0: c.ReadTimeout = DefaultReadTimeout } switch c.WriteTimeout { case -1: c.WriteTimeout = 0 case 0: c.WriteTimeout = c.ReadTimeout } if c.PoolTimeout == 0 { c.PoolTimeout = c.ReadTimeout + time.Second } if c.IdleTimeout == 0 { c.IdleTimeout = DefaultIdleTimeout } if c.MaxRetries == -1 { c.MaxRetries = 0 } else if c.MaxRetries == 0 { c.MaxRetries = DefaultMaxRetries } switch c.MinRetryBackoff { case -1: c.MinRetryBackoff = 0 case 0: c.MinRetryBackoff = DefaultMinRetryBackoff } switch c.MaxRetryBackoff { case -1: c.MaxRetryBackoff = 0 case 0: c.MaxRetryBackoff = DefaultMaxRetryBackoff } return nil } // Validate finds errors in the current configuration. func (c *Client) Validate() error { if err := c.Authentication.Validate(); err != nil { return fmt.Errorf("failed to validate authentication configuration: %w", err) } return nil } func (c *Client) RedisOptions() *redis.Options { // Note: IdleCheckFrequency is gone since go-redis no longer checks idle connections. // See https://github.com/redis/go-redis/discussions/2635 options := &redis.Options{ Network: "tcp", Dialer: c.Dialer, OnConnect: c.OnConnect, MaxRetries: c.MaxRetries, MinRetryBackoff: c.MinRetryBackoff, MaxRetryBackoff: c.MaxRetryBackoff, DialTimeout: c.DialTimeout, ReadTimeout: c.ReadTimeout, WriteTimeout: c.WriteTimeout, PoolFIFO: c.PoolFIFO, PoolSize: c.PoolSize, MinIdleConns: c.MinIdleConns, ConnMaxLifetime: c.MaxConnAge, PoolTimeout: c.PoolTimeout, ConnMaxIdleTime: c.IdleTimeout, TLSConfig: c.TLSConfig, Limiter: c.Limiter, } if c.Authentication.Enabled() { options.Password = c.Authentication.Password } return options } // Interface guard var _ IConfig = (*Client)(nil) ================================================ FILE: config/config.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package config import ( "fmt" "io" "log" "net" "os" "strconv" "time" "github.com/hashicorp/memberlist" "github.com/olric-data/olric/hasher" ) // IConfig is an interface that has to be implemented by Config and its nested // structs. It provides a clear and granular way to sanitize and validate // the configuration. type IConfig interface { // Sanitize methods should be used to set defaults. Sanitize() error // Validate method should be used to find configuration errors. Validate() error } const ( // SyncReplicationMode enables sync replication mode which means that the // caller is blocked until write/delete operation is applied by replica // owners. The default mode is SyncReplicationMode SyncReplicationMode = 0 // AsyncReplicationMode enables async replication mode which means that // write/delete operations are done in a background task. AsyncReplicationMode = 1 ) const ( LogLevelDebug = "DEBUG" LogLevelWarn = "WARN" LogLevelError = "ERROR" LogLevelInfo = "INFO" ) const ( // DefaultPort is for Olric DefaultPort = 3320 // DefaultDiscoveryPort is for memberlist DefaultDiscoveryPort = 3322 // DefaultPartitionCount denotes default partition count in the cluster. DefaultPartitionCount = 271 // DefaultLoadFactor is used by the consistent hashing function. Keep it small. DefaultLoadFactor = 1.25 // DefaultLogLevel determines the log level without extra configuration. // It's DEBUG. DefaultLogLevel = LogLevelDebug // DefaultLogVerbosity denotes default log verbosity level. // // * flog.V(1) - Generally useful for this to ALWAYS be visible to an operator // * Programmer errors // * Logging extra info about a panic // * CLI argument handling // * flog.V(2) - A reasonable default log level if you don't want verbosity. // * Information about config (listening on X, watching Y) // * Errors that repeat frequently that relate to conditions that can be // corrected (pod detected as unhealthy) // * flog.V(3) - Useful steady state information about the service and // important log messages that may correlate to // significant changes in the system. This is the recommended default log // level for most systems. // * Logging HTTP requests and their exit code // * System state changing (killing pod) // * Controller state change events (starting pods) // * Scheduler log messages // * flog.V(4) - Extended information about changes // * More info about system state changes // * flog.V(5) - Debug level verbosity // * Logging in particularly thorny parts of code where you may want to come // back later and check it // * flog.V(6) - Trace level verbosity // * Context to understand the steps leading up to neterrors and warnings // * More information for troubleshooting reported issues DefaultLogVerbosity = 3 // MinimumReplicaCount denotes default and minimum replica count in an Olric // cluster. MinimumReplicaCount = 1 // DefaultBootstrapTimeout denotes default timeout value to check bootstrapping // status. DefaultBootstrapTimeout = 10 * time.Second // DefaultJoinRetryInterval denotes a time gap between sequential join attempts. DefaultJoinRetryInterval = time.Second // DefaultMaxJoinAttempts denotes a maximum number of failed join attempts // before forming a standalone cluster. DefaultMaxJoinAttempts = 10 // MinimumMemberCountQuorum denotes minimum required count of members to form // a cluster. MinimumMemberCountQuorum = 1 // DefaultLRUSamples is a sane default for randomly selected keys // in approximate LRU implementation. It's 5. DefaultLRUSamples int = 5 // LRUEviction assigns this as EvictionPolicy in order to enable LRU eviction // algorithm. LRUEviction EvictionPolicy = "LRU" // DefaultStorageEngine denotes the storage engine implementation provided by // Olric project. DefaultStorageEngine = "ramblock" // DefaultRoutingTablePushInterval is interval between routing table push events. DefaultRoutingTablePushInterval = time.Minute // DefaultTriggerBalancerInterval is interval between two sequential call of balancer worker. DefaultTriggerBalancerInterval = 15 * time.Second // DefaultCheckEmptyFragmentsInterval is the default value of interval between // two sequential call of empty fragment cleaner. It's one minute by default. DefaultCheckEmptyFragmentsInterval = time.Minute // DefaultTriggerCompactionInterval is the default value of interval between // two sequential call of compaction workers. The compaction worker works until // its work is done. It's 10 minutes by default. DefaultTriggerCompactionInterval = 10 * time.Minute // DefaultLeaveTimeout is the default value of maximum amount of time before DefaultLeaveTimeout = 5 * time.Second DefaultReadQuorum = 1 DefaultWriteQuorum = 1 DefaultMemberCountQuorum = 1 // DefaultKeepAlivePeriod is the default value of TCP keepalive. It's 300 seconds. // This option is useful in order to detect dead peers (clients that cannot // be reached even if they look connected). Moreover, if there is network // equipment between clients and servers that need to see some traffic in // order to take the connection open, the option will prevent unexpected // connection closed events. DefaultKeepAlivePeriod = 300 * time.Second ) // Config represents the configuration structure for customizing the behavior and properties of Olric. type Config struct { // Authentication defines authentication settings, including password protection, for securing access. Authentication *Authentication // Interface denotes a binding interface. It can be used instead of BindAddr // if the interface is known but not the address. If both are provided, then // Olric verifies that the interface has the bind address that is provided. Interface string // LogVerbosity denotes the level of message verbosity. The default value // is 3. Valid values are between 1 to 6. LogVerbosity int32 // Default LogLevel is DEBUG. Available levels: "DEBUG", "WARN", "ERROR", "INFO" LogLevel string // BindAddr denotes the address that Olric will bind to for communication // with other Olric nodes. BindAddr string // BindPort denotes the address that Olric will bind to for communication // with other Olric nodes. BindPort int // Client denotes configuration for TCP clients in Olric and the official // Golang client. Client *Client // KeepAlivePeriod denotes whether the operating system should send // keep-alive messages on the connection. KeepAlivePeriod time.Duration // IdleClose will automatically close idle connections after the specified duration. // Use zero to disable this feature. IdleClose time.Duration // Timeout for bootstrap control // // An Olric node checks operation status before taking any action for the // cluster events, responding incoming requests and running API functions. // Bootstrapping status is one of the most important checkpoints for an // "operable" Olric node. BootstrapTimeout sets a deadline to check // bootstrapping status without blocking indefinitely. BootstrapTimeout time.Duration // Coordinator member pushes the routing table to cluster members in the case of // node join or left events. It also pushes the table periodically. RoutingTablePushInterval // is the interval between subsequent calls. Default is 1 minute. RoutingTablePushInterval time.Duration // TriggerBalancerInterval is interval between two sequential call of balancer worker. TriggerBalancerInterval time.Duration // The list of host:port which are used by memberlist for discovery. // Don't confuse it with Name. Peers []string // PartitionCount is 271, by default. PartitionCount uint64 // ReplicaCount is 1, by default. ReplicaCount int // Minimum number of successful reads to return a response for a read request. ReadQuorum int // Minimum number of successful writes to return a response for a write request. WriteQuorum int // Minimum number of members to form a cluster and run any query on the cluster. MemberCountQuorum int32 // Switch to control read-repair algorithm which helps to reduce entropy. ReadRepair bool // Default value is SyncReplicationMode. ReplicationMode int // LoadFactor is used by consistent hashing function. It determines the maximum // load for a server in the cluster. Keep it small. LoadFactor float64 // Olric can send push cluster events to cluster.events channel. Available cluster events: // // * node-join-event // * node-left-event // * fragment-migration-event // * fragment-received-event // // If you want to receive these events, set true to EnableClusterEventsChannel and subscribe to // cluster.events channel. Default is false. EnableClusterEventsChannel bool // Default hasher is github.com/cespare/xxhash/v2 Hasher hasher.Hasher // LogOutput is the writer where logs should be sent when no custom logger // is provided. If unset, stderr is used by default. // If Logger is set, LogOutput is ignored. LogOutput io.Writer // Logger is a user-provided custom logger. When this is set, Olric will use // it as-is and will not inspect or modify LogOutput. Logger *log.Logger // DMaps denotes a global configuration for DMaps. You can still overwrite it // by setting a DMap for a particular distributed map via DMaps.Custom field. // Most of the fields are related with distributed cache implementation. DMaps *DMaps // JoinRetryInterval is the time gap between attempts to join an existing // cluster. JoinRetryInterval time.Duration // MaxJoinAttempts denotes the maximum number of attempts to join an existing // cluster before forming a new one. MaxJoinAttempts int // Callback function. Olric calls this after // the server is ready to accept new connections. Started func() // ServiceDiscovery is a map that contains plugins implement ServiceDiscovery // interface. See pkg/service_discovery/service_discovery.go for details. ServiceDiscovery map[string]interface{} // Interface denotes a binding interface. It can be used instead of // memberlist.Loader.BindAddr if the interface is known but not the address. // If both are provided, then Olric verifies that the interface has the bind // address that is provided. MemberlistInterface string // Olric will broadcast a leave message but will not shut down the background // listeners, meaning the node will continue participating in gossip and state // updates. // // Sending a leave message will block until the leave message is successfully // broadcast to a member of the cluster, if any exist or until a specified timeout // is reached. LeaveTimeout time.Duration // MemberlistConfig is the memberlist configuration that Olric will // use to do the underlying membership management and gossip. Some // fields in the MemberlistConfig will be overwritten by Olric no // matter what: // // * Name - This will always be set to the same as the NodeName // in this configuration. // // * ClusterEvents - Olric uses a custom event delegate. // // * Delegate - Olric uses a custom delegate. // // You have to use NewMemberlistConfig to create a new one. // Then, you may need to modify it to tune for your environment. MemberlistConfig *memberlist.Config } // Validate finds errors in the current configuration. func (c *Config) Validate() error { if c.ReplicaCount < MinimumReplicaCount { return fmt.Errorf("cannot specify ReplicaCount smaller than MinimumReplicaCount") } if c.ReadQuorum <= 0 { return fmt.Errorf("cannot specify ReadQuorum less than or equal to zero") } if c.ReplicaCount < c.ReadQuorum { return fmt.Errorf("cannot specify ReadQuorum greater than ReplicaCount") } if c.WriteQuorum <= 0 { return fmt.Errorf("cannot specify WriteQuorum less than or equal to zero") } if c.ReplicaCount < c.WriteQuorum { return fmt.Errorf("cannot specify WriteQuorum greater than ReplicaCount") } if err := c.validateMemberlistConfig(); err != nil { return err } if c.MemberCountQuorum < MinimumMemberCountQuorum { return fmt.Errorf("cannot specify MemberCountQuorum smaller than MinimumMemberCountQuorum") } if c.BindAddr == "" { return fmt.Errorf("bindAddr cannot be empty") } if c.BindPort == 0 { return fmt.Errorf("bindPort cannot be empty or zero") } // Check peers. If Peers slice contains node's itself, return an error. port := strconv.Itoa(c.MemberlistConfig.BindPort) this := net.JoinHostPort(c.MemberlistConfig.BindAddr, port) for _, peer := range c.Peers { if this == peer { return fmt.Errorf("cannot be peer with itself") } } if err := c.Client.Validate(); err != nil { return fmt.Errorf("failed to validate client configuration: %w", err) } if err := c.DMaps.Validate(); err != nil { return fmt.Errorf("failed to validate DMap configuration: %w", err) } if err := c.Authentication.Validate(); err != nil { return fmt.Errorf("failed to sanitize authentication configuration: %w", err) } switch c.LogLevel { case LogLevelDebug, LogLevelWarn, LogLevelInfo, LogLevelError: default: return fmt.Errorf("invalid LogLevel: %s", c.LogLevel) } return nil } // Sanitize sets default values to empty configuration variables, if it's possible. func (c *Config) Sanitize() error { if c.LogOutput == nil { c.LogOutput = os.Stderr } if c.LogLevel == "" { c.LogLevel = DefaultLogLevel } if c.LogVerbosity <= 0 { c.LogVerbosity = DefaultLogVerbosity } if c.Logger == nil { c.Logger = log.New(c.LogOutput, "", log.LstdFlags) } if c.Hasher == nil { c.Hasher = hasher.NewDefaultHasher() } if c.BindAddr == "" { name, err := os.Hostname() if err != nil { return fmt.Errorf("failed to read hostname from kernel: %w", err) } c.BindAddr = name } // We currently don't support ephemeral port selection. Because it needs // improved flow control in server initialization stage. if c.BindPort == 0 { c.BindPort = DefaultPort } if c.LoadFactor == 0 { c.LoadFactor = DefaultLoadFactor } if c.PartitionCount == 0 { c.PartitionCount = DefaultPartitionCount } if c.ReplicaCount == 0 { c.ReplicaCount = MinimumReplicaCount } if c.ReadQuorum == 0 { c.ReadQuorum = DefaultReadQuorum } if c.WriteQuorum == 0 { c.WriteQuorum = DefaultWriteQuorum } if c.MemberCountQuorum == 0 { c.MemberCountQuorum = DefaultMemberCountQuorum } if c.MemberlistConfig == nil { m := memberlist.DefaultLocalConfig() // hostname is assigned to memberlist.BindAddr // memberlist.Name is assigned by olric.New m.BindPort = DefaultDiscoveryPort m.AdvertisePort = DefaultDiscoveryPort c.MemberlistConfig = m } if c.BootstrapTimeout == 0 { c.BootstrapTimeout = DefaultBootstrapTimeout } if c.JoinRetryInterval == 0 { c.JoinRetryInterval = DefaultJoinRetryInterval } if c.MaxJoinAttempts == 0 { c.MaxJoinAttempts = DefaultMaxJoinAttempts } if c.LeaveTimeout == 0 { c.LeaveTimeout = DefaultLeaveTimeout } if c.RoutingTablePushInterval == 0 { c.RoutingTablePushInterval = DefaultRoutingTablePushInterval } if c.TriggerBalancerInterval == 0 { c.TriggerBalancerInterval = DefaultTriggerBalancerInterval } if c.KeepAlivePeriod == 0 { c.KeepAlivePeriod = DefaultKeepAlivePeriod } if c.Client == nil { c.Client = NewClient() } if c.DMaps == nil { c.DMaps = &DMaps{} } if c.Authentication == nil { c.Authentication = &Authentication{} } if err := c.Authentication.Sanitize(); err != nil { return fmt.Errorf("failed to sanitize authentication configuration: %w", err) } if err := c.Client.Sanitize(); err != nil { return fmt.Errorf("failed to sanitize TCP client configuration: %w", err) } if err := c.DMaps.Sanitize(); err != nil { return fmt.Errorf("failed to sanitize DMap configuration: %w", err) } return nil } // New returns a Config with sane defaults. If you change a configuration parameter, // please run Sanitize and Validate functions respectively. // // New takes an env parameter used by memberlist: local, lan and wan. // // local: // // DefaultLocalConfig works like DefaultConfig, however it returns a configuration // that is optimized for a local loopback environments. The default configuration // is still very conservative and errs on the side of caution. // // lan: // // DefaultLANConfig returns a sane set of configurations for Memberlist. It uses // the hostname as the node name, and otherwise sets very conservative values // that are sane for most LAN environments. The default configuration errs on // the side of caution, choosing values that are optimized for higher convergence // at the cost of higher bandwidth usage. Regardless, these values are a good // starting point when getting started with memberlist. // // wan: // // DefaultWANConfig works like DefaultConfig, however it returns a configuration // that is optimized for most WAN environments. The default configuration is still // very conservative and errs on the side of caution. func New(env string) *Config { c := &Config{ BindAddr: "0.0.0.0", BindPort: DefaultPort, ReadRepair: false, ReplicaCount: 1, WriteQuorum: 1, ReadQuorum: 1, MemberCountQuorum: 1, Peers: []string{}, DMaps: &DMaps{}, Authentication: &Authentication{}, } m, err := NewMemberlistConfig(env) if err != nil { panic(fmt.Sprintf("unable to create a new memberlist config: %v", err)) } // memberlist.Name will be assigned by olric.New m.BindPort = DefaultDiscoveryPort m.AdvertisePort = DefaultDiscoveryPort c.MemberlistConfig = m if err := c.Sanitize(); err != nil { panic(fmt.Sprintf("unable to sanitize Olric config: %v", err)) } if err := c.Validate(); err != nil { panic(fmt.Sprintf("unable to validate Olric config: %v", err)) } return c } // Interface guard var _ IConfig = (*Config)(nil) ================================================ FILE: config/config_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package config import ( "bytes" "io/ioutil" "os" "testing" "time" "github.com/stretchr/testify/require" ) var testConfig = `server: bindAddr: "0.0.0.0" bindPort: 3320 serializer: "msgpack" keepAlivePeriod: "300s" idleClose: 300s bootstrapTimeout: "5s" partitionCount: 271 replicaCount: 2 writeQuorum: 1 readQuorum: 1 readRepair: false replicationMode: 0 # sync mode. for async, set 1 memberCountQuorum: 1 enableClusterEventsChannel: true authentication: password: "secret" client: dialTimeout: 8s readTimeout: 2s writeTimeout: 2s maxRetries: 5 minRetryBackoff: 10ms maxRetryBackoff: 520ms poolFIFO: true poolSize: 10 minIdleConns: 5 maxConnAge: 2h poolTimeout: 4s idleTimeout: 6m logging: verbosity: 6 level: "DEBUG" output: "stderr" memberlist: environment: "local" bindAddr: "0.0.0.0" bindPort: 3322 enableCompression: false joinRetryInterval: "1s" maxJoinAttempts: 10 peers: - "localhost:3325" advertiseAddr: "" advertisePort: 3322 suspicionMaxTimeoutMult: 6 disableTCPPings: false awarenessMaxMultiplier: 8 gossipNodes: 3 gossipVerifyIncoming: true gossipVerifyOutgoing: true dnsConfigPath: "/etc/resolv.conf" handoffQueueDepth: 1024 udpBufferSize: 1400 dmaps: engine: name: ramblock config: tableSize: 202134 numEvictionWorkers: 2 maxIdleDuration: 100s ttlDuration: 200s maxKeys: 300000 maxInuse: 2000000 lruSamples: 20 evictionPolicy: "LRU" custom: foobar: maxIdleDuration: "30s" ttlDuration: "500s" maxKeys: 600000 lruSamples: 60 evictionPolicy: "NONE" serviceDiscovery: path: "/usr/lib/olric-consul-plugin.so" provider: "consul" address: "http://consul:8500" passingOnly: true replaceExistingChecks: true insecureSkipVerify: true payload: 'SAMPLE-PAYLOAD'` func createTmpFile(t *testing.T, pattern string) *os.File { f, err := ioutil.TempFile("/tmp/", pattern) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } t.Cleanup(func() { err = f.Close() if err != nil { t.Fatalf("Expected nil. Got: %v", err) } err = os.Remove(f.Name()) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } }) return f } func TestConfig(t *testing.T) { w := bytes.NewBuffer([]byte(testConfig)) f := createTmpFile(t, "olric-yaml-config-test") _, err := f.Write(w.Bytes()) require.NoError(t, err) lc, err := Load(f.Name()) require.NoError(t, err) c := New("local") c.BindAddr = "0.0.0.0" c.BindPort = 3320 c.KeepAlivePeriod = 300 * time.Second c.IdleClose = 300 * time.Second c.BootstrapTimeout = 5 * time.Second c.PartitionCount = 271 c.ReplicaCount = 2 c.WriteQuorum = 1 c.ReadQuorum = 1 c.ReadRepair = false c.ReplicationMode = SyncReplicationMode c.MemberCountQuorum = 1 c.EnableClusterEventsChannel = true c.DMaps.Engine = NewEngine() c.Client.DialTimeout = 8 * time.Second c.Client.ReadTimeout = 2 * time.Second c.Client.WriteTimeout = 2 * time.Second c.Client.MaxRetries = 5 c.Client.MinRetryBackoff = 10 * time.Millisecond c.Client.MaxRetryBackoff = 520 * time.Millisecond c.Client.PoolFIFO = true c.Client.PoolSize = 10 c.Client.MinIdleConns = 5 c.Client.MaxConnAge = 2 * time.Hour c.Client.PoolTimeout = 4 * time.Second c.Client.IdleTimeout = 6 * time.Minute c.LogVerbosity = 6 c.LogLevel = "DEBUG" c.MemberlistConfig.BindAddr = "0.0.0.0" c.MemberlistConfig.BindPort = 3322 c.MemberlistConfig.EnableCompression = false c.JoinRetryInterval = time.Second c.MaxJoinAttempts = 10 c.Peers = []string{"localhost:3325"} c.MemberlistConfig.AdvertisePort = 3322 c.MemberlistConfig.SuspicionMaxTimeoutMult = 6 c.MemberlistConfig.DisableTcpPings = false c.MemberlistConfig.AwarenessMaxMultiplier = 8 c.MemberlistConfig.GossipNodes = 3 c.MemberlistConfig.GossipVerifyIncoming = true c.MemberlistConfig.GossipVerifyOutgoing = true c.MemberlistConfig.DNSConfigPath = "/etc/resolv.conf" c.MemberlistConfig.HandoffQueueDepth = 1024 c.MemberlistConfig.UDPBufferSize = 1400 c.DMaps.NumEvictionWorkers = 2 c.DMaps.TTLDuration = 200 * time.Second c.DMaps.MaxIdleDuration = 100 * time.Second c.DMaps.MaxKeys = 300000 c.DMaps.MaxInuse = 2000000 c.DMaps.LRUSamples = 20 c.DMaps.EvictionPolicy = LRUEviction c.DMaps.Engine.Name = DefaultStorageEngine c.DMaps.Engine.Config = map[string]interface{}{"tableSize": 202134} c.DMaps.Custom = map[string]DMap{"foobar": { MaxIdleDuration: 30 * time.Second, TTLDuration: 500 * time.Second, MaxKeys: 600000, LRUSamples: 60, EvictionPolicy: "NONE", }} c.ServiceDiscovery = make(map[string]interface{}) c.ServiceDiscovery["path"] = "/usr/lib/olric-consul-plugin.so" c.ServiceDiscovery["provider"] = "consul" c.ServiceDiscovery["address"] = "http://consul:8500" c.ServiceDiscovery["passingOnly"] = true c.ServiceDiscovery["replaceExistingChecks"] = true c.ServiceDiscovery["insecureSkipVerify"] = true c.ServiceDiscovery["payload"] = "SAMPLE-PAYLOAD" c.Authentication = &Authentication{ Password: "secret", } c.Client.Authentication = c.Authentication err = c.Sanitize() require.NoError(t, err) // Disable the following fields. They include unexported fields, pointers and mutexes. c.LogOutput = nil lc.LogOutput = nil c.Logger = nil lc.Logger = nil c.Client.Dialer = nil lc.Client.Dialer = nil require.Equal(t, c, lc) } func TestConfig_Initialize(t *testing.T) { c := &Config{} require.NoError(t, c.Sanitize()) require.NoError(t, c.Validate()) } ================================================ FILE: config/dmap.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package config import ( "fmt" "time" ) // EvictionPolicy denotes eviction policy. Currently: LRU or NONE. type EvictionPolicy string // Important note on DMap and DMaps structs: // Golang does not provide the typical notion of inheritance. // because of that I preferred to define the types explicitly. // DMap denotes configuration for a particular distributed map. Most of the // fields are related with distributed cache implementation. type DMap struct { // Engine contains storage engine configuration and their implementations. // If you don't have a custom storage engine implementation or configuration for // the default one, just leave it empty. Engine *Engine // MaxIdleDuration denotes maximum time for each entry to stay idle in the // DMap. It limits the lifetime of the entries relative to the time of the // last read or write access performed on them. The entries whose idle period // exceeds this limit are expired and evicted automatically. An entry is idle // if no Get, GetEntry, Put, Expire on it. Configuration // of MaxIdleDuration feature varies by preferred deployment method. MaxIdleDuration time.Duration // TTLDuration is useful to set a default TTL for every key/value pair a DMap // instance. TTLDuration time.Duration // MaxKeys denotes maximum key count on a particular node. So if you have 10 // nodes with MaxKeys=100000, your key count in the cluster should be around // MaxKeys*10=1000000 MaxKeys int // MaxInuse denotes maximum amount of in-use memory on a particular node. So // if you have 10 nodes with MaxInuse=100M (it has to be in bytes), amount of // in-use memory should be around MaxInuse*10=1G MaxInuse int // LRUSamples denotes amount of randomly selected key count by the approximate // LRU implementation. Lower values are better for high performance. It's 5 // by default. LRUSamples int // EvictionPolicy determines the eviction policy in use. It's NONE by default. // Set as LRU to enable LRU eviction policy. EvictionPolicy EvictionPolicy } // Sanitize sets default values to empty configuration variables, if it's possible. func (dm *DMap) Sanitize() error { if dm.EvictionPolicy == "" { dm.EvictionPolicy = "NONE" } if dm.LRUSamples <= 0 { dm.LRUSamples = DefaultLRUSamples } if dm.MaxInuse < 0 { dm.MaxInuse = 0 } if dm.MaxKeys < 0 { dm.MaxKeys = 0 } if dm.Engine == nil { dm.Engine = NewEngine() } if err := dm.Engine.Sanitize(); err != nil { return fmt.Errorf("failed to sanitize storage engine configuration: %w", err) } return nil } // Validate finds errors in the current configuration. func (dm *DMap) Validate() error { if err := dm.Engine.Validate(); err != nil { return fmt.Errorf("failed to validate storage engine configuration: %w", err) } return nil } var _ IConfig = (*DMap)(nil) ================================================ FILE: config/dmap_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package config import ( "testing" "github.com/stretchr/testify/require" ) func TestConfig_DMap(t *testing.T) { d := &DMap{ MaxInuse: -1, MaxKeys: -1, LRUSamples: -1, } require.NoError(t, d.Sanitize()) require.NoError(t, d.Validate()) require.Greater(t, d.MaxInuse, -1) require.Greater(t, d.MaxKeys, -1) require.Equal(t, DefaultLRUSamples, d.LRUSamples) require.Equal(t, EvictionPolicy("NONE"), d.EvictionPolicy) require.NotNil(t, d.Engine) } ================================================ FILE: config/dmaps.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package config import ( "fmt" "runtime" "time" ) // DMaps denotes a global configuration for DMaps. You can still overwrite it by // setting a DMap for a particular distributed map via Custom field. Most of the // fields are related with distributed cache implementation. type DMaps struct { // Engine contains configuration for a storage engine implementation. It may contain the implementation. // See Engine itself. Engine *Engine // NumEvictionWorkers denotes the number of goroutines that are used to find // keys for eviction. This is a global configuration variable. So you cannot set // // different values per DMap. NumEvictionWorkers int64 // MaxIdleDuration denotes maximum time for each entry to stay idle in the DMap. // It limits the lifetime of the entries relative to the time of the last // read or write access performed on them. The entries whose idle period exceeds // this limit are expired and evicted automatically. An entry is idle if no Get, // Put, Expire on it. Configuration of MaxIdleDuration feature varies by preferred // deployment method. MaxIdleDuration time.Duration // TTLDuration is useful to set a default TTL for every key/value pair a // distributed map instance. TTLDuration time.Duration // MaxKeys denotes maximum key count on a particular node. So if you have 10 // nodes with MaxKeys=100000, your key count in the cluster should be around // MaxKeys*10=1000000 MaxKeys int // MaxInuse denotes maximum amount of in-use memory on a particular node. // So if you have 10 nodes with MaxInuse=100M (it has to be in bytes), amount // of in-use memory should be around MaxInuse*10=1G MaxInuse int // LRUSamples denotes amount of randomly selected key count by the approximate // LRU implementation. Lower values are better for high performance. It's // 5 by default. LRUSamples int // EvictionPolicy determines the eviction policy in use. It's NONE by default. // Set as LRU to enable LRU eviction policy. EvictionPolicy EvictionPolicy // CheckEmptyFragmentsInterval is the interval between two sequential calls of empty // fragment cleaner. This is a global configuration variable. So you cannot set // different values per DMap. CheckEmptyFragmentsInterval time.Duration // TriggerCompactionInterval is interval between two sequential call of compaction worker. // This is a global configuration variable. So you cannot set // different values per DMap. TriggerCompactionInterval time.Duration // Custom is useful to set custom cache config per DMap instance. Custom map[string]DMap } // Sanitize sets default values to empty configuration variables, if it's possible. func (dm *DMaps) Sanitize() error { if dm.Engine == nil { dm.Engine = NewEngine() } if dm.Custom == nil { dm.Custom = make(map[string]DMap) } if dm.EvictionPolicy == "" { dm.EvictionPolicy = "NONE" } if dm.LRUSamples <= 0 { dm.LRUSamples = DefaultLRUSamples } if dm.MaxInuse < 0 { dm.MaxInuse = 0 } if dm.MaxKeys < 0 { dm.MaxKeys = 0 } if dm.NumEvictionWorkers <= 0 { dm.NumEvictionWorkers = int64(runtime.NumCPU()) } if dm.CheckEmptyFragmentsInterval.Microseconds() == 0 { dm.CheckEmptyFragmentsInterval = DefaultCheckEmptyFragmentsInterval } if dm.TriggerCompactionInterval.Microseconds() == 0 { dm.TriggerCompactionInterval = DefaultTriggerCompactionInterval } for _, d := range dm.Custom { if err := d.Sanitize(); err != nil { return err } } if err := dm.Engine.Sanitize(); err != nil { return fmt.Errorf("failed to sanitize storage engine configuration: %w", err) } return nil } func (dm *DMaps) Validate() error { if err := dm.Engine.Validate(); err != nil { return fmt.Errorf("failed to validate storage engine configuration: %w", err) } return nil } var _ IConfig = (*DMaps)(nil) ================================================ FILE: config/engine.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package config import ( "fmt" "github.com/olric-data/olric/internal/ramblock" "github.com/olric-data/olric/pkg/storage" ) // Engine contains storage engine configuration and their implementations. // If you don't have a custom storage engine implementation or configuration for // the default one, just call NewStorageEngine() function to use it with sane defaults. type Engine struct { Name string Implementation storage.Engine // Config is a map that contains configuration of the storage engines, for // both plugins and imported ones. If you want to use a storage engine other // than the default one, you must set configuration for it. Config map[string]interface{} } // NewEngine initializes Engine with sane defaults. // Olric will set its own storage engine implementation and related configuration, // if there is no other engine. func NewEngine() *Engine { return &Engine{ Config: make(map[string]interface{}), } } // Validate finds errors in the current configuration. func (s *Engine) Validate() error { if s.Config == nil { s.Config = make(map[string]interface{}) } return nil } // Sanitize sets default values to empty configuration variables, if it's possible. func (s *Engine) Sanitize() error { if s.Name == "" { s.Name = DefaultStorageEngine } // Backward compatibility: accept the old name "kvstore" if s.Name == "kvstore" { s.Name = DefaultStorageEngine } if s.Implementation == nil { switch s.Name { case DefaultStorageEngine: cfg := ramblock.DefaultConfig().ToMap() for key, value := range cfg { _, ok := s.Config[key] if !ok { s.Config[key] = value } } kv, err := ramblock.New(storage.NewConfig(s.Config)) if err != nil { return err } s.Implementation = kv default: return fmt.Errorf("unknown storage engine: %s", s.Name) } } else { s.Name = s.Implementation.Name() } return nil } // Interface guard var _ IConfig = (*Engine)(nil) ================================================ FILE: config/engine_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package config import ( "testing" "github.com/stretchr/testify/require" ) func TestEngine_KVStore_Backward_Compat(t *testing.T) { e := NewEngine() e.Name = "kvstore" require.NoError(t, e.Sanitize()) require.NoError(t, e.Validate()) require.Equal(t, DefaultStorageEngine, e.Name) require.NotNil(t, e.Implementation) } func TestEngine_Dont_Overwrite_TableSize(t *testing.T) { e := NewEngine() e.Name = DefaultStorageEngine e.Config = map[string]interface{}{ "tableSize": 1235, } require.NoError(t, e.Sanitize()) require.NoError(t, e.Validate()) require.Equal(t, 1235, e.Config["tableSize"]) } ================================================ FILE: config/internal/loader/loader.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package loader import "gopkg.in/yaml.v2" type server struct { Name string `yaml:"name"` BindAddr string `yaml:"bindAddr"` BindPort int `yaml:"bindPort"` Interface string `yaml:"interface"` ReplicationMode int `yaml:"replicationMode"` PartitionCount uint64 `yaml:"partitionCount"` LoadFactor float64 `yaml:"loadFactor"` KeepAlivePeriod string `yaml:"keepAlivePeriod"` IdleClose string `yaml:"idleClose"` BootstrapTimeout string `yaml:"bootstrapTimeout"` ReplicaCount int `yaml:"replicaCount"` WriteQuorum int `yaml:"writeQuorum"` ReadQuorum int `yaml:"readQuorum"` ReadRepair bool `yaml:"readRepair"` MemberCountQuorum int32 `yaml:"memberCountQuorum"` RoutingTablePushInterval string `yaml:"routingTablePushInterval"` TriggerBalancerInterval string `yaml:"triggerBalancerInterval"` LeaveTimeout string `yaml:"leaveTimeout"` EnableClusterEventsChannel bool `yaml:"enableClusterEventsChannel"` } type authentication struct { Password string `yaml:"password"` } type client struct { DialTimeout string `yaml:"dialTimeout"` ReadTimeout string `yaml:"readTimeout"` WriteTimeout string `yaml:"writeTimeout"` MaxRetries int `yaml:"maxRetries"` MinRetryBackoff string `yaml:"minRetryBackoff"` MaxRetryBackoff string `yaml:"maxRetryBackoff"` PoolFIFO bool `yaml:"poolFIFO"` PoolSize int `yaml:"poolSize"` MinIdleConns int `yaml:"minIdleConns"` MaxConnAge string `yaml:"maxConnAge"` PoolTimeout string `yaml:"poolTimeout"` IdleTimeout string `yaml:"idleTimeout"` } // logging contains configuration variables of logging section of config file. type logging struct { Verbosity int32 `yaml:"verbosity"` Level string `yaml:"level"` Output string `yaml:"output"` } type memberlist struct { Environment string `yaml:"environment"` // required BindAddr string `yaml:"bindAddr"` // required BindPort int `yaml:"bindPort"` // required Interface string `yaml:"interface"` EnableCompression *bool `yaml:"enableCompression"` JoinRetryInterval string `yaml:"joinRetryInterval"` // required MaxJoinAttempts int `yaml:"maxJoinAttempts"` // required Peers []string `yaml:"peers"` IndirectChecks *int `yaml:"indirectChecks"` RetransmitMult *int `yaml:"retransmitMult"` SuspicionMult *int `yaml:"suspicionMult"` TCPTimeout *string `yaml:"tcpTimeout"` PushPullInterval *string `yaml:"pushPullInterval"` ProbeTimeout *string `yaml:"probeTimeout"` ProbeInterval *string `yaml:"probeInterval"` GossipInterval *string `yaml:"gossipInterval"` GossipToTheDeadTime *string `yaml:"gossipToTheDeadTime"` AdvertiseAddr *string `yaml:"advertiseAddr"` AdvertisePort *int `yaml:"advertisePort"` SuspicionMaxTimeoutMult *int `yaml:"suspicionMaxTimeoutMult"` DisableTCPPings *bool `yaml:"disableTCPPings"` AwarenessMaxMultiplier *int `yaml:"awarenessMaxMultiplier"` GossipNodes *int `yaml:"gossipNodes"` GossipVerifyIncoming *bool `yaml:"gossipVerifyIncoming"` GossipVerifyOutgoing *bool `yaml:"gossipVerifyOutgoing"` DNSConfigPath *string `yaml:"dnsConfigPath"` HandoffQueueDepth *int `yaml:"handoffQueueDepth"` UDPBufferSize *int `yaml:"udpBufferSize"` } type engine struct { Name string `yaml:"name"` Config map[string]interface{} `yaml:"config"` } type dmap struct { Engine *engine `yaml:"engine"` MaxIdleDuration string `yaml:"maxIdleDuration"` TTLDuration string `yaml:"ttlDuration"` MaxKeys int `yaml:"maxKeys"` MaxInuse int `yaml:"maxInuse"` LRUSamples int `yaml:"lruSamples"` EvictionPolicy string `yaml:"evictionPolicy"` } type dmaps struct { Engine *engine `yaml:"engine"` NumEvictionWorkers int64 `yaml:"numEvictionWorkers"` MaxIdleDuration string `yaml:"maxIdleDuration"` TTLDuration string `yaml:"ttlDuration"` MaxKeys int `yaml:"maxKeys"` MaxInuse int `yaml:"maxInuse"` LRUSamples int `yaml:"lruSamples"` EvictionPolicy string `yaml:"evictionPolicy"` CheckEmptyFragmentsInterval string `yaml:"checkEmptyFragmentsInterval"` TriggerCompactionInterval string `yaml:"triggerCompactionInterval"` Custom map[string]dmap `yaml:"custom"` } type serviceDiscovery map[string]interface{} // Loader is the main configuration struct type Loader struct { Memberlist memberlist `yaml:"memberlist"` Logging logging `yaml:"logging"` Server server `yaml:"server"` Client client `yaml:"client"` DMaps dmaps `yaml:"dmaps"` ServiceDiscovery serviceDiscovery `yaml:"serviceDiscovery"` Authentication authentication `yaml:"authentication"` } // New tries to read Olric configuration from a YAML file. func New(data []byte) (*Loader, error) { var lc Loader if err := yaml.Unmarshal(data, &lc); err != nil { return nil, err } return &lc, nil } ================================================ FILE: config/load.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package config import ( "fmt" "io" "io/ioutil" "log" "os" "reflect" "time" "github.com/hashicorp/memberlist" "github.com/olric-data/olric/config/internal/loader" "github.com/olric-data/olric/hasher" "github.com/pkg/errors" ) // mapYamlToConfig maps a parsed YAML to related configuration struct. func mapYamlToConfig(rawDst, rawSrc interface{}) error { dst := reflect.ValueOf(rawDst).Elem() src := reflect.ValueOf(rawSrc).Elem() for j := 0; j < src.NumField(); j++ { for i := 0; i < dst.NumField(); i++ { if src.Type().Field(j).Name == dst.Type().Field(i).Name { if src.Field(j).Kind() == dst.Field(i).Kind() { dst.Field(i).Set(src.Field(j)) continue } // Special cases if dst.Field(i).Type() == reflect.TypeOf(time.Duration(0)) { rawValue := src.Field(j).String() if rawValue != "" { value, err := time.ParseDuration(rawValue) if err != nil { return err } dst.Field(i).Set(reflect.ValueOf(value)) } continue } return fmt.Errorf("failed to map %s to an appropriate field in config", dst.Type().Field(j).Name) } } } return nil } func loadDMapConfig(c *loader.Loader) (*DMaps, error) { res := &DMaps{} if c.DMaps.MaxIdleDuration != "" { maxIdleDuration, err := time.ParseDuration(c.DMaps.MaxIdleDuration) if err != nil { return nil, errors.WithMessage(err, "failed to parse dmap.MaxIdleDuration") } res.MaxIdleDuration = maxIdleDuration } if c.DMaps.TTLDuration != "" { ttlDuration, err := time.ParseDuration(c.DMaps.TTLDuration) if err != nil { return nil, errors.WithMessage(err, "failed to parse dmap.TTLDuration") } res.TTLDuration = ttlDuration } if c.DMaps.CheckEmptyFragmentsInterval != "" { checkEmptyFragmentsInterval, err := time.ParseDuration(c.DMaps.CheckEmptyFragmentsInterval) if err != nil { return nil, errors.WithMessage(err, "failed to parse dmap.MaxIdleDuration") } res.CheckEmptyFragmentsInterval = checkEmptyFragmentsInterval } if c.DMaps.TriggerCompactionInterval != "" { triggerCompactionInterval, err := time.ParseDuration(c.DMaps.TriggerCompactionInterval) if err != nil { return nil, errors.WithMessage(err, "failed to parse dmap.triggerCompactionInterval") } res.TriggerCompactionInterval = triggerCompactionInterval } res.NumEvictionWorkers = c.DMaps.NumEvictionWorkers res.MaxKeys = c.DMaps.MaxKeys res.MaxInuse = c.DMaps.MaxInuse res.EvictionPolicy = EvictionPolicy(c.DMaps.EvictionPolicy) res.LRUSamples = c.DMaps.LRUSamples if c.DMaps.Engine != nil { e := NewEngine() e.Name = c.DMaps.Engine.Name e.Config = c.DMaps.Engine.Config res.Engine = e } if c.DMaps.Custom != nil { res.Custom = make(map[string]DMap) for name, dc := range c.DMaps.Custom { cc := DMap{ MaxInuse: dc.MaxInuse, MaxKeys: dc.MaxKeys, EvictionPolicy: EvictionPolicy(dc.EvictionPolicy), LRUSamples: dc.LRUSamples, } if dc.Engine != nil { e := NewEngine() e.Name = dc.Engine.Name e.Config = dc.Engine.Config cc.Engine = e } if dc.MaxIdleDuration != "" { maxIdleDuration, err := time.ParseDuration(dc.MaxIdleDuration) if err != nil { return nil, errors.WithMessagef(err, "failed to parse dmaps.%s.MaxIdleDuration", name) } cc.MaxIdleDuration = maxIdleDuration } if dc.TTLDuration != "" { ttlDuration, err := time.ParseDuration(dc.TTLDuration) if err != nil { return nil, errors.WithMessagef(err, "failed to parse dmaps.%s.TTLDuration", name) } cc.TTLDuration = ttlDuration } res.Custom[name] = cc } } return res, nil } // loadMemberlistConfig creates a new *memberlist.Config by parsing olric.yaml func loadMemberlistConfig(c *loader.Loader, mc *memberlist.Config) (*memberlist.Config, error) { var err error if c.Memberlist.BindAddr == "" { name, err := os.Hostname() if err != nil { return nil, err } c.Memberlist.BindAddr = name } mc.BindAddr = c.Memberlist.BindAddr mc.BindPort = c.Memberlist.BindPort if c.Memberlist.EnableCompression != nil { mc.EnableCompression = *c.Memberlist.EnableCompression } if c.Memberlist.TCPTimeout != nil { mc.TCPTimeout, err = time.ParseDuration(*c.Memberlist.TCPTimeout) if err != nil { return nil, err } } if c.Memberlist.IndirectChecks != nil { mc.IndirectChecks = *c.Memberlist.IndirectChecks } if c.Memberlist.RetransmitMult != nil { mc.RetransmitMult = *c.Memberlist.RetransmitMult } if c.Memberlist.SuspicionMult != nil { mc.SuspicionMult = *c.Memberlist.SuspicionMult } if c.Memberlist.PushPullInterval != nil { mc.PushPullInterval, err = time.ParseDuration(*c.Memberlist.PushPullInterval) if err != nil { return nil, err } } if c.Memberlist.ProbeTimeout != nil { mc.ProbeTimeout, err = time.ParseDuration(*c.Memberlist.ProbeTimeout) if err != nil { return nil, err } } if c.Memberlist.ProbeInterval != nil { mc.ProbeInterval, err = time.ParseDuration(*c.Memberlist.ProbeInterval) if err != nil { return nil, err } } if c.Memberlist.GossipInterval != nil { mc.GossipInterval, err = time.ParseDuration(*c.Memberlist.GossipInterval) if err != nil { return nil, err } } if c.Memberlist.GossipToTheDeadTime != nil { mc.GossipToTheDeadTime, err = time.ParseDuration(*c.Memberlist.GossipToTheDeadTime) if err != nil { return nil, err } } if c.Memberlist.AdvertiseAddr != nil { mc.AdvertiseAddr = *c.Memberlist.AdvertiseAddr } if c.Memberlist.AdvertisePort != nil { mc.AdvertisePort = *c.Memberlist.AdvertisePort } else { mc.AdvertisePort = mc.BindPort } if c.Memberlist.SuspicionMaxTimeoutMult != nil { mc.SuspicionMaxTimeoutMult = *c.Memberlist.SuspicionMaxTimeoutMult } if c.Memberlist.DisableTCPPings != nil { mc.DisableTcpPings = *c.Memberlist.DisableTCPPings } if c.Memberlist.AwarenessMaxMultiplier != nil { mc.AwarenessMaxMultiplier = *c.Memberlist.AwarenessMaxMultiplier } if c.Memberlist.GossipNodes != nil { mc.GossipNodes = *c.Memberlist.GossipNodes } if c.Memberlist.GossipVerifyIncoming != nil { mc.GossipVerifyIncoming = *c.Memberlist.GossipVerifyIncoming } if c.Memberlist.GossipVerifyOutgoing != nil { mc.GossipVerifyOutgoing = *c.Memberlist.GossipVerifyOutgoing } if c.Memberlist.DNSConfigPath != nil { mc.DNSConfigPath = *c.Memberlist.DNSConfigPath } if c.Memberlist.HandoffQueueDepth != nil { mc.HandoffQueueDepth = *c.Memberlist.HandoffQueueDepth } if c.Memberlist.UDPBufferSize != nil { mc.UDPBufferSize = *c.Memberlist.UDPBufferSize } return mc, nil } // Load reads and loads Olric configuration. func Load(filename string) (*Config, error) { if _, err := os.Stat(filename); os.IsNotExist(err) { return nil, fmt.Errorf("file doesn't exists: %s", filename) } f, err := os.Open(filename) if err != nil { return nil, err } data, err := ioutil.ReadAll(f) if err != nil { return nil, err } c, err := loader.New(data) if err != nil { return nil, err } var logOutput io.Writer switch { case c.Logging.Output == "stderr": logOutput = os.Stderr case c.Logging.Output == "stdout": logOutput = os.Stdout default: logOutput = os.Stderr } if c.Logging.Level == "" { c.Logging.Level = DefaultLogLevel } rawMc, err := NewMemberlistConfig(c.Memberlist.Environment) if err != nil { return nil, err } memberlistConfig, err := loadMemberlistConfig(c, rawMc) if err != nil { return nil, err } var ( joinRetryInterval, keepAlivePeriod, idleClose, bootstrapTimeout, triggerBalancerInterval, leaveTimeout, routingTablePushInterval time.Duration ) if c.Server.KeepAlivePeriod != "" { keepAlivePeriod, err = time.ParseDuration(c.Server.KeepAlivePeriod) if err != nil { return nil, errors.WithMessage(err, fmt.Sprintf("failed to parse server.keepAlivePeriod: '%s'", c.Server.KeepAlivePeriod)) } } if c.Server.IdleClose != "" { idleClose, err = time.ParseDuration(c.Server.IdleClose) if err != nil { return nil, errors.WithMessage(err, fmt.Sprintf("failed to parse server.idleClose: '%s'", c.Server.IdleClose)) } } if c.Server.BootstrapTimeout != "" { bootstrapTimeout, err = time.ParseDuration(c.Server.BootstrapTimeout) if err != nil { return nil, errors.WithMessage(err, fmt.Sprintf("failed to parse server.bootstrapTimeout: '%s'", c.Server.BootstrapTimeout)) } } if c.Memberlist.JoinRetryInterval != "" { joinRetryInterval, err = time.ParseDuration(c.Memberlist.JoinRetryInterval) if err != nil { return nil, errors.WithMessage(err, fmt.Sprintf("failed to parse memberlist.joinRetryInterval: '%s'", c.Memberlist.JoinRetryInterval)) } } if c.Server.RoutingTablePushInterval != "" { routingTablePushInterval, err = time.ParseDuration(c.Server.RoutingTablePushInterval) if err != nil { return nil, errors.WithMessage(err, fmt.Sprintf("failed to parse server.routingTablePushInterval: '%s'", c.Server.RoutingTablePushInterval)) } } if c.Server.TriggerBalancerInterval != "" { triggerBalancerInterval, err = time.ParseDuration(c.Server.TriggerBalancerInterval) if err != nil { return nil, errors.WithMessage(err, fmt.Sprintf("failed to parse server.triggerBalancerInterval: '%s'", c.Server.TriggerBalancerInterval)) } } if c.Server.LeaveTimeout != "" { leaveTimeout, err = time.ParseDuration(c.Server.LeaveTimeout) if err != nil { return nil, errors.WithMessage(err, fmt.Sprintf("failed to parse server.leaveTimeout: '%s'", c.Server.LeaveTimeout)) } } clientConfig := Client{ Authentication: &Authentication{ Password: c.Authentication.Password, }, } err = mapYamlToConfig(&clientConfig, &c.Client) if err != nil { return nil, err } dmapConfig, err := loadDMapConfig(c) if err != nil { return nil, err } cfg := &Config{ BindAddr: c.Server.BindAddr, BindPort: c.Server.BindPort, Interface: c.Server.Interface, ServiceDiscovery: c.ServiceDiscovery, MemberlistInterface: c.Memberlist.Interface, MemberlistConfig: memberlistConfig, Client: &clientConfig, LogLevel: c.Logging.Level, JoinRetryInterval: joinRetryInterval, RoutingTablePushInterval: routingTablePushInterval, TriggerBalancerInterval: triggerBalancerInterval, EnableClusterEventsChannel: c.Server.EnableClusterEventsChannel, MaxJoinAttempts: c.Memberlist.MaxJoinAttempts, Peers: c.Memberlist.Peers, PartitionCount: c.Server.PartitionCount, ReplicaCount: c.Server.ReplicaCount, WriteQuorum: c.Server.WriteQuorum, ReadQuorum: c.Server.ReadQuorum, ReplicationMode: c.Server.ReplicationMode, ReadRepair: c.Server.ReadRepair, LoadFactor: c.Server.LoadFactor, MemberCountQuorum: c.Server.MemberCountQuorum, Logger: log.New(logOutput, "", log.LstdFlags), LogOutput: logOutput, LogVerbosity: c.Logging.Verbosity, Hasher: hasher.NewDefaultHasher(), KeepAlivePeriod: keepAlivePeriod, IdleClose: idleClose, BootstrapTimeout: bootstrapTimeout, LeaveTimeout: leaveTimeout, DMaps: dmapConfig, Authentication: &Authentication{ Password: c.Authentication.Password, }, } if err := cfg.Sanitize(); err != nil { return nil, err } if err := cfg.Validate(); err != nil { return nil, err } return cfg, nil } ================================================ FILE: config/memberlist.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package config import ( "fmt" "net" "strings" "github.com/hashicorp/go-multierror" "github.com/hashicorp/memberlist" ) func (c *Config) validateMemberlistConfig() error { var result error if c.MemberlistConfig.AdvertiseAddr != "" { if ip := net.ParseIP(c.MemberlistConfig.AdvertiseAddr); ip == nil { result = multierror.Append(result, fmt.Errorf("memberlist: AdvertiseAddr has to be a valid IPv4 or IPv6 address")) } } if c.MemberlistConfig.BindAddr == "" { result = multierror.Append(result, fmt.Errorf("memberlist: BindAddr cannot be an empty string")) } return result } // NewMemberlistConfig returns a new memberlist.Config for a given environment. // // It takes an env parameter: local, lan and wan. // // local: // DefaultLocalConfig works like DefaultConfig, however it returns a configuration that // is optimized for a local loopback environments. The default configuration is still very conservative // and errs on the side of caution. // // lan: // DefaultLANConfig returns a sane set of configurations for Memberlist. It uses the hostname // as the node name, and otherwise sets very conservative values that are sane for most LAN environments. // The default configuration errs on the side of caution, choosing values that are optimized for higher convergence // at the cost of higher bandwidth usage. Regardless, these values are a good starting point when getting started with // memberlist. // // wan: // DefaultWANConfig works like DefaultConfig, however it returns a configuration that is optimized for most WAN // environments. The default configuration is still very conservative and errs on the side of caution. func NewMemberlistConfig(env string) (*memberlist.Config, error) { e := strings.ToLower(env) switch e { case "local": return memberlist.DefaultLocalConfig(), nil case "lan": return memberlist.DefaultLANConfig(), nil case "wan": return memberlist.DefaultWANConfig(), nil } return nil, fmt.Errorf("unknown env: %s", env) } ================================================ FILE: config/network.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package config import ( "fmt" "net" "runtime" "strconv" "github.com/hashicorp/go-sockaddr" ) // The following functions are mostly extracted from Serf. See setupAgent function in cmd/serf/command/agent/command.go // Thanks for the extraordinary software. // // Source: https://github.com/hashicorp/serf/blob/master/cmd/serf/command/agent/command.go#L204 func addrParts(address string) (string, int, error) { // Get the address addr, err := net.ResolveTCPAddr("tcp", address) if err != nil { return "", 0, err } return addr.IP.String(), addr.Port, nil } func getBindIPFromNetworkInterface(addrs []net.Addr) (string, error) { for _, a := range addrs { var addrIP net.IP if runtime.GOOS == "windows" { // Waiting for https://github.com/golang/go/issues/5395 to use IPNet only addr, ok := a.(*net.IPAddr) if !ok { continue } addrIP = addr.IP } else { addr, ok := a.(*net.IPNet) if !ok { continue } addrIP = addr.IP } // Skip self-assigned IPs if addrIP.IsLinkLocalUnicast() { continue } return addrIP.String(), nil } return "", fmt.Errorf("failed to find usable address for interface") } func getBindIP(ifname, address string) (string, error) { bindIP, _, err := addrParts(address) if err != nil { return "", fmt.Errorf("invalid BindAddr: %w", err) } // Check if we have an interface if iface, _ := net.InterfaceByName(ifname); iface != nil { addrs, err := iface.Addrs() if err != nil { return "", fmt.Errorf("failed to get interface addresses: %w", err) } if len(addrs) == 0 { return "", fmt.Errorf("interface '%s' has no addresses", ifname) } // If there is no bind IP, pick an address if bindIP == "0.0.0.0" { addr, err := getBindIPFromNetworkInterface(addrs) if err != nil { return "", fmt.Errorf("ip scan on %s: %w", ifname, err) } return addr, nil } // If there is a bind IP, ensure it is available for _, a := range addrs { addr, ok := a.(*net.IPNet) if !ok { continue } if addr.IP.String() == bindIP { return bindIP, nil } } return "", fmt.Errorf("interface '%s' has no '%s' address", ifname, bindIP) } if bindIP == "0.0.0.0" { // if we're not bound to a specific IP, let's use a suitable private IP address. ipStr, err := sockaddr.GetPrivateIP() if err != nil { return "", fmt.Errorf("failed to get private interface addresses: %w", err) } // if we could not find a private address, we need to expand our search to a public // ip address if ipStr == "" { ipStr, err = sockaddr.GetPublicIP() if err != nil { return "", fmt.Errorf("failed to get public interface addresses: %w", err) } } if ipStr == "" { return "", fmt.Errorf("no private IP address found, and explicit IP not provided") } parsed := net.ParseIP(ipStr) if parsed == nil { return "", fmt.Errorf("failed to parse private IP address: %q", ipStr) } bindIP = parsed.String() } return bindIP, nil } // SetupNetworkConfig tries to find an appropriate bindIP to bind and propagate. func (c *Config) SetupNetworkConfig() (err error) { address := net.JoinHostPort(c.BindAddr, strconv.Itoa(c.BindPort)) c.BindAddr, err = getBindIP(c.Interface, address) if err != nil { return err } address = net.JoinHostPort(c.MemberlistConfig.BindAddr, strconv.Itoa(c.MemberlistConfig.BindPort)) c.MemberlistConfig.BindAddr, err = getBindIP(c.MemberlistInterface, address) if err != nil { return err } if c.MemberlistConfig.AdvertiseAddr != "" { advertisePort := c.MemberlistConfig.AdvertisePort if advertisePort == 0 { advertisePort = c.MemberlistConfig.BindPort } address := net.JoinHostPort(c.MemberlistConfig.AdvertiseAddr, strconv.Itoa(advertisePort)) advertiseAddr, _, err := addrParts(address) if err != nil { return err } c.MemberlistConfig.AdvertiseAddr = advertiseAddr } return nil } ================================================ FILE: config/network_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package config import ( "testing" "github.com/stretchr/testify/require" ) func TestConfig_SetupNetworkConfig(t *testing.T) { c := &Config{} require.NoError(t, c.Sanitize()) require.NoError(t, c.Validate()) require.NoError(t, c.SetupNetworkConfig()) } func TestConfig_SetupNetworkConfig_Memberlist_AdvertiseAddr(t *testing.T) { c := &Config{} require.NoError(t, c.Sanitize()) require.NoError(t, c.Validate()) c.MemberlistConfig.AdvertiseAddr = "localhost" require.NoError(t, c.SetupNetworkConfig()) require.NotEqual(t, "localhost", c.MemberlistConfig.AdvertiseAddr) } ================================================ FILE: docker/README.md ================================================ # Multi-container environment with Docker Compose We provide a multi-container environment to test, develop and deploy Olric clusters. This environment includes nginx as TCP reverse proxy and Consul for service discovery. ## Usage In this folder, simply run: ``` docker-compose up olric ``` To create a multi-node cluster: ``` docker-compose up --scale olric=10 olric ``` Sample output: ``` docker-compose up olric Creating docker_nginx_1 ... done Creating docker_consul_1 ... done Creating docker_olric_1 ... done Creating docker_olric_2 ... done Attaching to docker_olric_1 olric_1 | 2020/08/12 15:53:18 [olric-server] pid: 1 has been started on 172.25.0.4:3320 olric_1 | 2020/08/12 15:53:18 [INFO] Service discovery plugin is enabled, provider: consul olric_1 | 2020/08/12 15:53:18 [DEBUG] memberlist: Stream connection from=172.25.0.3:56830 olric_1 | 2020/08/12 15:53:19 [ERROR] Join attempt returned error: no peers found => olric.go:2 ``` You can modify `olric-server-consul.yaml` file to try different configuration options. If Consul service works without any problem, you can visit [http://localhost:8500](http://localhost:8500) to monitor cluster health. ### Accessing to the cluster `nginx` service exposes port `3320` to access the cluster. You can list the cluster members with `CLUSTER.MEMBERS` command. ``` $ redis-cli -p 3320 127.0.0.1:3320> CLUSTER.MEMBERS 1) 1) "172.18.0.9:3320" 2) (integer) 1745597203895069302 3) "true" 2) 1) "172.18.0.10:3320" 2) (integer) 1745597204061500052 3) "false" 3) 1) "172.18.0.11:3320" 2) (integer) 1745597204182767469 3) "false" 4) 1) "172.18.0.3:3320" 2) (integer) 1745597204275319219 3) "false" 5) 1) "172.18.0.6:3320" 2) (integer) 1745597204337977552 3) "false" 6) 1) "172.18.0.4:3320" 2) (integer) 1745597204369791844 3) "false" 7) 1) "172.18.0.12:3320" 2) (integer) 1745597204385693552 3) "false" 8) 1) "172.18.0.7:3320" 2) (integer) 1745597204523284927 3) "false" 9) 1) "172.18.0.13:3320" 2) (integer) 1745597204665281636 3) "false" 10) 1) "172.18.0.8:3320" 2) (integer) 1745597208386416971 3) "false" ``` Let's taste the DMap: ``` $ redis-cli -p 3320 127.0.0.1:3320> DM.PUT test my-key my-value OK 127.0.0.1:3320> DM.GET test my-key "my-value" 127.0.0.1:3320> ``` ## Service discovery Olric provides a service discovery subsystem via a plugin interface. We currently have three service discovery plugins: * [olric-consul-plugin](https://github.com/olric-data/olric-consul-plugin): Consul-backed service discovery, * [olric-nats-plugin](https://github.com/justinfx/olric-nats-plugin): Nats-backed service discovery, * [olric-cloud-plugin](https://github.com/olric-data/olric-cloud-plugin): Service discovery plugin for cloud environments (AWS, GKE, Azure and Kubernetes) We use the Consul plugin in this document: ### Consul Consul is easy to use and a proven way to discover nodes in a clustered environment. Olric discover the other nodes via [olric-consul-plugin](https://github.com/olric-data/olric-consul-plugin). Here is a simple payload for this setup: ```json { "Name": "olric-cluster", "Tags": [ "primary", "v1" ], "Port": 3322, "EnableTagOverride": false, "Check": { "Name": "Olric node on 3322", "Interval": "1s", "Timeout": "10s" } } ``` `3322` is used by [hashicorp/memberlist](https://github.com/hashicorp/memberlist) to maintain an eventually consistent view of the cluster. Consul dials this port to control the node. `Address`, `ID` and `Check.TCP` fields is being filled by the plugin. You can still give your own configuration values, if you know what you are doing. Please check out `olric-server-consul.yaml` to see how to create an Olric cluster with Consul. ================================================ FILE: docker/docker-compose.yml ================================================ services: nginx: image: nginx:latest restart: on-failure volumes: - ${PWD}/nginx.conf:/etc/nginx/nginx.conf:ro ports: - '3320:3320' consul: platform: linux/amd64 image: public.ecr.aws/bitnami/consul:latest volumes: - consul_data:/bitnami/consul ports: - '8300:8300' - '8301:8301' - '8301:8301/udp' - '8500:8500' - '8600:8600' - '8600:8600/udp' olric: platform: linux/amd64 image: ghcr.io/olric-data/olric-consul-plugin:latest restart: on-failure volumes: - ${PWD}/olric-server-consul.yaml:/etc/olric-server.yaml:ro depends_on: - nginx - consul volumes: consul_data: driver: local ================================================ FILE: docker/nginx.conf ================================================ user nginx; events { worker_connections 1000; } stream { server { listen 3320; proxy_pass olric:3320; } } ================================================ FILE: docker/olric-server-consul.yaml ================================================ server: # BindAddr denotes the address that Olric will bind to for communication # with other Olric nodes. bindAddr: 0.0.0.0 # BindPort denotes the address that Olric will bind to for communication # with other Olric nodes. bindPort: 3320 # KeepAlivePeriod denotes whether the operating system should send # keep-alive messages on the connection. keepAlivePeriod: 300s # IdleClose will automatically close idle connections after the specified duration. # Use zero to disable this feature. # idleClose: 300s # Timeout for bootstrap control # # An Olric node checks operation status before taking any action for the # cluster events, responding incoming requests and running API functions. # Bootstrapping status is one of the most important checkpoints for an # "operable" Olric node. BootstrapTimeout sets a deadline to check # bootstrapping status without blocking indefinitely. bootstrapTimeout: 5s # PartitionCount is 271, by default. partitionCount: 271 # ReplicaCount is 1, by default. replicaCount: 1 # Minimum number of successful writes to return a response for a write request. writeQuorum: 1 # Minimum number of successful reads to return a response for a read request. readQuorum: 1 # Switch to control read-repair algorithm which helps to reduce entropy. readRepair: false # Default value is SyncReplicationMode. replicationMode: 0 # sync mode. for async, set 1 # Minimum number of members to form a cluster and run any query on the cluster. memberCountQuorum: 1 # Coordinator member pushes the routing table to cluster members in the case of # node join or left events. It also pushes the table periodically. routingTablePushInterval # is the interval between subsequent calls. Default is 1 minute. routingTablePushInterval: 1m # Olric can send push cluster events to cluster.events channel. Available cluster events: # # * node-join-event # * node-left-event # * fragment-migration-event # * fragment-received-event # # If you want to receive these events, set true to EnableClusterEventsChannel and subscribe to # cluster.events channel. Default is false. enableClusterEventsChannel: true client: # Timeout for TCP dial. # # The timeout includes name resolution, if required. When using TCP, and the host in the address parameter # resolves to multiple IP addresses, the timeout is spread over each consecutive dial, such that each is # given an appropriate fraction of the time to connect. dialTimeout: 5s # Timeout for socket reads. If reached, commands will fail # with a timeout instead of blocking. Use value -1 for no timeout and 0 for default. # Default is DefaultReadTimeout readTimeout: 3s # Timeout for socket writes. If reached, commands will fail # with a timeout instead of blocking. # Default is DefaultWriteTimeout writeTimeout: 3s # Maximum number of retries before giving up. # Default is 3 retries; -1 (not 0) disables retries. #maxRetries: 3 # Minimum backoff between each retry. # Default is 8 milliseconds; -1 disables backoff. #minRetryBackoff: 8ms # Maximum backoff between each retry. # Default is 512 milliseconds; -1 disables backoff. #maxRetryBackoff: 512ms # Type of connection pool. # true for FIFO pool, false for LIFO pool. # Note that fifo has higher overhead compared to lifo. #poolFIFO: false # Maximum number of socket connections. # Default is 10 connections per every available CPU as reported by runtime.GOMAXPROCS. #poolSize: 0 # Minimum number of idle connections which is useful when establishing # new connection is slow. #minIdleConns: # Connection age at which client retires (closes) the connection. # Default is to not close aged connections. #maxConnAge: # Amount of time client waits for connection if all connections are busy before # returning an error. Default is ReadTimeout + 1 second. #poolTimeout: 3s # Amount of time after which client closes idle connections. # Should be less than server's timeout. # Default is 5 minutes. -1 disables idle timeout check. idleTimeout: 5m # Frequency of idle checks made by idle connections reaper. # Default is 1 minute. -1 disables idle connections reaper, # but idle connections are still discarded by the client # if IdleTimeout is set. idleCheckFrequency: 1m logging: # DefaultLogVerbosity denotes default log verbosity level. # # * 1 - Generally useful for this to ALWAYS be visible to an operator # * Programmer errors # * Logging extra info about a panic # * CLI argument handling # * 2 - A reasonable default log level if you don't want verbosity. # * Information about config (listening on X, watching Y) # * Errors that repeat frequently that relate to conditions that can be # corrected # * 3 - Useful steady state information about the service and # important log messages that may correlate to # significant changes in the system. This is the recommended default log # level for most systems. # * Logging HTTP requests and their exit code # * System state changing # * Controller state change events # * Scheduler log messages # * 4 - Extended information about changes # * More info about system state changes # * 5 - Debug level verbosity # * Logging in particularly thorny parts of code where you may want to come # back later and check it # * 6 - Trace level verbosity # * Context to understand the steps leading up to neterrors and warnings # * More information for troubleshooting reported issues verbosity: 3 # Default LogLevel is DEBUG. Available levels: "DEBUG", "WARN", "ERROR", "INFO" level: WARN output: stderr memberlist: environment: lan # Configuration related to what address to bind to and ports to # listen on. The port is used for both UDP and TCP gossip. It is # assumed other nodes are running on this port, but they do not need # to. bindAddr: 0.0.0.0 bindPort: 3322 # EnableCompression is used to control message compression. This can # be used to reduce bandwidth usage at the cost of slightly more CPU # utilization. This is only available starting at protocol version 1. enableCompression: false # JoinRetryInterval is the time gap between attempts to join an existing # cluster. joinRetryInterval: 1ms # MaxJoinAttempts denotes the maximum number of attemps to join an existing # cluster before forming a new one. maxJoinAttempts: 1 # See service discovery plugins #peers: # - "localhost:3325" #advertiseAddr: "" #advertisePort: 3322 #suspicionMaxTimeoutMult: 6 #disableTCPPings: false #awarenessMaxMultiplier: 8 #gossipNodes: 3 #gossipVerifyIncoming: true #gossipVerifyOutgoing: true #dnsConfigPath: "/etc/resolv.conf" #handoffQueueDepth: 1024 #udpBufferSize: 1400 dmaps: engine: name: ramblock config: tableSize: 524288 # bytes # checkEmptyFragmentsInterval: 1m # triggerCompactionInterval: 10m # numEvictionWorkers: 1 # maxIdleDuration: "" # ttlDuration: "100s" # maxKeys: 100000 # maxInuse: 1000000 # lRUSamples: 10 # evictionPolicy: "LRU" # custom: # foobar: # maxIdleDuration: "60s" # ttlDuration: "300s" # maxKeys: 500000 # lRUSamples: 20 # evictionPolicy: "NONE" serviceDiscovery: # path is a required property and used by Olric. It has to be a full path. path: "/usr/lib/olric-consul-plugin.so" # provider is just informal, provider: "consul" # Plugin specific configuration # Consul server, used by the plugin. It's required address: "http://consul:8500" # Specifies that the server should return only nodes with all checks in the passing state. passingOnly: true # Missing health checks from the request will be deleted from the agent. Using this parameter # allows to idempotently register a service and its checks without having to manually deregister # checks. replaceExistingChecks: true # InsecureSkipVerify controls whether a client verifies the # server's certificate chain and host name. # If InsecureSkipVerify is true, TLS accepts any certificate # presented by the server and any host name in that certificate. # In this mode, TLS is susceptible to man-in-the-middle attacks. # This should be used only for testing. insecureSkipVerify: true # service record payload: ' { "Name": "olric-cluster", "Tags": [ "primary", "v1" ], "Port": 3322, "EnableTagOverride": false, "Check": { "Name": "Olric node on 3322", "Interval": "1s", "Timeout": "10s" } } ' ================================================ FILE: embedded_client.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package olric import ( "context" "encoding/json" "sync" "time" "github.com/olric-data/olric/internal/discovery" "github.com/olric-data/olric/internal/dmap" "github.com/olric-data/olric/internal/protocol" "github.com/olric-data/olric/internal/util" "github.com/olric-data/olric/stats" ) // EmbeddedLockContext is returned by Lock and LockWithTimeout methods. // It should be stored in a proper way to release the lock. type EmbeddedLockContext struct { key string token []byte dm *EmbeddedDMap } // Unlock releases the lock. func (l *EmbeddedLockContext) Unlock(ctx context.Context) error { err := l.dm.dm.Unlock(ctx, l.key, l.token) return convertDMapError(err) } // Lease takes the duration to update the expiry for the given Lock. func (l *EmbeddedLockContext) Lease(ctx context.Context, duration time.Duration) error { err := l.dm.dm.Lease(ctx, l.key, l.token, duration) return convertDMapError(err) } // EmbeddedClient is an Olric client implementation for embedded-member scenario. type EmbeddedClient struct { db *Olric } // EmbeddedDMap is an DMap client implementation for embedded-member scenario. type EmbeddedDMap struct { mtx sync.RWMutex clusterClient *ClusterClient config *dmapConfig member discovery.Member dm *dmap.DMap client *EmbeddedClient name string } func (dm *EmbeddedDMap) setOrGetClusterClient() (Client, error) { // Acquire the read lock and try to access the cluster client, if any. dm.mtx.RLock() if dm.clusterClient != nil { dm.mtx.RUnlock() return dm.clusterClient, nil } dm.mtx.RUnlock() // The cluster client is unset, try to create a new one. dm.mtx.Lock() defer dm.mtx.Unlock() // Check the existing value last time. There can be another running instances // of this function. if dm.clusterClient != nil { return dm.clusterClient, nil } // Create a new cluster client here. c, err := NewClusterClient([]string{dm.client.db.rt.This().String()}) if err != nil { return nil, err } dm.clusterClient = c return dm.clusterClient, nil } // Pipeline is a mechanism to realise Redis Pipeline technique. // // Pipelining is a technique to extremely speed up processing by packing // operations to batches, send them at once to Redis and read a replies in a // singe step. // See https://redis.io/topics/pipelining // // Pay attention, that Pipeline is not a transaction, so you can get unexpected // results in case of big pipelines and small read/write timeouts. // Redis client has retransmission logic in case of timeouts, pipeline // can be retransmitted and commands can be executed more than once. func (dm *EmbeddedDMap) Pipeline(opts ...PipelineOption) (*DMapPipeline, error) { cc, err := dm.setOrGetClusterClient() if err != nil { return nil, err } clusterDMap, err := cc.NewDMap(dm.name) if err != nil { return nil, err } return clusterDMap.Pipeline(opts...) } // RefreshMetadata fetches a list of available members and the latest routing // table version. It also closes stale clients, if there are any. EmbeddedClient has // this method to implement the Client interface. It doesn't need to refresh metadata manually. func (e *EmbeddedClient) RefreshMetadata(_ context.Context) error { // EmbeddedClient already has the latest metadata. return nil } // Scan returns an iterator to loop over the keys. // // Available scan options: // // * Count // * Match func (dm *EmbeddedDMap) Scan(ctx context.Context, options ...ScanOption) (Iterator, error) { cc, err := dm.setOrGetClusterClient() if err != nil { return nil, err } cdm, err := cc.NewDMap(dm.name) if err != nil { return nil, err } i, err := cdm.Scan(ctx, options...) if err != nil { return nil, err } e := &EmbeddedIterator{ client: dm.client, dm: dm.dm, } clusterIterator := i.(*ClusterIterator) clusterIterator.scanner = e.scanOnOwners e.clusterIterator = clusterIterator return e, nil } // Lock sets a lock for the given key. Acquired lock is only for the key in // this dmap. // // It returns immediately if it acquires the lock for the given key. Otherwise, // it waits until deadline. // // You should know that the locks are approximate, and only to be used for // non-critical purposes. func (dm *EmbeddedDMap) Lock(ctx context.Context, key string, deadline time.Duration) (LockContext, error) { token, err := dm.dm.Lock(ctx, key, 0*time.Second, deadline) if err != nil { return nil, convertDMapError(err) } return &EmbeddedLockContext{ key: key, token: token, dm: dm, }, nil } // LockWithTimeout sets a lock for the given key. If the lock is still unreleased // the end of given period of time, // it automatically releases the lock. Acquired lock is only for the key in // this dmap. // // It returns immediately if it acquires the lock for the given key. Otherwise, // it waits until deadline. // // You should know that the locks are approximate, and only to be used for // non-critical purposes. func (dm *EmbeddedDMap) LockWithTimeout(ctx context.Context, key string, timeout, deadline time.Duration) (LockContext, error) { token, err := dm.dm.Lock(ctx, key, timeout, deadline) if err != nil { return nil, convertDMapError(err) } return &EmbeddedLockContext{ key: key, token: token, dm: dm, }, nil } // Destroy flushes the given DMap on the cluster. You should know that there // is no global lock on DMaps. So if you call Put/PutEx and Destroy methods // concurrently on the cluster, Put call may set new values to the DMap. func (dm *EmbeddedDMap) Destroy(ctx context.Context) error { return dm.dm.Destroy(ctx) } // Expire updates the expiry for the given key. It returns ErrKeyNotFound if // the DB does not contain the key. It's thread-safe. func (dm *EmbeddedDMap) Expire(ctx context.Context, key string, timeout time.Duration) error { return dm.dm.Expire(ctx, key, timeout) } // Name exposes name of the DMap. func (dm *EmbeddedDMap) Name() string { return dm.name } // GetPut atomically sets the key to value and returns the old value stored at key. It returns nil if there is no // previous value. func (dm *EmbeddedDMap) GetPut(ctx context.Context, key string, value interface{}) (*GetResponse, error) { e, err := dm.dm.GetPut(ctx, key, value) if err != nil { return nil, err } return &GetResponse{ entry: e, }, nil } // Decr atomically decrements the key by delta. The return value is the new value // after being decremented or an error. func (dm *EmbeddedDMap) Decr(ctx context.Context, key string, delta int) (int, error) { return dm.dm.Decr(ctx, key, delta) } // Incr atomically increments the key by delta. The return value is the new value // after being incremented or an error. func (dm *EmbeddedDMap) Incr(ctx context.Context, key string, delta int) (int, error) { return dm.dm.Incr(ctx, key, delta) } // IncrByFloat atomically increments the key by delta. The return value is the new value after being incremented or an error. func (dm *EmbeddedDMap) IncrByFloat(ctx context.Context, key string, delta float64) (float64, error) { return dm.dm.IncrByFloat(ctx, key, delta) } // Delete deletes values for the given keys. Delete will not return error // if key doesn't exist. It's thread-safe. It is safe to modify the contents // of the argument after Delete returns. func (dm *EmbeddedDMap) Delete(ctx context.Context, keys ...string) (int, error) { return dm.dm.Delete(ctx, keys...) } // Get gets the value for the given key. It returns ErrKeyNotFound if the DB // does not contain the key. It's thread-safe. It is safe to modify the contents // of the returned value. See GetResponse for the details. func (dm *EmbeddedDMap) Get(ctx context.Context, key string) (*GetResponse, error) { result, err := dm.dm.Get(ctx, key) if err != nil { return nil, convertDMapError(err) } return &GetResponse{ entry: result, }, nil } // Put sets the value for the given key. It overwrites any previous value for // that key, and it's thread-safe. The key has to be a string. value type is arbitrary. // It is safe to modify the contents of the arguments after Put returns but not before. func (dm *EmbeddedDMap) Put(ctx context.Context, key string, value interface{}, options ...PutOption) error { var pc dmap.PutConfig for _, opt := range options { opt(&pc) } err := dm.dm.Put(ctx, key, value, &pc) if err != nil { return convertDMapError(err) } return nil } // Close stops background routines and frees allocated resources. func (dm *EmbeddedDMap) Close(ctx context.Context) error { dm.mtx.RLock() clusterClient := dm.clusterClient dm.mtx.RUnlock() if clusterClient != nil { return dm.clusterClient.Close(ctx) } return nil } func (e *EmbeddedClient) NewDMap(name string, options ...DMapOption) (DMap, error) { dm, err := e.db.dmap.NewDMap(name) if err != nil { return nil, convertDMapError(err) } var dc dmapConfig for _, opt := range options { opt(&dc) } return &EmbeddedDMap{ config: &dc, dm: dm, name: name, client: e, member: e.db.rt.This(), }, nil } // Stats exposes some useful metrics to monitor an Olric node. func (e *EmbeddedClient) Stats(ctx context.Context, address string, options ...StatsOption) (stats.Stats, error) { if err := e.db.isOperable(); err != nil { // this node is not bootstrapped yet. return stats.Stats{}, err } var cfg statsConfig for _, opt := range options { opt(&cfg) } if address == e.db.rt.This().String() { return e.db.stats(cfg), nil } statsCmd := protocol.NewStats() if cfg.CollectRuntime { statsCmd.SetCollectRuntime() } cmd := statsCmd.Command(ctx) rc := e.db.client.Get(address) err := rc.Process(ctx, cmd) if err != nil { return stats.Stats{}, processProtocolError(err) } if err = cmd.Err(); err != nil { return stats.Stats{}, processProtocolError(err) } data, err := cmd.Bytes() if err != nil { return stats.Stats{}, processProtocolError(err) } var s stats.Stats err = json.Unmarshal(data, &s) if err != nil { return stats.Stats{}, processProtocolError(err) } return s, nil } // Close stops background routines and frees allocated resources. func (e *EmbeddedClient) Close(_ context.Context) error { return nil } // Ping sends a ping message to an Olric node. Returns PONG if message is empty, // otherwise return a copy of the message as a bulk. This command is often used to test // if a connection is still alive, or to measure latency. func (e *EmbeddedClient) Ping(ctx context.Context, addr, message string) (string, error) { response, err := e.db.ping(ctx, addr, message) if err != nil { return "", err } return util.BytesToString(response), nil } // RoutingTable returns the latest version of the routing table. func (e *EmbeddedClient) RoutingTable(ctx context.Context) (RoutingTable, error) { return e.db.routingTable(ctx) } // Members returns a thread-safe list of cluster members. func (e *EmbeddedClient) Members(_ context.Context) ([]Member, error) { members := e.db.rt.Discovery().GetMembers() coordinator := e.db.rt.Discovery().GetCoordinator() var result []Member for _, member := range members { m := Member{ Name: member.Name, ID: member.ID, Birthdate: member.Birthdate, } if coordinator.ID == member.ID { m.Coordinator = true } result = append(result, m) } return result, nil } // NewPubSub returns a new PubSub client with the given options. func (e *EmbeddedClient) NewPubSub(options ...PubSubOption) (*PubSub, error) { return newPubSub(e.db.client, options...) } // NewEmbeddedClient creates and returns a new EmbeddedClient instance. func (db *Olric) NewEmbeddedClient() *EmbeddedClient { return &EmbeddedClient{db: db} } var ( _ Client = (*EmbeddedClient)(nil) _ DMap = (*EmbeddedDMap)(nil) ) ================================================ FILE: embedded_client_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package olric import ( "context" "fmt" "runtime" "testing" "time" "github.com/olric-data/olric/internal/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" ) func TestEmbeddedClient_NewDMap(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) e := db.NewEmbeddedClient() _, err := e.NewDMap("mydmap") require.NoError(t, err) } func TestEmbeddedClient_DMap_Put(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) e := db.NewEmbeddedClient() dm, err := e.NewDMap("mydmap") require.NoError(t, err) err = dm.Put(context.Background(), "mykey", "myvalue") require.NoError(t, err) } func TestEmbeddedClient_DMap_Put_EX(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() e := db.NewEmbeddedClient() dm, err := e.NewDMap("mydmap") require.NoError(t, err) err = dm.Put(ctx, "mykey", "myvalue", EX(time.Second)) require.NoError(t, err) <-time.After(time.Second) _, err = dm.Get(ctx, "mykey") require.ErrorIs(t, err, ErrKeyNotFound) } func TestEmbeddedClient_DMap_Put_PX(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() e := db.NewEmbeddedClient() dm, err := e.NewDMap("mydmap") require.NoError(t, err) err = dm.Put(ctx, "mykey", "myvalue", PX(time.Millisecond)) require.NoError(t, err) <-time.After(time.Millisecond) _, err = dm.Get(ctx, "mykey") require.ErrorIs(t, err, ErrKeyNotFound) } func TestEmbeddedClient_DMap_Put_EXAT(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() e := db.NewEmbeddedClient() dm, err := e.NewDMap("mydmap") require.NoError(t, err) err = dm.Put(ctx, "mykey", "myvalue", EXAT(time.Duration(time.Now().Add(time.Second).UnixNano()))) require.NoError(t, err) <-time.After(time.Second) _, err = dm.Get(ctx, "mykey") require.ErrorIs(t, err, ErrKeyNotFound) } func TestEmbeddedClient_DMap_Put_PXAT(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() e := db.NewEmbeddedClient() dm, err := e.NewDMap("mydmap") require.NoError(t, err) err = dm.Put(ctx, "mykey", "myvalue", PXAT(time.Duration(time.Now().Add(time.Millisecond).UnixNano()))) require.NoError(t, err) <-time.After(time.Millisecond) _, err = dm.Get(ctx, "mykey") require.ErrorIs(t, err, ErrKeyNotFound) } func TestEmbeddedClient_DMap_Put_NX(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() e := db.NewEmbeddedClient() dm, err := e.NewDMap("mydmap") require.NoError(t, err) err = dm.Put(ctx, "mykey", "myvalue", NX()) require.NoError(t, err) <-time.After(time.Millisecond) _, err = dm.Get(ctx, "mykey") require.NoError(t, err) } func TestEmbeddedClient_DMap_Put_XX(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() e := db.NewEmbeddedClient() dm, err := e.NewDMap("mydmap") require.NoError(t, err) err = dm.Put(ctx, "mykey", "myvalue", XX()) require.ErrorIs(t, err, ErrKeyNotFound) } func TestEmbeddedClient_DMap_Get(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) e := db.NewEmbeddedClient() dm, err := e.NewDMap("mydmap") require.NoError(t, err) err = dm.Put(context.Background(), "mykey", "myvalue") require.NoError(t, err) gr, err := dm.Get(context.Background(), "mykey") require.NoError(t, err) value, err := gr.String() require.NoError(t, err) require.Equal(t, "myvalue", value) } func TestEmbeddedClient_DMap_Delete(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) e := db.NewEmbeddedClient() dm, err := e.NewDMap("mydmap") require.NoError(t, err) err = dm.Put(context.Background(), "mykey", "myvalue") require.NoError(t, err) count, err := dm.Delete(context.Background(), "mykey") require.NoError(t, err) require.Equal(t, 1, count) _, err = dm.Get(context.Background(), "mykey") require.ErrorIs(t, err, ErrKeyNotFound) } func TestEmbeddedClient_DMap_Delete_Many_Keys(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) e := db.NewEmbeddedClient() dm, err := e.NewDMap("mydmap") require.NoError(t, err) var keys []string for i := 0; i < 10; i++ { key := testutil.ToKey(i) err = dm.Put(context.Background(), key, "myvalue") require.NoError(t, err) keys = append(keys, key) } count, err := dm.Delete(context.Background(), keys...) require.NoError(t, err) require.Equal(t, 10, count) } func TestEmbeddedClient_DMap_Atomic_Incr(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) e := db.NewEmbeddedClient() dm, err := e.NewDMap("mydmap") require.NoError(t, err) ctx := context.Background() var errGr errgroup.Group for i := 0; i < 100; i++ { errGr.Go(func() error { _, err = dm.Incr(ctx, "mykey", 1) return err }) } require.NoError(t, errGr.Wait()) gr, err := dm.Get(context.Background(), "mykey") res, err := gr.Int() require.NoError(t, err) require.Equal(t, 100, res) } func TestEmbeddedClient_DMap_Atomic_Decr(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) e := db.NewEmbeddedClient() dm, err := e.NewDMap("mydmap") require.NoError(t, err) ctx := context.Background() err = dm.Put(ctx, "mykey", 100) require.NoError(t, err) var errGr errgroup.Group for i := 0; i < 100; i++ { errGr.Go(func() error { _, err = dm.Decr(ctx, "mykey", 1) return err }) } require.NoError(t, errGr.Wait()) gr, err := dm.Get(context.Background(), "mykey") res, err := gr.Int() require.NoError(t, err) require.Equal(t, 0, res) } func TestEmbeddedClient_DMap_GetPut(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) e := db.NewEmbeddedClient() dm, err := e.NewDMap("mydmap") require.NoError(t, err) gr, err := dm.GetPut(context.Background(), "mykey", "myvalue") require.NoError(t, err) _, err = gr.String() require.ErrorIs(t, err, ErrNilResponse) gr, err = dm.GetPut(context.Background(), "mykey", "myvalue-2") require.NoError(t, err) value, err := gr.String() require.NoError(t, err) require.Equal(t, "myvalue", value) } func TestEmbeddedClient_DMap_Atomic_IncrByFloat(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) e := db.NewEmbeddedClient() dm, err := e.NewDMap("mydmap") require.NoError(t, err) ctx := context.Background() var errGr errgroup.Group for i := 0; i < 100; i++ { errGr.Go(func() error { _, err = dm.IncrByFloat(ctx, "mykey", 1.2) return err }) } require.NoError(t, errGr.Wait()) gr, err := dm.Get(context.Background(), "mykey") res, err := gr.Float64() require.NoError(t, err) require.Equal(t, 120.0000000000002, res) } func TestEmbeddedClient_DMap_Expire(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) e := db.NewEmbeddedClient() dm, err := e.NewDMap("mydmap") require.NoError(t, err) ctx := context.Background() err = dm.Put(ctx, "mykey", "myvalue") require.NoError(t, err) err = dm.Expire(ctx, "mykey", time.Millisecond) require.NoError(t, err) <-time.After(2 * time.Millisecond) _, err = dm.Get(context.Background(), "mykey") require.ErrorIs(t, err, ErrKeyNotFound) } func TestEmbeddedClient_DMap_Destroy(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) e := db.NewEmbeddedClient() dm, err := e.NewDMap("mydmap") require.NoError(t, err) ctx := context.Background() for i := 0; i < 100; i++ { err = dm.Put(ctx, fmt.Sprintf("mykey-%d", i), "myvalue") require.NoError(t, err) } err = dm.Destroy(ctx) require.NoError(t, err) // Destroy is an async command. Wait for some time to see its effect. <-time.After(100 * time.Millisecond) stats, err := e.Stats(ctx, e.db.rt.This().String()) require.NoError(t, err) var total int for _, part := range stats.Partitions { total += part.Length } require.Greater(t, 100, total) } func TestEmbeddedClient_DMap_Lock(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) e := db.NewEmbeddedClient() dm, err := e.NewDMap("mydmap") require.NoError(t, err) ctx := context.Background() key := "lock.key.test" lx, err := dm.Lock(ctx, key, time.Second) require.NoError(t, err) err = lx.Unlock(ctx) require.NoError(t, err) } func TestEmbeddedClient_DMap_Lock_ErrLockNotAcquired(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) e := db.NewEmbeddedClient() dm, err := e.NewDMap("mydmap") require.NoError(t, err) ctx := context.Background() key := "lock.key.test" _, err = dm.Lock(ctx, key, time.Second) require.NoError(t, err) _, err = dm.Lock(ctx, key, time.Millisecond) require.ErrorIs(t, err, ErrLockNotAcquired) } func TestEmbeddedClient_DMap_Lock_ErrNoSuchLock(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) e := db.NewEmbeddedClient() dm, err := e.NewDMap("mydmap") require.NoError(t, err) ctx := context.Background() key := "lock.key.test" lx, err := dm.Lock(ctx, key, time.Second) require.NoError(t, err) err = lx.Unlock(ctx) require.NoError(t, err) err = lx.Unlock(ctx) require.ErrorIs(t, err, ErrNoSuchLock) } func TestEmbeddedClient_DMap_LockWithTimeout(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) e := db.NewEmbeddedClient() dm, err := e.NewDMap("mydmap") require.NoError(t, err) ctx := context.Background() key := "lock.key.test" lx, err := dm.LockWithTimeout(ctx, key, 5*time.Second, time.Second) require.NoError(t, err) err = lx.Unlock(ctx) require.NoError(t, err) } func TestEmbeddedClient_DMap_LockWithTimeout_Timeout(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) e := db.NewEmbeddedClient() dm, err := e.NewDMap("mydmap") require.NoError(t, err) ctx := context.Background() key := "lock.key.test" lx, err := dm.LockWithTimeout(ctx, key, time.Millisecond, time.Second) require.NoError(t, err) <-time.After(2 * time.Millisecond) err = lx.Unlock(ctx) require.ErrorIs(t, err, ErrNoSuchLock) } func TestEmbeddedClient_DMap_LockWithTimeout_ErrLockNotAcquired(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) e := db.NewEmbeddedClient() dm, err := e.NewDMap("mydmap") require.NoError(t, err) ctx := context.Background() key := "lock.key.test" _, err = dm.LockWithTimeout(ctx, key, 10*time.Second, time.Second) require.NoError(t, err) _, err = dm.LockWithTimeout(ctx, key, 10*time.Second, time.Millisecond) require.ErrorIs(t, err, ErrLockNotAcquired) } func TestEmbeddedClient_DMap_LockWithTimeout_ErrNoSuchLock(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) e := db.NewEmbeddedClient() dm, err := e.NewDMap("mydmap") require.NoError(t, err) ctx := context.Background() key := "lock.key.test" lx, err := dm.LockWithTimeout(ctx, key, time.Second, time.Second) require.NoError(t, err) err = lx.Unlock(ctx) require.NoError(t, err) err = lx.Unlock(ctx) require.ErrorIs(t, err, ErrNoSuchLock) } func TestEmbeddedClient_DMap_LockWithTimeout_ErrNoSuchLock_Timeout(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) e := db.NewEmbeddedClient() dm, err := e.NewDMap("mydmap") require.NoError(t, err) ctx := context.Background() key := "lock.key.test" lx, err := dm.LockWithTimeout(ctx, key, time.Millisecond, time.Second) require.NoError(t, err) <-time.After(time.Millisecond) err = lx.Unlock(ctx) require.ErrorIs(t, err, ErrNoSuchLock) } func TestEmbeddedClient_DMap_LockWithTimeout_Then_Lease(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) e := db.NewEmbeddedClient() dm, err := e.NewDMap("mydmap") require.NoError(t, err) ctx := context.Background() key := "lock.key.test" lx, err := dm.LockWithTimeout(ctx, key, 50*time.Millisecond, time.Second) require.NoError(t, err) // Expand its timeout value err = lx.Lease(ctx, time.Hour) require.NoError(t, err) <-time.After(100 * time.Millisecond) _, err = dm.Lock(ctx, key, time.Millisecond) require.ErrorIs(t, err, ErrLockNotAcquired) } func TestEmbeddedClient_RoutingTable_Standalone(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) e := db.NewEmbeddedClient() rt, err := e.RoutingTable(context.Background()) require.NoError(t, err) require.Len(t, rt, int(db.config.PartitionCount)) for _, route := range rt { require.Len(t, route.PrimaryOwners, 1) require.Equal(t, db.rt.This().String(), route.PrimaryOwners[0]) require.Len(t, route.ReplicaOwners, 0) } } func TestEmbeddedClient_RoutingTable_Cluster(t *testing.T) { cluster := newTestOlricCluster(t) cluster.addMember(t) // Cluster coordinator <-time.After(250 * time.Millisecond) cluster.addMember(t) db2 := cluster.addMember(t) e := db2.NewEmbeddedClient() rt, err := e.RoutingTable(context.Background()) require.NoError(t, err) require.Len(t, rt, int(db2.config.PartitionCount)) owners := make(map[string]struct{}) for _, route := range rt { for _, owner := range route.PrimaryOwners { owners[owner] = struct{}{} } } require.Len(t, owners, 3) } func TestEmbeddedClient_Member(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) cluster.addMember(t) e := db.NewEmbeddedClient() members, err := e.Members(context.Background()) require.NoError(t, err) require.Len(t, members, 2) coordinator := db.rt.Discovery().GetCoordinator() for _, member := range members { require.NotEqual(t, "", member.Name) require.NotEqual(t, 0, member.ID) require.NotEqual(t, 0, member.Birthdate) if coordinator.ID == member.ID { require.True(t, member.Coordinator) } else { require.False(t, member.Coordinator) } } } func TestEmbeddedClient_Ping(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) e := db.NewEmbeddedClient() ctx := context.Background() response, err := e.Ping(ctx, db.rt.This().String(), "") require.NoError(t, err) require.Equal(t, DefaultPingResponse, response) } func TestEmbeddedClient_Ping_WithMessage(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) e := db.NewEmbeddedClient() ctx := context.Background() message := "Olric is the best" response, err := e.Ping(ctx, db.rt.This().String(), message) require.NoError(t, err) require.Equal(t, message, response) } func TestEmbeddedClient_DMap_Put_PX_With_NX(t *testing.T) { cluster := newTestOlricCluster(t) db0 := cluster.addMember(t) db1 := cluster.addMember(t) ctx := context.Background() e := db0.NewEmbeddedClient() dm0, err := e.NewDMap("mydmap") require.NoError(t, err) err = dm0.Put(ctx, "mykey", "myvalue", PX(time.Minute), NX()) require.NoError(t, err) <-time.After(time.Millisecond) e = db1.NewEmbeddedClient() dm1, err := e.NewDMap("mydmap") require.NoError(t, err) gr, err := dm1.Get(ctx, "mykey") require.NoError(t, err) assert.NotZero(t, gr.TTL()) } func TestEmbeddedClient_Issue263(t *testing.T) { initNumRoutines := runtime.NumGoroutine() cluster := newTestOlricCluster(t) db := cluster.addMember(t) e := db.NewEmbeddedClient() ctx, cancel := context.WithCancel(context.Background()) dm, err := e.NewDMap("mydmap") require.NoError(t, err) // Create N key-value pairs: const N = 100 for i := range N { key := fmt.Sprintf("key-%d", i) value := fmt.Sprintf("value-%d", i) err := dm.Put(ctx, key, value) require.NoError(t, err) } // Iterate M times over N keys: const M = 100 for range M { iter, err := dm.Scan(ctx) require.NoError(t, err) for iter.Next() { // Do nothing } iter.Close() } require.NoError(t, dm.Close(ctx)) require.NoError(t, e.Close(ctx)) require.NoError(t, db.Shutdown(ctx)) cancel() assert.Equal(t, initNumRoutines, runtime.NumGoroutine()) runtime.GC() time.Sleep(time.Second) s := runtime.MemStats{} runtime.ReadMemStats(&s) const ( KB = 1 << 10 MB = KB << 10 ) buf := make([]byte, MB) stackSize := runtime.Stack(buf, true) t.Logf("Non-freed objects: %d\n", s.Mallocs-s.Frees) t.Logf("Mem in use (KB): %d\n", s.HeapAlloc/KB) t.Logf("Go-routines remained: %d\n", runtime.NumGoroutine()) t.Logf("Stack traces:\n%s\n", buf[:stackSize]) } ================================================ FILE: embedded_iterator.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package olric import ( "sync" "github.com/olric-data/olric/internal/dmap" "github.com/olric-data/olric/internal/protocol" ) // EmbeddedIterator implements distributed query on DMaps. type EmbeddedIterator struct { mtx sync.Mutex client *EmbeddedClient dm *dmap.DMap clusterIterator *ClusterIterator } func (e *EmbeddedIterator) scanOnOwners() error { owners := e.clusterIterator.getOwners() for idx, owner := range owners { cursor := e.clusterIterator.loadCursor(owner) if e.client.db.rt.This().String() == owner { keys, newCursor, err := e.dm.Scan(e.clusterIterator.partID, cursor, e.clusterIterator.config) if err != nil { return err } e.clusterIterator.updateIterator(keys, newCursor, owner) if newCursor == 0 { e.clusterIterator.removeScannedOwner(idx) } continue } // Build a scan command here s := protocol.NewScan(e.clusterIterator.partID, e.clusterIterator.dm.Name(), cursor) if e.clusterIterator.config.HasCount { s.SetCount(e.clusterIterator.config.Count) } if e.clusterIterator.config.HasMatch { s.SetMatch(e.clusterIterator.config.Match) } if e.clusterIterator.config.Replica { s.SetReplica() } scanCmd := s.Command(e.clusterIterator.ctx) // Fetch a Redis client for the given owner. rc := e.clusterIterator.clusterClient.client.Get(owner) err := rc.Process(e.clusterIterator.ctx, scanCmd) if err != nil { return err } keys, newCursor, err := scanCmd.Result() if err != nil { return err } e.clusterIterator.updateIterator(keys, newCursor, owner) if newCursor == 0 { e.clusterIterator.removeScannedOwner(idx) } } return nil } // Next returns true if there is more key in the iterator implementation. // Otherwise, it returns false. func (e *EmbeddedIterator) Next() bool { return e.clusterIterator.Next() } // Key returns a key name from the distributed map. func (e *EmbeddedIterator) Key() string { return e.clusterIterator.Key() } // Close stops the iteration and releases allocated resources. func (e *EmbeddedIterator) Close() { e.clusterIterator.Close() } ================================================ FILE: embedded_iterator_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package olric import ( "context" "fmt" "testing" "github.com/olric-data/olric/internal/testutil" "github.com/stretchr/testify/require" ) func TestEmbeddedClient_ScanMatch(t *testing.T) { cl := newTestOlricCluster(t) db := cl.addMember(t) cl.addMember(t) e := db.NewEmbeddedClient() dm, err := e.NewDMap("mydmap") require.NoError(t, err) ctx := context.Background() evenKeys := make(map[string]bool) for i := 0; i < 100; i++ { var key string if i%2 == 0 { key = fmt.Sprintf("even:%s", testutil.ToKey(i)) evenKeys[key] = false } else { key = fmt.Sprintf("odd:%s", testutil.ToKey(i)) } err = dm.Put(ctx, key, i) require.NoError(t, err) } i, err := dm.Scan(ctx, Match("^even:")) require.NoError(t, err) var count int defer i.Close() for i.Next() { count++ require.Contains(t, evenKeys, i.Key()) } require.Equal(t, 50, count) } func TestEmbeddedClient_Scan(t *testing.T) { cl := newTestOlricCluster(t) db := cl.addMember(t) cl.addMember(t) e := db.NewEmbeddedClient() dm, err := e.NewDMap("mydmap") require.NoError(t, err) ctx := context.Background() allKeys := make(map[string]bool) for i := 0; i < 100; i++ { err = dm.Put(ctx, testutil.ToKey(i), i) require.NoError(t, err) allKeys[testutil.ToKey(i)] = false } i, err := dm.Scan(ctx) require.NoError(t, err) var count int defer i.Close() for i.Next() { count++ require.Contains(t, allKeys, i.Key()) } require.Equal(t, 100, count) } ================================================ FILE: events/cluster_events.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package events import ( "bytes" "encoding/json" "fmt" "reflect" "strconv" "strings" "github.com/olric-data/olric/internal/util" ) const ( ClusterEventsChannel = "cluster.events" KindNodeJoinEvent = "node-join-event" KindNodeLeftEvent = "node-left-event" KindFragmentMigrationEvent = "fragment-migration-event" KindFragmentReceivedEvent = "fragment-received-event" ) type Event interface { Encode() (string, error) } // encodeEvents encodes given interface to its JSON representation and preserves the order in fields slice. func encodeEvent(data interface{}, fields []string, valueExtractor func(r reflect.Value, field string) (interface{}, error)) (string, error) { buf := bytes.NewBuffer(nil) buf.WriteString("{") r := reflect.Indirect(reflect.ValueOf(data)) for i, field := range fields { sf, ok := r.Type().FieldByName(field) if !ok { return "", fmt.Errorf("field not found: %s", field) } tag := strings.Trim(string(sf.Tag), "json:") tag, err := strconv.Unquote(tag) if err != nil { return "", err } value, err := valueExtractor(r, field) if err != nil { return "", err } if i != 0 { buf.WriteString(",") } // marshal key key, err := json.Marshal(tag) if err != nil { return "", err } buf.Write(key) buf.WriteString(":") // marshal value val, err := json.Marshal(value) if err != nil { return "", err } buf.Write(val) } buf.WriteString("}") return util.BytesToString(buf.Bytes()), nil } type NodeJoinEvent struct { Kind string `json:"kind"` Source string `json:"source"` NodeJoin string `json:"node_join"` Timestamp int64 `json:"timestamp"` } func (n *NodeJoinEvent) Encode() (string, error) { fields := []string{"Timestamp", "Source", "Kind", "NodeJoin"} return encodeEvent(n, fields, func(r reflect.Value, field string) (interface{}, error) { var value interface{} switch field { case "Timestamp": value = r.FieldByName(field).Int() case "Source", "Kind", "NodeJoin": value = r.FieldByName(field).String() default: return nil, fmt.Errorf("invalid field: %s", field) } return value, nil }) } type NodeLeftEvent struct { Kind string `json:"kind"` Source string `json:"source"` NodeLeft string `json:"node_left"` Timestamp int64 `json:"timestamp"` } func (n *NodeLeftEvent) Encode() (string, error) { fields := []string{"Timestamp", "Source", "Kind", "NodeLeft"} return encodeEvent(n, fields, func(r reflect.Value, field string) (interface{}, error) { var value interface{} switch field { case "Timestamp": value = r.FieldByName(field).Int() case "Source", "Kind", "NodeLeft": value = r.FieldByName(field).String() default: return nil, fmt.Errorf("invalid field: %s", field) } return value, nil }) } type FragmentMigrationEvent struct { Kind string `json:"kind"` Source string `json:"source"` Target string `json:"target"` Identifier string `json:"identifier"` PartitionID uint64 `json:"partition_id"` DataStructure string `json:"data_structure"` Length int `json:"length"` IsBackup bool `json:"is_backup"` Timestamp int64 `json:"timestamp"` } func (f *FragmentMigrationEvent) Encode() (string, error) { fields := []string{ "Timestamp", "Source", "Kind", "Target", "DataStructure", "PartitionID", "Identifier", "IsBackup", "Length", } return encodeEvent(f, fields, func(r reflect.Value, field string) (interface{}, error) { var value interface{} switch field { case "IsBackup": value = r.FieldByName(field).Bool() case "PartitionID": value = r.FieldByName(field).Uint() case "Timestamp", "Length": value = r.FieldByName(field).Int() case "Source", "Kind", "Target", "DataStructure", "Identifier": value = r.FieldByName(field).String() default: return nil, fmt.Errorf("invalid field: %s", field) } return value, nil }) } type FragmentReceivedEvent struct { Kind string `json:"kind"` Source string `json:"source"` Identifier string `json:"identifier"` PartitionID uint64 `json:"partition_id"` DataStructure string `json:"data_structure"` Length int `json:"length"` IsBackup bool `json:"is_backup"` Timestamp int64 `json:"timestamp"` } func (f *FragmentReceivedEvent) Encode() (string, error) { fields := []string{ "Timestamp", "Source", "Kind", "DataStructure", "PartitionID", "Identifier", "IsBackup", "Length", } return encodeEvent(f, fields, func(r reflect.Value, field string) (interface{}, error) { var value interface{} switch field { case "IsBackup": value = r.FieldByName(field).Bool() case "PartitionID": value = r.FieldByName(field).Uint() case "Timestamp", "Length": value = r.FieldByName(field).Int() case "Source", "Kind", "DataStructure", "Identifier": value = r.FieldByName(field).String() default: return nil, fmt.Errorf("invalid field: %s", field) } return value, nil }) } ================================================ FILE: events/cluster_events_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package events import ( "testing" "github.com/stretchr/testify/require" ) func TestClusterEvents_NodeJoinEvent(t *testing.T) { var timestamp int64 = 585199808000 // Author's birthdate! n := NodeJoinEvent{ Kind: KindNodeJoinEvent, Source: "127.0.0.1:3423", NodeJoin: "127.0.0.1:3576", Timestamp: timestamp, } result, err := n.Encode() require.NoError(t, err) expected := `{"timestamp":585199808000,"source":"127.0.0.1:3423","kind":"node-join-event","node_join":"127.0.0.1:3576"}` require.Equal(t, expected, result) } func TestClusterEvents_NodeLeftEvent(t *testing.T) { var timestamp int64 = 585199808000 // Author's birthdate! n := NodeLeftEvent{ Kind: KindNodeLeftEvent, Source: "127.0.0.1:3423", NodeLeft: "127.0.0.1:3576", Timestamp: timestamp, } result, err := n.Encode() require.NoError(t, err) expected := `{"timestamp":585199808000,"source":"127.0.0.1:3423","kind":"node-left-event","node_left":"127.0.0.1:3576"}` require.Equal(t, expected, result) } func TestClusterEvents_FragmentMigrationEvent(t *testing.T) { var timestamp int64 = 585199808000 // Author's birthdate! n := FragmentMigrationEvent{ Kind: KindFragmentMigrationEvent, Source: "127.0.0.1:3423", Target: "127.0.0.1:3576", Identifier: "mydmap", PartitionID: 123, DataStructure: "dmap", Length: 1234, Timestamp: timestamp, } result, err := n.Encode() require.NoError(t, err) expected := `{"timestamp":585199808000,"source":"127.0.0.1:3423","kind":"fragment-migration-event","target":"127.0.0.1:3576","data_structure":"dmap","partition_id":123,"identifier":"mydmap","is_backup":false,"length":1234}` require.Equal(t, expected, result) } func TestClusterEvents_FragmentReceivedEvent(t *testing.T) { var timestamp int64 = 585199808000 // Author's birthdate! n := FragmentReceivedEvent{ Kind: KindFragmentReceivedEvent, Source: "127.0.0.1:3423", Identifier: "mydmap", PartitionID: 123, DataStructure: "dmap", Length: 1234, Timestamp: timestamp, } result, err := n.Encode() require.NoError(t, err) expected := `{"timestamp":585199808000,"source":"127.0.0.1:3423","kind":"fragment-received-event","data_structure":"dmap","partition_id":123,"identifier":"mydmap","is_backup":false,"length":1234}` require.Equal(t, expected, result) } ================================================ FILE: get_response.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package olric import ( "errors" "time" "github.com/olric-data/olric/internal/resp" "github.com/olric-data/olric/pkg/storage" ) var ErrNilResponse = errors.New("storage entry is nil") type GetResponse struct { entry storage.Entry } func (g *GetResponse) Scan(v interface{}) error { if g.entry == nil { return ErrNilResponse } return resp.Scan(g.entry.Value(), v) } func (g *GetResponse) Int() (int, error) { v := new(int) err := g.Scan(v) if err != nil { return 0, err } return *v, nil } func (g *GetResponse) String() (string, error) { v := new(string) err := g.Scan(v) if err != nil { return "", err } return *v, nil } func (g *GetResponse) Int8() (int8, error) { v := new(int8) err := g.Scan(v) if err != nil { return 0, err } return *v, nil } func (g *GetResponse) Int16() (int16, error) { v := new(int16) err := g.Scan(v) if err != nil { return 0, err } return *v, nil } func (g *GetResponse) Int32() (int32, error) { v := new(int32) err := g.Scan(v) if err != nil { return 0, err } return *v, nil } func (g *GetResponse) Int64() (int64, error) { v := new(int64) err := g.Scan(v) if err != nil { return 0, err } return *v, nil } func (g *GetResponse) Uint() (uint, error) { v := new(uint) err := g.Scan(v) if err != nil { return 0, err } return *v, nil } func (g *GetResponse) Uint8() (uint8, error) { v := new(uint8) err := g.Scan(v) if err != nil { return 0, err } return *v, nil } func (g *GetResponse) Uint16() (uint16, error) { v := new(uint16) err := g.Scan(v) if err != nil { return 0, err } return *v, nil } func (g *GetResponse) Uint32() (uint32, error) { v := new(uint32) err := g.Scan(v) if err != nil { return 0, err } return *v, nil } func (g *GetResponse) Uint64() (uint64, error) { v := new(uint64) err := g.Scan(v) if err != nil { return 0, err } return *v, nil } func (g *GetResponse) Float32() (float32, error) { v := new(float32) err := g.Scan(v) if err != nil { return 0, err } return *v, nil } func (g *GetResponse) Float64() (float64, error) { v := new(float64) err := g.Scan(v) if err != nil { return 0, err } return *v, nil } func (g *GetResponse) Bool() (bool, error) { v := new(bool) err := g.Scan(v) if err != nil { return false, err } return *v, nil } func (g *GetResponse) Time() (time.Time, error) { v := new(time.Time) err := g.Scan(v) if err != nil { return time.Time{}, err } return *v, nil } func (g *GetResponse) Duration() (time.Duration, error) { v := new(time.Duration) err := g.Scan(v) if err != nil { return 0, err } return *v, nil } func (g *GetResponse) Byte() ([]byte, error) { v := new([]byte) err := g.Scan(v) if err != nil { return nil, err } return *v, nil } func (g *GetResponse) TTL() int64 { return g.entry.TTL() } func (g *GetResponse) Timestamp() int64 { return g.entry.Timestamp() } ================================================ FILE: get_response_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package olric import ( "bytes" "context" "encoding/json" "testing" "time" "github.com/olric-data/olric/internal/dmap" "github.com/olric-data/olric/internal/resp" "github.com/olric-data/olric/internal/testcluster" "github.com/stretchr/testify/require" ) func TestDMap_Get_GetResponse(t *testing.T) { cluster := testcluster.New(dmap.NewService) s := cluster.AddMember(nil).(*dmap.Service) defer cluster.Shutdown() ctx := context.Background() dm, err := s.NewDMap("mydmap") require.NoError(t, err) t.Run("Scan", func(t *testing.T) { var value = 100 err = dm.Put(ctx, "mykey-scan", value, nil) require.NoError(t, err) e, err := dm.Get(ctx, "mykey-scan") require.NoError(t, err) gr := &GetResponse{entry: e} scannedValue := new(int) err = gr.Scan(scannedValue) require.NoError(t, err) require.Equal(t, value, *scannedValue) }) t.Run("Byte", func(t *testing.T) { var value = []byte("olric") err = dm.Put(ctx, "mykey-byte", value, nil) require.NoError(t, err) e, err := dm.Get(ctx, "mykey-byte") require.NoError(t, err) gr := &GetResponse{entry: e} scannedValue, err := gr.Byte() require.NoError(t, err) require.Equal(t, value, scannedValue) }) t.Run("TTL", func(t *testing.T) { var value = []byte("olric") err = dm.Put(ctx, "mykey-byte", value, &dmap.PutConfig{ HasEX: true, EX: time.Second, }) require.NoError(t, err) e, err := dm.Get(ctx, "mykey-byte") require.NoError(t, err) gr := &GetResponse{entry: e} ttl := gr.TTL() require.Greater(t, ttl, int64(0)) }) t.Run("Timestamp", func(t *testing.T) { var value = []byte("olric") err = dm.Put(ctx, "mykey-byte", value, nil) require.NoError(t, err) e, err := dm.Get(ctx, "mykey-byte") require.NoError(t, err) gr := &GetResponse{entry: e} timestamp := gr.Timestamp() require.Greater(t, timestamp, int64(0)) }) t.Run("Int", func(t *testing.T) { var value = 100 err = dm.Put(ctx, "mykey-Int", value, nil) require.NoError(t, err) e, err := dm.Get(ctx, "mykey-Int") require.NoError(t, err) gr := &GetResponse{entry: e} scannedValue, err := gr.Int() require.NoError(t, err) require.Equal(t, value, scannedValue) }) t.Run("String", func(t *testing.T) { var value = "olric" err = dm.Put(ctx, "mykey-String", value, nil) require.NoError(t, err) e, err := dm.Get(ctx, "mykey-String") require.NoError(t, err) gr := &GetResponse{entry: e} scannedValue, err := gr.String() require.NoError(t, err) require.Equal(t, value, scannedValue) }) t.Run("Int8", func(t *testing.T) { var value int8 = 10 err = dm.Put(ctx, "mykey-Int8", value, nil) require.NoError(t, err) e, err := dm.Get(ctx, "mykey-Int8") require.NoError(t, err) gr := &GetResponse{entry: e} scannedValue, err := gr.Int8() require.NoError(t, err) require.Equal(t, value, scannedValue) }) t.Run("Int16", func(t *testing.T) { var value int16 = 10 err = dm.Put(ctx, "mykey-Int16", value, nil) require.NoError(t, err) e, err := dm.Get(ctx, "mykey-Int16") require.NoError(t, err) gr := &GetResponse{entry: e} scannedValue, err := gr.Int16() require.NoError(t, err) require.Equal(t, value, scannedValue) }) t.Run("Int32", func(t *testing.T) { var value int32 = 10 err = dm.Put(ctx, "mykey-Int32", value, nil) require.NoError(t, err) e, err := dm.Get(ctx, "mykey-Int32") require.NoError(t, err) gr := &GetResponse{entry: e} scannedValue, err := gr.Int32() require.NoError(t, err) require.Equal(t, value, scannedValue) }) t.Run("Int64", func(t *testing.T) { var value int64 = 10 err = dm.Put(ctx, "mykey-Int64", value, nil) require.NoError(t, err) e, err := dm.Get(ctx, "mykey-Int64") require.NoError(t, err) gr := &GetResponse{entry: e} scannedValue, err := gr.Int64() require.NoError(t, err) require.Equal(t, value, scannedValue) }) t.Run("Int64", func(t *testing.T) { var value int64 = 10 err = dm.Put(ctx, "mykey-Int64", value, nil) require.NoError(t, err) e, err := dm.Get(ctx, "mykey-Int64") require.NoError(t, err) gr := &GetResponse{entry: e} scannedValue, err := gr.Int64() require.NoError(t, err) require.Equal(t, value, scannedValue) }) t.Run("Uint", func(t *testing.T) { var value uint = 10 err = dm.Put(ctx, "mykey-Uint", value, nil) require.NoError(t, err) e, err := dm.Get(ctx, "mykey-Uint") require.NoError(t, err) gr := &GetResponse{entry: e} scannedValue, err := gr.Uint() require.NoError(t, err) require.Equal(t, value, scannedValue) }) t.Run("Uint8", func(t *testing.T) { var value uint8 = 10 err = dm.Put(ctx, "mykey-Uint8", value, nil) require.NoError(t, err) e, err := dm.Get(ctx, "mykey-Uint8") require.NoError(t, err) gr := &GetResponse{entry: e} scannedValue, err := gr.Uint8() require.NoError(t, err) require.Equal(t, value, scannedValue) }) t.Run("Uint16", func(t *testing.T) { var value uint16 = 10 err = dm.Put(ctx, "mykey-Uint16", value, nil) require.NoError(t, err) e, err := dm.Get(ctx, "mykey-Uint16") require.NoError(t, err) gr := &GetResponse{entry: e} scannedValue, err := gr.Uint16() require.NoError(t, err) require.Equal(t, value, scannedValue) }) t.Run("Uint32", func(t *testing.T) { var value uint32 = 10 err = dm.Put(ctx, "mykey-Uint32", value, nil) require.NoError(t, err) e, err := dm.Get(ctx, "mykey-Uint32") require.NoError(t, err) gr := &GetResponse{entry: e} scannedValue, err := gr.Uint32() require.NoError(t, err) require.Equal(t, value, scannedValue) }) t.Run("Uint64", func(t *testing.T) { var value uint64 = 10 err = dm.Put(ctx, "mykey-Uint64", value, nil) require.NoError(t, err) e, err := dm.Get(ctx, "mykey-Uint64") require.NoError(t, err) gr := &GetResponse{entry: e} scannedValue, err := gr.Uint64() require.NoError(t, err) require.Equal(t, value, scannedValue) }) t.Run("Float32", func(t *testing.T) { var value float32 = 10.12 err = dm.Put(ctx, "mykey-Float32", value, nil) require.NoError(t, err) e, err := dm.Get(ctx, "mykey-Float32") require.NoError(t, err) gr := &GetResponse{entry: e} scannedValue, err := gr.Float32() require.NoError(t, err) require.Equal(t, value, scannedValue) }) t.Run("Float64", func(t *testing.T) { var value = 10.12 err = dm.Put(ctx, "mykey-Float64", value, nil) require.NoError(t, err) e, err := dm.Get(ctx, "mykey-Float64") require.NoError(t, err) gr := &GetResponse{entry: e} scannedValue, err := gr.Float64() require.NoError(t, err) require.Equal(t, value, scannedValue) }) t.Run("Bool", func(t *testing.T) { err = dm.Put(ctx, "mykey-Bool", true, nil) require.NoError(t, err) e, err := dm.Get(ctx, "mykey-Bool") require.NoError(t, err) gr := &GetResponse{entry: e} scannedValue, err := gr.Bool() require.NoError(t, err) require.Equal(t, true, scannedValue) }) t.Run("time.Time", func(t *testing.T) { var value = time.Now() err = dm.Put(ctx, "mykey-time.Time", value, nil) require.NoError(t, err) e, err := dm.Get(ctx, "mykey-time.Time") require.NoError(t, err) gr := &GetResponse{entry: e} buf := bytes.NewBuffer(nil) enc := resp.New(buf) err = enc.Encode(value) require.NoError(t, err) expectedValue := new(time.Time) err = resp.Scan(buf.Bytes(), expectedValue) require.NoError(t, err) scannedValue, err := gr.Time() require.NoError(t, err) require.Equal(t, *expectedValue, scannedValue) }) t.Run("time.Duration", func(t *testing.T) { var value = time.Second err = dm.Put(ctx, "mykey-time.Duration", value, nil) require.NoError(t, err) e, err := dm.Get(ctx, "mykey-time.Duration") require.NoError(t, err) gr := &GetResponse{entry: e} buf := bytes.NewBuffer(nil) enc := resp.New(buf) err = enc.Encode(value) require.NoError(t, err) expectedValue := new(time.Duration) err = resp.Scan(buf.Bytes(), expectedValue) require.NoError(t, err) scannedValue, err := gr.Duration() require.NoError(t, err) require.Equal(t, *expectedValue, scannedValue) }) t.Run("BinaryUnmarshaler", func(t *testing.T) { var value = &myType{ Database: "olric", } err = dm.Put(ctx, "mykey-BinaryUnmarshaler", value, nil) require.NoError(t, err) e, err := dm.Get(ctx, "mykey-BinaryUnmarshaler") require.NoError(t, err) gr := &GetResponse{entry: e} v := myType{} err = gr.Scan(&v) require.NoError(t, err) require.Equal(t, value, &v) }) } type myType struct { Database string } func (mt *myType) MarshalBinary() ([]byte, error) { return json.Marshal(mt) } func (mt *myType) UnmarshalBinary(data []byte) error { return json.Unmarshal(data, &mt) } ================================================ FILE: go.mod ================================================ module github.com/olric-data/olric go 1.23.0 require ( github.com/RoaringBitmap/roaring v1.9.4 github.com/buraksezer/consistent v0.10.0 github.com/cespare/xxhash/v2 v2.3.0 github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/go-sockaddr v1.0.7 github.com/hashicorp/logutils v1.0.0 github.com/hashicorp/memberlist v0.5.3 github.com/pkg/errors v0.9.1 github.com/redis/go-redis/v9 v9.8.0 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 github.com/stretchr/testify v1.10.0 github.com/tidwall/btree v1.7.0 github.com/tidwall/match v1.1.1 github.com/tidwall/redcon v1.6.2 github.com/vmihailenco/msgpack/v5 v5.4.1 golang.org/x/sync v0.14.0 gopkg.in/yaml.v2 v2.4.0 ) require ( github.com/armon/go-metrics v0.4.1 // indirect github.com/bits-and-blooms/bitset v1.22.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/google/btree v1.1.3 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect github.com/hashicorp/go-metrics v0.5.4 // indirect github.com/hashicorp/go-msgpack/v2 v2.1.3 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/miekg/dns v1.1.65 // indirect github.com/mschoch/smat v0.2.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect golang.org/x/mod v0.24.0 // indirect golang.org/x/net v0.38.0 // indirect golang.org/x/sys v0.32.0 // indirect golang.org/x/tools v0.31.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) ================================================ FILE: go.sum ================================================ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/RoaringBitmap/roaring v1.9.4 h1:yhEIoH4YezLYT04s1nHehNO64EKFTop/wBhxv2QzDdQ= github.com/RoaringBitmap/roaring v1.9.4/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bits-and-blooms/bitset v1.22.0 h1:Tquv9S8+SGaS3EhyA+up3FXzmkhxPGjQQCkcs2uw7w4= github.com/bits-and-blooms/bitset v1.22.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/buraksezer/consistent v0.10.0 h1:hqBgz1PvNLC5rkWcEBVAL9dFMBWz6I0VgUCW25rrZlU= github.com/buraksezer/consistent v0.10.0/go.mod h1:6BrVajWq7wbKZlTOUPs/XVfR8c0maujuPowduSpZqmw= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-metrics v0.5.4 h1:8mmPiIJkTPPEbAiV97IxdAGNdRdaWwVap1BU6elejKY= github.com/hashicorp/go-metrics v0.5.4/go.mod h1:CG5yz4NZ/AI/aQt9Ucm/vdBnbh7fvmv4lxZ350i+QQI= github.com/hashicorp/go-msgpack/v2 v2.1.3 h1:cB1w4Zrk0O3jQBTcFMKqYQWRFfsSQ/TYKNyUUVyCP2c= github.com/hashicorp/go-msgpack/v2 v2.1.3/go.mod h1:SjlwKKFnwBXvxD/I1bEcfJIBbEJ+MCUn39TxymNR5ZU= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-sockaddr v1.0.7 h1:G+pTkSO01HpR5qCxg7lxfsFEZaG+C0VssTy/9dbT+Fw= github.com/hashicorp/go-sockaddr v1.0.7/go.mod h1:FZQbEYa1pxkQ7WLpyXJ6cbjpT8q0YgQaK/JakXqGyWw= github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/logutils v1.0.0 h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI65Y= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/memberlist v0.5.3 h1:tQ1jOCypD0WvMemw/ZhhtH+PWpzcftQvgCorLu0hndk= github.com/hashicorp/memberlist v0.5.3/go.mod h1:h60o12SZn/ua/j0B6iKAZezA4eDaGsIuPO70eOaJ6WE= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.1.65 h1:0+tIPHzUW0GCge7IiK3guGP57VAw7hoPDfApjkMD1Fc= github.com/miekg/dns v1.1.65/go.mod h1:Dzw9769uoKVaLuODMDZz9M6ynFU6Em65csPuoi8G0ck= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/redis/go-redis/v9 v9.8.0 h1:q3nRvjrlge/6UD7eTu/DSg2uYiU2mCL0G/uzBWqhicI= github.com/redis/go-redis/v9 v9.8.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tidwall/btree v1.1.0/go.mod h1:TzIRzen6yHbibdSfK6t8QimqbUnoxUSrZfeW7Uob0q4= github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI= github.com/tidwall/btree v1.7.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/redcon v1.6.2 h1:5qfvrrybgtO85jnhSravmkZyC0D+7WstbfCs3MmPhow= github.com/tidwall/redcon v1.6.2/go.mod h1:p5Wbsgeyi2VSTBWOcA5vRXrOb9arFTcU2+ZzFjqV75Y= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8= github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.31.0 h1:0EedkvKDbh+qistFTd0Bcwe/YLh4vHwWEkiI0toFIBU= golang.org/x/tools v0.31.0/go.mod h1:naFTU+Cev749tSJRXJlna0T3WxKvb1kWEx15xA4SdmQ= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= ================================================ FILE: hasher/hasher.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package hasher import "github.com/cespare/xxhash/v2" // NewDefaultHasher returns an instance of xxhash package which implements the 64-bit variant of // xxHash (XXH64) as described at http://cyan4973.github.io/xxHash/. func NewDefaultHasher() Hasher { return xxhasher{} } type xxhasher struct{} func (x xxhasher) Sum64(key []byte) uint64 { return xxhash.Sum64(key) } // Hasher is responsible for generating unsigned, 64 bit hash of provided byte slice. // Hasher should minimize collisions (generating same hash for different byte slice) // and while performance is also important fast functions are preferable (i.e. // you can use FarmHash family). type Hasher interface { Sum64([]byte) uint64 } ================================================ FILE: integration_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package olric import ( "context" "errors" "fmt" "io" "testing" "time" "github.com/olric-data/olric/config" "github.com/stretchr/testify/require" ) func TestIntegration_NodesJoinOrLeftDuringQuery(t *testing.T) { // TODO: https://github.com/olric-data/olric/issues/227 t.Skip("TestIntegration_NodesJoinOrLeftDuringQuery: flaky test") newConfig := func() *config.Config { c := config.New("local") c.PartitionCount = config.DefaultPartitionCount c.ReplicaCount = 2 c.WriteQuorum = 1 c.ReadRepair = true c.ReadQuorum = 1 c.LogOutput = io.Discard require.NoError(t, c.Sanitize()) require.NoError(t, c.Validate()) return c } cluster := newTestOlricCluster(t) db := cluster.addMemberWithConfig(t, newConfig()) db2 := cluster.addMemberWithConfig(t, newConfig()) t.Log("Wait for 1 second before inserting keys") <-time.After(time.Second) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) for i := 0; i < 100000; i++ { err = dm.Put(ctx, fmt.Sprintf("mykey-%d", i), "myvalue") require.NoError(t, err) if i == 5999 { go cluster.addMemberWithConfig(t, newConfig()) } } go cluster.addMemberWithConfig(t, newConfig()) t.Log("Fetch all keys") for i := 0; i < 100000; i++ { _, err = dm.Get(context.Background(), fmt.Sprintf("mykey-%d", i)) if errors.Is(err, ErrConnRefused) { // Rewind i-- require.NoError(t, c.RefreshMetadata(context.Background())) continue } require.NoError(t, err) if i == 5999 { err = c.client.Close(db2.name) require.NoError(t, err) t.Logf("Shutdown one of the nodes: %s", db2.name) require.NoError(t, db2.Shutdown(ctx)) go cluster.addMemberWithConfig(t, newConfig()) t.Log("Wait for \"NodeLeave\" event propagation") <-time.After(time.Second) } } for i := 0; i < 100000; i++ { _, err = dm.Get(context.Background(), fmt.Sprintf("mykey-%d", i)) require.NoError(t, err) } } func TestIntegration_DMap_Cache_Eviction_LRU_MaxKeys(t *testing.T) { maxKeys := 100000 newConfig := func() *config.Config { c := config.New("local") c.PartitionCount = config.DefaultPartitionCount c.ReplicaCount = 1 c.WriteQuorum = 1 c.ReadQuorum = 1 c.LogOutput = io.Discard c.DMaps.MaxKeys = maxKeys c.DMaps.EvictionPolicy = config.LRUEviction require.NoError(t, c.Sanitize()) require.NoError(t, c.Validate()) return c } cluster := newTestOlricCluster(t) db := cluster.addMemberWithConfig(t, newConfig()) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) for i := 0; i < maxKeys; i++ { err = dm.Put(ctx, fmt.Sprintf("mykey-%d", i), "myvalue") require.NoError(t, err) } var total int for i := 0; i < maxKeys; i++ { err = dm.Put(ctx, fmt.Sprintf("mykey-%d", i), "myvalue", NX()) if err == ErrKeyFound { err = nil } else { total++ } require.NoError(t, err) } require.Greater(t, total, 0) t.Logf("number of misses: %d, utilization rate: %f", total, float64(100)-(float64(total*100))/float64(maxKeys)) } func TestIntegration_DMap_Cache_Eviction_MaxKeys(t *testing.T) { maxKeys := 100000 newConfig := func() *config.Config { c := config.New("local") c.PartitionCount = config.DefaultPartitionCount c.ReplicaCount = 1 c.WriteQuorum = 1 c.ReadQuorum = 1 c.LogOutput = io.Discard c.DMaps.MaxKeys = maxKeys require.NoError(t, c.Sanitize()) require.NoError(t, c.Validate()) return c } cluster := newTestOlricCluster(t) db := cluster.addMemberWithConfig(t, newConfig()) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) for i := 0; i < maxKeys; i++ { err = dm.Put(ctx, fmt.Sprintf("mykey-%d", i), "myvalue") require.NoError(t, err) } var total int for i := maxKeys; i < 2*maxKeys; i++ { err = dm.Put(ctx, fmt.Sprintf("mykey-%d", i), "myvalue", NX()) if err == ErrKeyFound { err = nil } else { total++ } require.NoError(t, err) } for i := 0; i < maxKeys; i++ { _, err = dm.Get(ctx, fmt.Sprintf("mykey-%d", i)) if err == ErrKeyNotFound { err = nil total++ } require.NoError(t, err) } require.Equal(t, maxKeys, total) } func TestIntegration_DMap_Cache_Eviction_MaxIdleDuration(t *testing.T) { maxKeys := 100000 newConfig := func() *config.Config { c := config.New("local") c.PartitionCount = config.DefaultPartitionCount c.ReplicaCount = 1 c.WriteQuorum = 1 c.ReadQuorum = 1 c.LogOutput = io.Discard c.DMaps.MaxIdleDuration = 100 * time.Millisecond require.NoError(t, c.Sanitize()) require.NoError(t, c.Validate()) return c } cluster := newTestOlricCluster(t) db := cluster.addMemberWithConfig(t, newConfig()) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) for i := 0; i < maxKeys; i++ { err = dm.Put(ctx, fmt.Sprintf("mykey-%d", i), "myvalue") require.NoError(t, err) } <-time.After(250 * time.Millisecond) var total int for i := 0; i < maxKeys; i++ { _, err = dm.Get(ctx, fmt.Sprintf("mykey-%d", i)) if err == ErrKeyNotFound { err = nil total++ } require.NoError(t, err) } require.Greater(t, total, 0) } func TestIntegration_DMap_Cache_Eviction_TTLDuration(t *testing.T) { maxKeys := 100000 newConfig := func() *config.Config { c := config.New("local") c.PartitionCount = config.DefaultPartitionCount c.ReplicaCount = 1 c.WriteQuorum = 1 c.ReadQuorum = 1 c.LogOutput = io.Discard c.DMaps.TTLDuration = 100 * time.Millisecond require.NoError(t, c.Sanitize()) require.NoError(t, c.Validate()) return c } cluster := newTestOlricCluster(t) db := cluster.addMemberWithConfig(t, newConfig()) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) for i := 0; i < maxKeys; i++ { err = dm.Put(ctx, fmt.Sprintf("mykey-%d", i), "myvalue") require.NoError(t, err) } <-time.After(250 * time.Millisecond) var total int for i := 0; i < maxKeys; i++ { _, err := dm.Get(ctx, fmt.Sprintf("mykey-%d", i)) if err == ErrKeyNotFound { err = nil total++ } require.NoError(t, err) } require.Equal(t, maxKeys, total) } func TestIntegration_DMap_Cache_Eviction_LRU_MaxInuse(t *testing.T) { maxKeys := 100000 newConfig := func() *config.Config { c := config.New("local") c.PartitionCount = config.DefaultPartitionCount c.ReplicaCount = 1 c.WriteQuorum = 1 c.ReadQuorum = 1 c.LogOutput = io.Discard c.DMaps.MaxInuse = 100 // bytes c.DMaps.EvictionPolicy = "LRU" require.NoError(t, c.Sanitize()) require.NoError(t, c.Validate()) return c } cluster := newTestOlricCluster(t) db := cluster.addMemberWithConfig(t, newConfig()) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) for i := 0; i < maxKeys; i++ { err = dm.Put(ctx, fmt.Sprintf("mykey-%d", i), "myvalue") require.NoError(t, err) } <-time.After(250 * time.Millisecond) var total int for i := 0; i < maxKeys; i++ { _, err = dm.Get(ctx, fmt.Sprintf("mykey-%d", i)) if err == ErrKeyNotFound { err = nil total++ } require.NoError(t, err) } require.Greater(t, total, 0) } func TestIntegration_Kill_Nodes_During_Operation(t *testing.T) { newConfig := func() *config.Config { c := config.New("local") c.PartitionCount = config.DefaultPartitionCount c.ReplicaCount = 3 c.WriteQuorum = 1 c.ReadRepair = true c.ReadQuorum = 1 c.LogOutput = io.Discard require.NoError(t, c.Sanitize()) require.NoError(t, c.Validate()) return c } cluster := newTestOlricCluster(t) db := cluster.addMemberWithConfig(t, newConfig()) cluster.addMemberWithConfig(t, newConfig()) db3 := cluster.addMemberWithConfig(t, newConfig()) cluster.addMemberWithConfig(t, newConfig()) db5 := cluster.addMemberWithConfig(t, newConfig()) t.Log("Wait for 1 second before inserting keys") <-time.After(time.Second) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() require.NoError(t, err) dm, err := c.NewDMap("mydmap") require.NoError(t, err) t.Log("Insert keys") for i := 0; i < 100000; i++ { err = dm.Put(ctx, fmt.Sprintf("mykey-%d", i), "myvalue") require.NoError(t, err) } t.Log("Fetch all keys") for i := 0; i < 100000; i++ { _, err = dm.Get(ctx, fmt.Sprintf("mykey-%d", i)) if err == ErrKeyNotFound { err = nil } require.NoError(t, err) } t.Logf("Terminate %s", db3.rt.This()) require.NoError(t, db3.Shutdown(context.Background())) t.Logf("Terminate %s", db5.rt.This()) require.NoError(t, db5.Shutdown(context.Background())) t.Log("Wait for \"NodeLeave\" event propagation") <-time.After(time.Second) for i := 0; i < 100000; i++ { _, err = dm.Get(context.Background(), fmt.Sprintf("mykey-%d", i)) if errors.Is(err, ErrConnRefused) { i-- fmt.Println(c.RefreshMetadata(context.Background())) continue } require.NoError(t, err) } } func scanIntegrationTestCommon(t *testing.T, embedded bool, keyFunc func(i int) string, options ...ScanOption) []map[string]struct{} { newConfig := func() *config.Config { c := config.New("local") c.PartitionCount = config.DefaultPartitionCount c.ReplicaCount = 2 c.WriteQuorum = 1 c.ReadRepair = false c.ReadQuorum = 1 c.LogOutput = io.Discard c.TriggerBalancerInterval = time.Millisecond require.NoError(t, c.Sanitize()) require.NoError(t, c.Validate()) return c } cluster := newTestOlricCluster(t) db := cluster.addMemberWithConfig(t, newConfig()) db2 := cluster.addMemberWithConfig(t, newConfig()) _ = cluster.addMemberWithConfig(t, newConfig()) t.Log("Wait for 1 second before inserting keys") <-time.After(time.Second) ctx := context.Background() var c Client var err error if embedded { c = db.NewEmbeddedClient() } else { c, err = NewClusterClient([]string{db.name}) require.NoError(t, err) } defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) passOne := make(map[string]struct{}) passTwo := make(map[string]struct{}) for i := 0; i < 10000; i++ { key := keyFunc(i) err = dm.Put(ctx, key, "myvalue") require.NoError(t, err) passOne[key] = struct{}{} passTwo[key] = struct{}{} } t.Logf("Shutdown one of the nodes: %s", db2.name) require.NoError(t, db2.Shutdown(ctx)) t.Log("Wait for \"NodeLeave\" event propagation") <-time.After(time.Second) t.Log("First pass") s, err := dm.Scan(context.Background(), options...) require.NoError(t, err) for s.Next() { delete(passOne, s.Key()) } s.Close() db3 := cluster.addMemberWithConfig(t, newConfig()) t.Logf("Add a new member: %s", db3.rt.This()) <-time.After(time.Second) t.Log("Second pass") s, err = dm.Scan(context.Background(), options...) require.NoError(t, err) for s.Next() { delete(passTwo, s.Key()) } return []map[string]struct{}{passOne, passTwo} } func TestIntegration_Network_Partitioning_Cluster_DM_SCAN(t *testing.T) { keyGenerator := func(i int) string { return fmt.Sprintf("mykey-%d", i) } result := scanIntegrationTestCommon(t, false, keyGenerator) passOne, passTwo := result[0], result[1] require.Empty(t, passOne) require.Empty(t, passTwo) } func TestIntegration_Network_Partitioning_Cluster_DM_SCAN_Match(t *testing.T) { var oddNumbers int keyGenerator := func(i int) string { if i%2 == 0 { return fmt.Sprintf("even:%d", i) } oddNumbers++ return fmt.Sprintf("odd:%d", i) } result := scanIntegrationTestCommon(t, false, keyGenerator, Match("^even:")) passOne, passTwo := result[0], result[1] require.Len(t, passOne, oddNumbers) require.Len(t, passTwo, oddNumbers) } func TestIntegration_Network_Partitioning_Embedded_DM_SCAN(t *testing.T) { keyGenerator := func(i int) string { return fmt.Sprintf("mykey-%d", i) } result := scanIntegrationTestCommon(t, true, keyGenerator) passOne, passTwo := result[0], result[1] require.Empty(t, passOne) require.Empty(t, passTwo) } func TestIntegration_Network_Partitioning_Embedded_DM_SCAN_Match(t *testing.T) { var oddNumbers int keyGenerator := func(i int) string { if i%2 == 0 { return fmt.Sprintf("even:%d", i) } oddNumbers++ return fmt.Sprintf("odd:%d", i) } result := scanIntegrationTestCommon(t, true, keyGenerator, Match("^even:")) passOne, passTwo := result[0], result[1] require.Len(t, passOne, oddNumbers) require.Len(t, passTwo, oddNumbers) } ================================================ FILE: internal/bufpool/bufpool.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package bufpool import ( "bytes" "sync" ) // BufPool maintains a buffer pool. type BufPool struct { p sync.Pool } // New creates a new BufPool. func New() *BufPool { return &BufPool{ p: sync.Pool{ New: func() interface{} { return new(bytes.Buffer) }, }, } } // Put resets the buffer and puts it back to the pool. func (p *BufPool) Put(b *bytes.Buffer) { b.Reset() p.p.Put(b) } // Get returns an empty buffer from the pool. It creates a new buffer, if there // is no bytes.Buffer available in the pool. func (p *BufPool) Get() *bytes.Buffer { return p.p.Get().(*bytes.Buffer) } ================================================ FILE: internal/bufpool/bufpool_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package bufpool import ( "testing" "github.com/stretchr/testify/require" ) func TestBufPool(t *testing.T) { p := New() b := make([]byte, 100) for i := 0; i < 1000; i++ { buf := p.Get() nr, err := buf.Write(b) require.NoError(t, err) require.Equal(t, 100, nr) p.Put(buf) } } ================================================ FILE: internal/checkpoint/checkpoint.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package checkpoint import "sync/atomic" var ( required int32 passed int32 ) func Add() { atomic.AddInt32(&required, 1) } func Pass() { atomic.AddInt32(&passed, 1) } func AllPassed() bool { return atomic.LoadInt32(&passed) == required } ================================================ FILE: internal/checkpoint/checkpoint_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package checkpoint import ( "sync" "testing" "github.com/stretchr/testify/require" ) func TestCheckpoint(t *testing.T) { var wg sync.WaitGroup for i := 0; i < 10; i++ { wg.Add(1) go func() { defer wg.Done() Add() }() } wg.Wait() for i := 0; i < 10; i++ { wg.Add(1) go func() { defer wg.Done() Pass() }() } wg.Wait() require.Equal(t, true, AllPassed()) } ================================================ FILE: internal/cluster/balancer/balancer.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package balancer import ( "context" "strings" "sync" "time" "github.com/olric-data/olric/config" "github.com/olric-data/olric/internal/cluster/partitions" "github.com/olric-data/olric/internal/cluster/routingtable" "github.com/olric-data/olric/internal/discovery" "github.com/olric-data/olric/internal/environment" "github.com/olric-data/olric/internal/service" "github.com/olric-data/olric/pkg/flog" ) type Balancer struct { sync.Mutex log *flog.Logger config *config.Config primary *partitions.Partitions backup *partitions.Partitions rt *routingtable.RoutingTable wg sync.WaitGroup ctx context.Context cancel context.CancelFunc } func New(e *environment.Environment) *Balancer { c := e.Get("config").(*config.Config) log := e.Get("logger").(*flog.Logger) ctx, cancel := context.WithCancel(context.Background()) return &Balancer{ config: c, primary: e.Get("primary").(*partitions.Partitions), backup: e.Get("backup").(*partitions.Partitions), rt: e.Get("routingtable").(*routingtable.RoutingTable), log: log, ctx: ctx, cancel: cancel, } } func (b *Balancer) isAlive() bool { select { case <-b.ctx.Done(): // The node is gone. return false default: } return true } func (b *Balancer) scanPartition(sign uint64, part *partitions.Partition, owners ...discovery.Member) { ownersStr := func() string { var names []string for _, owner := range owners { names = append(names, owner.String()) } return strings.Join(names, ",") }() part.Map().Range(func(rawName, rawFragment interface{}) bool { f := rawFragment.(partitions.Fragment) if f.Stats().Length == 0 { return false } name := strings.TrimPrefix(rawName.(string), "dmap.") b.log.V(2).Printf("[INFO] Moving %s fragment: %s (kind: %s) on PartID: %d to %s", f.Name(), name, part.Kind(), part.ID(), ownersStr) err := f.Move(part, name, owners) if err != nil { b.log.V(2).Printf("[ERROR] Failed to move %s fragment: %s on PartID: %d to %s: %v", f.Name(), name, part.ID(), ownersStr, err) } // if this returns true, the iteration continues return !b.breakLoop(sign) }) } func (b *Balancer) primaryCopies() { sign := b.rt.Signature() for partID := uint64(0); partID < b.config.PartitionCount; partID++ { if b.breakLoop(sign) { break } part := b.primary.PartitionByID(partID) if part.Length() == 0 { // Empty partition. Skip it. continue } owner := part.Owner() // Here we don't use CompareByID function because the routing table is an // eventually consistent data structure and a node can try to move data // to previous instance(the same name but a different birthdate) // of itself. So just check the name. if owner.CompareByName(b.rt.This()) { // Already belongs to me. continue } // This is a previous owner. Move the keys. b.scanPartition(sign, part, owner) } } func (b *Balancer) breakLoop(sign uint64) bool { if !b.isAlive() { return true } if sign != b.rt.Signature() { // Routing table is updated. Just quit. Another balancer goroutine // will work on the new table immediately. return true } return false } func (b *Balancer) backupCopies() { sign := b.rt.Signature() LOOP: for partID := uint64(0); partID < b.config.PartitionCount; partID++ { if b.breakLoop(sign) { break } part := b.backup.PartitionByID(partID) if part.Length() == 0 || part.OwnerCount() == 0 { continue } var ( counter = 1 currentOwners []discovery.Member ) owners := part.Owners() for i := len(owners) - 1; i >= 0; i-- { if counter > b.config.ReplicaCount-1 { break } counter++ owner := owners[i] // Here we don't use CompareById function because the routing table // is an eventually consistent data structure and a node can try to // move data to previous instance(the same name but a different birthdate) // of itself. So just check the name. if b.rt.This().CompareByName(owner) { // Already belongs to me. continue LOOP } currentOwners = append(currentOwners, owner) } if len(currentOwners) == 0 { continue LOOP } b.scanPartition(sign, part, currentOwners...) } } func (b *Balancer) triggerBalancer() { b.Lock() defer b.Unlock() if err := b.rt.CheckBootstrap(); err != nil { b.log.V(2).Printf("[WARN] Balancer awaits for bootstrapping") return } b.primaryCopies() if b.config.ReplicaCount > config.MinimumReplicaCount { b.backupCopies() } } func (b *Balancer) BalanceEagerly() { b.triggerBalancer() } func (b *Balancer) balance() { defer b.wg.Done() timer := time.NewTimer(b.config.TriggerBalancerInterval) defer timer.Stop() for { timer.Reset(b.config.TriggerBalancerInterval) select { case <-timer.C: b.triggerBalancer() case <-b.ctx.Done(): return } } } func (b *Balancer) Start() error { b.wg.Add(1) go b.balance() return nil } func (b *Balancer) RegisterHandlers() {} func (b *Balancer) Shutdown(ctx context.Context) error { select { case <-b.ctx.Done(): // already closed return nil default: } b.cancel() done := make(chan struct{}) go func() { b.wg.Wait() close(done) }() select { case <-ctx.Done(): err := ctx.Err() if err != nil { return err } case <-done: } return nil } var _ service.Service = (*Balancer)(nil) ================================================ FILE: internal/cluster/balancer/balancer_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package balancer import ( "context" "errors" "fmt" "net" "strconv" "strings" "testing" "time" "github.com/olric-data/olric/config" "github.com/olric-data/olric/internal/cluster/partitions" "github.com/olric-data/olric/internal/cluster/routingtable" "github.com/olric-data/olric/internal/discovery" "github.com/olric-data/olric/internal/environment" "github.com/olric-data/olric/internal/server" "github.com/olric-data/olric/internal/testutil" "github.com/olric-data/olric/internal/testutil/mockfragment" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" ) func newTestEnvironment(c *config.Config) *environment.Environment { if c == nil { c = testutil.NewConfig() } e := environment.New() e.Set("config", c) e.Set("logger", testutil.NewFlogger(c)) e.Set("primary", partitions.New(c.PartitionCount, partitions.PRIMARY)) e.Set("backup", partitions.New(c.PartitionCount, partitions.BACKUP)) e.Set("client", server.NewClient(c.Client)) return e } func newBalancerForTest(e *environment.Environment) *Balancer { rt := routingtable.New(e) srv := e.Get("server").(*server.Server) go func() { err := srv.ListenAndServe() if err != nil { panic(fmt.Sprintf("ListenAndServe returned an error: %v", err)) } }() <-srv.StartedCtx.Done() e.Set("routingtable", rt) b := New(e) return b } type mockCluster struct { t *testing.T peerPorts []int errGr errgroup.Group ctx context.Context cancel context.CancelFunc } func newMockCluster(t *testing.T) *mockCluster { ctx, cancel := context.WithCancel(context.Background()) return &mockCluster{ t: t, ctx: ctx, cancel: cancel, } } func (mc *mockCluster) addNode(e *environment.Environment) *Balancer { if e == nil { e = newTestEnvironment(nil) } c := e.Get("config").(*config.Config) c.TriggerBalancerInterval = time.Millisecond c.DMaps.CheckEmptyFragmentsInterval = time.Millisecond port, err := testutil.GetFreePort() if err != nil { require.NoError(mc.t, err) } c.MemberlistConfig.BindPort = port var peers []string for _, peerPort := range mc.peerPorts { peers = append(peers, net.JoinHostPort("127.0.0.1", strconv.Itoa(peerPort))) } c.Peers = peers srv := testutil.NewServer(c) e.Set("server", srv) b := newBalancerForTest(e) err = b.Start() if err != nil { require.NoError(mc.t, err) } err = b.rt.Join() require.NoError(mc.t, err) err = b.rt.Start() if err != nil { require.NoError(mc.t, err) } mc.errGr.Go(func() error { <-mc.ctx.Done() return srv.Shutdown(context.Background()) }) mc.errGr.Go(func() error { <-mc.ctx.Done() return b.rt.Shutdown(context.Background()) }) mc.peerPorts = append(mc.peerPorts, port) mc.t.Cleanup(func() { require.NoError(mc.t, b.Shutdown(context.Background())) }) return b } func (mc *mockCluster) shutdown() { mc.cancel() require.NoError(mc.t, mc.errGr.Wait()) } func TestBalance_Primary_Move(t *testing.T) { cluster := newMockCluster(t) defer cluster.shutdown() e1 := newTestEnvironment(nil) cluster.addNode(e1) fragments := make(map[uint64]*mockfragment.MockFragment) // Create a MockFragment and insert some fake data c := e1.Get("config").(*config.Config) part := e1.Get(strings.ToLower(partitions.PRIMARY.String())).(*partitions.Partitions) for partID := uint64(0); partID < c.PartitionCount; partID++ { part := part.PartitionByID(partID) s := mockfragment.New() s.Fill() part.Map().Store("dmap.test-data", s) fragments[partID] = s } e2 := newTestEnvironment(nil) b2 := cluster.addNode(e2) err := testutil.TryWithInterval(10, 100*time.Millisecond, func() error { if !b2.rt.IsBootstrapped() { return errors.New("the second node cannot be bootstrapped") } return nil }) require.NoError(t, err) for partID, f := range fragments { result := f.Result() if len(result) == 0 { continue } require.Len(t, result, 1) require.NotNil(t, result[partitions.PRIMARY]) r := result[partitions.PRIMARY] require.NotNil(t, r[partID]) require.Equal(t, "test-data", r[partID].Name) require.Equal(t, []discovery.Member{b2.rt.This()}, r[partID].Owners) } } func checkBackupOwnership(e *environment.Environment) error { c := e.Get("config").(*config.Config) primary := e.Get(strings.ToLower(partitions.PRIMARY.String())).(*partitions.Partitions) backup := e.Get(strings.ToLower(partitions.BACKUP.String())).(*partitions.Partitions) for partID := uint64(0); partID < c.PartitionCount; partID++ { primaryOwner := primary.PartitionByID(partID).Owner() part := backup.PartitionByID(partID) for _, owner := range part.Owners() { if primaryOwner.CompareByID(owner) { return fmt.Errorf("%s is the primary and backup owner of partID: %d at the same time", primaryOwner, partID) } } } return nil } func TestBalance_Empty_Backup_Move(t *testing.T) { cluster := newMockCluster(t) defer cluster.shutdown() c1 := testutil.NewConfig() c1.ReplicaCount = 2 e1 := newTestEnvironment(c1) b1 := cluster.addNode(e1) b1.rt.UpdateEagerly() err := checkBackupOwnership(e1) require.NoError(t, err) c2 := testutil.NewConfig() c2.ReplicaCount = 2 e2 := newTestEnvironment(c2) b2 := cluster.addNode(e2) err = testutil.TryWithInterval(10, 100*time.Millisecond, func() error { if !b2.rt.IsBootstrapped() { return errors.New("the second node cannot be bootstrapped") } return nil }) require.NoError(t, err) b1.rt.UpdateEagerly() err = checkBackupOwnership(e2) require.NoError(t, err) } func TestBalance_Backup_Move(t *testing.T) { cluster := newMockCluster(t) defer cluster.shutdown() c1 := testutil.NewConfig() c1.ReplicaCount = 2 e1 := newTestEnvironment(c1) b1 := cluster.addNode(e1) fragments := make(map[uint64]*mockfragment.MockFragment) c := e1.Get("config").(*config.Config) part := e1.Get(strings.ToLower(partitions.BACKUP.String())).(*partitions.Partitions) for partID := uint64(0); partID < c.PartitionCount; partID++ { part := part.PartitionByID(partID) s := mockfragment.New() s.Fill() part.Map().Store("dmap.test-data", s) fragments[partID] = s } c2 := testutil.NewConfig() c2.ReplicaCount = 2 e2 := newTestEnvironment(c2) b2 := cluster.addNode(e2) err := testutil.TryWithInterval(10, 100*time.Millisecond, func() error { if !b2.rt.IsBootstrapped() { return errors.New("the second node cannot be bootstrapped") } return nil }) require.NoError(t, err) for i := 0; i < 5; i++ { b1.rt.UpdateEagerly() err = checkBackupOwnership(e2) require.NoError(t, err) } for partID, f := range fragments { result := f.Result() if len(result) == 0 { continue } require.Len(t, result, 1) require.NotNil(t, result[partitions.BACKUP]) r := result[partitions.BACKUP] require.NotNil(t, r[partID]) require.Equal(t, "test-data", r[partID].Name) require.Equal(t, []discovery.Member{b2.rt.This()}, r[partID].Owners) } } ================================================ FILE: internal/cluster/partitions/fragment.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package partitions import ( "github.com/olric-data/olric/internal/discovery" "github.com/olric-data/olric/pkg/storage" ) type Fragment interface { Name() string Stats() storage.Stats Move(*Partition, string, []discovery.Member) error Compaction() (bool, error) Destroy() error Close() error } ================================================ FILE: internal/cluster/partitions/hkey.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package partitions import ( "sync" "unsafe" "github.com/olric-data/olric/hasher" ) var ( hashFunc hasher.Hasher once sync.Once ) func SetHashFunc(h hasher.Hasher) { once.Do(func() { hashFunc = h }) } func HKey(name, key string) uint64 { tmp := name + key return hashFunc.Sum64(*(*[]byte)(unsafe.Pointer(&tmp))) } ================================================ FILE: internal/cluster/partitions/hkey_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package partitions import ( "testing" "github.com/olric-data/olric/hasher" "github.com/stretchr/testify/require" ) func TestPartitions_HKey(t *testing.T) { SetHashFunc(hasher.NewDefaultHasher()) hkey := HKey("storage-unit-name", "some-key") require.NotEqualf(t, 0, hkey, "HKey is zero. This shouldn't be normal") } ================================================ FILE: internal/cluster/partitions/partition.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package partitions import ( "sync" "sync/atomic" "github.com/olric-data/olric/internal/discovery" ) // Partition is a basic, logical storage unit in Olric and stores DMaps in a sync.Map. type Partition struct { sync.RWMutex id uint64 kind Kind m *sync.Map owners atomic.Value } func (p *Partition) Kind() Kind { return p.kind } func (p *Partition) ID() uint64 { return p.id } func (p *Partition) Map() *sync.Map { return p.m } // Owner returns partition Owner. It's not thread-safe. func (p *Partition) Owner() discovery.Member { if p.Kind() == BACKUP { // programming error. it cannot occur at production! panic("cannot call this if backup is true") } owners := p.owners.Load().([]discovery.Member) if len(owners) == 0 { panic("owners list cannot be empty") } return owners[len(owners)-1] } // OwnerCount returns the current Owner count of a partition. func (p *Partition) OwnerCount() int { owners := p.owners.Load() if owners == nil { return 0 } return len(owners.([]discovery.Member)) } // Owners loads the partition owners from atomic.Value and returns. func (p *Partition) Owners() []discovery.Member { owners := p.owners.Load() if owners == nil { return []discovery.Member{} } return owners.([]discovery.Member) } func (p *Partition) SetOwners(owners []discovery.Member) { p.owners.Store(owners) } func (p *Partition) Length() int { var length int p.Map().Range(func(_, tmp interface{}) bool { u := tmp.(Fragment) length += u.Stats().Length // Continue scanning. return true }) return length } ================================================ FILE: internal/cluster/partitions/partition_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package partitions import ( "sync" "testing" "github.com/olric-data/olric/internal/discovery" "github.com/olric-data/olric/pkg/storage" "github.com/stretchr/testify/require" ) type testFragment struct { length int } func (tf *testFragment) Stats() storage.Stats { return storage.Stats{Length: tf.length} } func (tf *testFragment) Name() string { return "test-data-structure" } func (tf *testFragment) Move(_ *Partition, _ string, _ []discovery.Member) error { return nil } func (tf *testFragment) Close() error { return nil } func (tf *testFragment) Destroy() error { return nil } func (tf *testFragment) Compaction() (bool, error) { return false, nil } func TestPartition(t *testing.T) { p := Partition{ id: 1, kind: PRIMARY, m: &sync.Map{}, } tmp := []discovery.Member{{ Name: "test-member", }} p.SetOwners(tmp) t.Run("Owners", func(t *testing.T) { owners := p.Owners() require.Equal(t, tmp, owners, "Partition owners slice is different") }) t.Run("Owner", func(t *testing.T) { owner := p.Owner() require.Equal(t, tmp[0], owner, "Partition owners slice is different") }) t.Run("OwnerCount", func(t *testing.T) { count := p.OwnerCount() require.Equal(t, 1, count) }) t.Run("Length", func(t *testing.T) { s1 := &testFragment{length: 10} s2 := &testFragment{length: 20} p.Map().Store("s1", s1) p.Map().Store("s2", s2) length := p.Length() require.Equal(t, 30, length) }) } ================================================ FILE: internal/cluster/partitions/partitions.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package partitions import ( "sync" "github.com/olric-data/olric/internal/discovery" ) type Kind int func (k Kind) String() string { switch { case k == PRIMARY: return "Primary" case k == BACKUP: return "Backup" default: return "Unknown" } } const ( PRIMARY = Kind(iota + 1) BACKUP ) type Partitions struct { count uint64 kind Kind m map[uint64]*Partition } func New(count uint64, kind Kind) *Partitions { ps := &Partitions{ kind: kind, count: count, m: make(map[uint64]*Partition), } for i := uint64(0); i < count; i++ { ps.m[i] = &Partition{ id: i, kind: kind, m: &sync.Map{}, } } return ps } // PartitionByID returns the partition for the given HKey func (ps *Partitions) PartitionByID(partID uint64) *Partition { return ps.m[partID] } // PartitionIDByHKey returns partition ID for a given HKey. func (ps *Partitions) PartitionIDByHKey(hkey uint64) uint64 { return hkey % ps.count } // PartitionByHKey returns the partition for the given HKey func (ps *Partitions) PartitionByHKey(hkey uint64) *Partition { partID := ps.PartitionIDByHKey(hkey) return ps.m[partID] } // PartitionOwnersByHKey loads the partition owners list for a given hkey. func (ps *Partitions) PartitionOwnersByHKey(hkey uint64) []discovery.Member { part := ps.PartitionByHKey(hkey) return part.owners.Load().([]discovery.Member) } // PartitionOwnersByID loads the partition owners list for a given hkey. func (ps *Partitions) PartitionOwnersByID(partID uint64) []discovery.Member { part := ps.PartitionByID(partID) return part.owners.Load().([]discovery.Member) } ================================================ FILE: internal/cluster/partitions/partitions_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package partitions import ( "reflect" "testing" "github.com/olric-data/olric/internal/discovery" ) func TestPartitions(t *testing.T) { var partitionCount uint64 = 271 ps := New(partitionCount, PRIMARY) t.Run("PartitionById", func(t *testing.T) { for partID := uint64(0); partID < partitionCount; partID++ { part := ps.PartitionByID(partID) if part.ID() != partID { t.Fatalf("Expected PartID: %d. Got: %d", partID, part.ID()) } if part.Kind() != PRIMARY { t.Fatalf("Expected Kind: %s. Got: %s", PRIMARY, part.Kind()) } } }) t.Run("PartitionIdByHKey", func(t *testing.T) { // 1 % 271 = 1 partID := ps.PartitionIDByHKey(1) if partID != 1 { t.Fatalf("Expected PartID: 1. Got: %d", partID) } }) t.Run("PartitionByHKey", func(t *testing.T) { // 1 % 271 = 1 part := ps.PartitionByHKey(1) if part.ID() != 1 { t.Fatalf("Expected PartID: 1. Got: %d", part.ID()) } }) t.Run("PartitionOwnersByHKey", func(t *testing.T) { part := ps.PartitionByHKey(1) tmp := []discovery.Member{{ Name: "test-member", }} part.SetOwners(tmp) owners := ps.PartitionOwnersByHKey(1) if !reflect.DeepEqual(owners, tmp) { t.Fatalf("Partition owners slice is different") } }) t.Run("PartitionOwnersById", func(t *testing.T) { part := ps.PartitionByID(1) tmp := []discovery.Member{{ Name: "test-member", }} part.SetOwners(tmp) owners := ps.PartitionOwnersByID(1) if !reflect.DeepEqual(owners, tmp) { t.Fatalf("Partition owners slice is different") } }) t.Run("Kind as string", func(t *testing.T) { // 1 % 271 = 1 part := ps.PartitionByHKey(1) if part.Kind().String() != PRIMARY.String() { t.Fatalf("Expected partition kind: %s. Got: %d", PRIMARY, part.Kind()) } }) } ================================================ FILE: internal/cluster/routingtable/callback.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package routingtable func (r *RoutingTable) AddCallback(f func()) { r.callbackMtx.Lock() defer r.callbackMtx.Unlock() r.callbacks = append(r.callbacks, f) } func (r *RoutingTable) runCallbacks() { defer r.wg.Done() r.callbackMtx.Lock() defer r.callbackMtx.Unlock() for _, f := range r.callbacks { select { case <-r.ctx.Done(): return default: } f() } } ================================================ FILE: internal/cluster/routingtable/callback_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package routingtable import ( "sync/atomic" "testing" "time" "github.com/olric-data/olric/internal/testutil" ) func TestRoutingTable_Callback(t *testing.T) { c := testutil.NewConfig() rt := newRoutingTableForTest(c, testutil.NewServer(c)) var num int32 increase := func() { atomic.AddInt32(&num, 1) } rt.AddCallback(increase) rt.wg.Add(1) go rt.runCallbacks() <-time.After(100 * time.Millisecond) modified := atomic.LoadInt32(&num) if modified != 1 { t.Fatalf("Expected number: 1. Got: %v", modified) } } ================================================ FILE: internal/cluster/routingtable/discovery.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package routingtable import ( "context" "errors" "time" ) var ( ErrServerGone = errors.New("server is gone") ErrNotJoinedYet = errors.New("not joined yet") ErrClusterJoin = errors.New("cannot join the cluster") // ErrOperationTimeout is returned when an operation times out. ErrOperationTimeout = errors.New("operation timeout") ) // bootstrapCoordinator prepares the very first routing table and bootstraps the coordinator node. func (r *RoutingTable) bootstrapCoordinator() error { r.Lock() defer r.Unlock() r.fillRoutingTable() _, err := r.updateRoutingTableOnCluster() if err != nil { return err } // The coordinator bootstraps itself. r.markBootstrapped() r.log.V(2).Printf("[INFO] The cluster coordinator has been bootstrapped") return nil } func (r *RoutingTable) attemptToJoin() error { attempts := 0 for attempts < r.config.MaxJoinAttempts { select { case <-r.ctx.Done(): // The node is gone. return ErrServerGone default: } attempts++ n, err := r.discovery.Join() if err == nil { r.log.V(2).Printf("[INFO] Join completed. Synced with %d initial nodes", n) return nil } r.log.V(2).Printf("[ERROR] Join attempt returned error: %s", err) if r.IsBootstrapped() { r.log.V(2).Printf("[INFO] Bootstrapped by the cluster coordinator") return nil } r.log.V(2).Printf("[INFO] Awaits for %s to join again (%d/%d)", r.config.JoinRetryInterval, attempts, r.config.MaxJoinAttempts) <-time.After(r.config.JoinRetryInterval) } return ErrClusterJoin } func (r *RoutingTable) tryWithInterval(ctx context.Context, interval time.Duration, f func() error) error { ticker := time.NewTicker(interval) defer ticker.Stop() var funcErr error funcErr = f() if funcErr == nil { // Done. No need to try with interval return nil } loop: for { select { case <-ctx.Done(): // context is done err := ctx.Err() if errors.Is(err, context.DeadlineExceeded) { break loop } if errors.Is(err, context.Canceled) { return ErrServerGone } return err case <-ticker.C: funcErr = f() if funcErr == nil { break loop } } } return funcErr } ================================================ FILE: internal/cluster/routingtable/discovery_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package routingtable import ( "context" "errors" "testing" "time" "github.com/olric-data/olric/internal/testutil" ) func TestRoutingTable_tryWithInterval(t *testing.T) { c := testutil.NewConfig() srv := testutil.NewServer(c) rt := newRoutingTableForTest(c, srv) var foobarError = errors.New("foobar") ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) defer cancel() err := rt.tryWithInterval(ctx, time.Millisecond, func() error { return foobarError }) if err != foobarError { t.Fatalf("Expected foobarError. Got: %v", foobarError) } } func TestRoutingTable_attemptToJoin(t *testing.T) { c := testutil.NewConfig() c.MaxJoinAttempts = 3 c.JoinRetryInterval = 100 * time.Millisecond c.Peers = []string{"127.0.0.1:0"} // An invalid peer srv := testutil.NewServer(c) rt := newRoutingTableForTest(c, srv) err := rt.discovery.Start() if err != nil { t.Fatalf("Expected nil. Got: %v", err) } defer func() { err = rt.discovery.Shutdown() if err != nil { t.Fatalf("Expected nil. Got: %v", err) } }() err = rt.attemptToJoin() if err != ErrClusterJoin { t.Fatalf("Expected ErrClusterJoin. Got: %v", err) } } ================================================ FILE: internal/cluster/routingtable/distribute.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package routingtable import ( "errors" "github.com/buraksezer/consistent" "github.com/olric-data/olric/internal/discovery" "github.com/olric-data/olric/internal/protocol" ) func (r *RoutingTable) distributePrimaryCopies(partID uint64) []discovery.Member { // First you need to create a copy of the owners list. Don't modify the current list. part := r.primary.PartitionByID(partID) owners := make([]discovery.Member, part.OwnerCount()) copy(owners, part.Owners()) // Find the new partition owner. newOwner := r.consistent.GetPartitionOwner(int(partID)) // First run. if len(owners) == 0 { owners = append(owners, newOwner.(discovery.Member)) return owners } // Prune dead nodes for i := 0; i < len(owners); i++ { owner := owners[i] current, err := r.discovery.FindMemberByName(owner.Name) if err != nil { r.log.V(6).Printf("[DEBUG] Failed to find %s in the cluster: %v", owner, err) owners = append(owners[:i], owners[i+1:]...) i-- r.log.V(3).Printf("[INFO] Member: %s has been deleted from the primary owners list of PartID: %v", owner.String(), partID) continue } if !owner.CompareByID(current) { r.log.V(3).Printf("[WARN] One of the partitions owners is probably re-joined: %s", current) owners = append(owners[:i], owners[i+1:]...) i-- continue } } // Prune empty nodes for i := 0; i < len(owners); i++ { owner := owners[i] cmd := protocol.NewLengthOfPart(partID).Command(r.ctx) rc := r.client.Get(owner.String()) err := rc.Process(r.ctx, cmd) if err != nil { r.log.V(6).Printf("[DEBUG] Failed to check key count on backup "+ "partition: %d: %v", partID, err) // Pass it. If the node is down, memberlist package will send a leave event. continue } count, err := cmd.Result() if err != nil { r.log.V(6).Printf("[DEBUG] Failed to check key count on backup "+ "partition: %d: %v", partID, err) // Pass it. If the node is down, memberlist package will send a leave event. continue } if count == 0 { // Empty partition. Delete it from ownership list. owners = append(owners[:i], owners[i+1:]...) i-- } } // Here add the new partition newOwner. for i, owner := range owners { if owner.CompareByID(newOwner.(discovery.Member)) { // Remove it from the current position owners = append(owners[:i], owners[i+1:]...) // Append it again to head return append(owners, newOwner.(discovery.Member)) } } return append(owners, newOwner.(discovery.Member)) } func (r *RoutingTable) getReplicaOwners(partID uint64) ([]consistent.Member, error) { for i := r.config.ReplicaCount; i > 0; i-- { newOwners, err := r.consistent.GetClosestNForPartition(int(partID), i) if errors.Is(err, consistent.ErrInsufficientMemberCount) { continue } if err != nil { // Fail early return nil, err } return newOwners, nil } return nil, consistent.ErrInsufficientMemberCount } func isOwner(member discovery.Member, owners []consistent.Member) bool { for _, owner := range owners { if member.Name == owner.String() { return true } } return false } func (r *RoutingTable) distributeBackups(partID uint64) []discovery.Member { part := r.backup.PartitionByID(partID) owners := make([]discovery.Member, part.OwnerCount()) copy(owners, part.Owners()) newOwners, err := r.getReplicaOwners(partID) if err != nil { r.log.V(3).Printf("[ERROR] Failed to get replica owners for PartID: %d: %v", partID, err) return nil } // Remove the primary owner newOwners = newOwners[1:] // First run if len(owners) == 0 { for _, owner := range newOwners { owners = append(owners, owner.(discovery.Member)) } return owners } // Prune dead nodes for i := 0; i < len(owners); i++ { backup := owners[i] cur, err := r.discovery.FindMemberByName(backup.Name) if err != nil { r.log.V(6).Printf("[DEBUG] Failed to find %s in the cluster: %v", backup, err) // Delete it. owners = append(owners[:i], owners[i+1:]...) i-- r.log.V(6).Printf("[INFO] Member: %s has been deleted from the backup owners list of PartID: %v", backup.String(), partID) continue } if !backup.CompareByID(cur) { r.log.V(3).Printf("[WARN] One of the backup owners is probably re-joined: %s", cur) // Delete it. owners = append(owners[:i], owners[i+1:]...) i-- continue } } // Prune empty nodes for i := 0; i < len(owners); i++ { backup := owners[i] cmd := protocol.NewLengthOfPart(partID).SetReplica().Command(r.ctx) rc := r.client.Get(backup.String()) err := rc.Process(r.ctx, cmd) if err != nil { r.log.V(6).Printf("[DEBUG] Failed to check key count on backup "+ "partition: %d: %v", partID, err) // Pass it. If the node is down, memberlist package will send a leave event. continue } count, err := cmd.Result() if err != nil { r.log.V(6).Printf("[DEBUG] Failed to check key count on backup "+ "partition: %d: %v", partID, err) // Pass it. If the node is down, memberlist package will send a leave event. continue } if count != 0 { // About this scenario: // // * ReplicaCount = 3 // * Create three nodes and insert some keys // * Kill one of the nodes // * Now we have replicas that it's impossible to transfer its ownership // * Since we cannot drop a healthy replica, we prefer to keep it until // a new node joined. Then, we transfer the ownership safely. // * During this incident, a node owns a primary and backup replicas at the same time. if !isOwner(backup, newOwners) { r.log.V(3).Printf("[WARN] %s hosts primary and replica copies "+ "for PartID: %d", backup, partID) } continue } // Empty node, delete it. owners = append(owners[:i], owners[i+1:]...) i-- } // Here add the new backup owners. for _, newOwner := range newOwners { var exists bool for i, owner := range owners { if owner.CompareByID(newOwner.(discovery.Member)) { exists = true // Remove it from the current position owners = append(owners[:i], owners[i+1:]...) // Append it again to head owners = append(owners, newOwner.(discovery.Member)) break } } if !exists { owners = append(owners, newOwner.(discovery.Member)) } } return owners } ================================================ FILE: internal/cluster/routingtable/distribute_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package routingtable import ( "context" "errors" "testing" "time" "github.com/olric-data/olric/internal/testutil" ) func TestRoutingTable_distributedBackups(t *testing.T) { cluster := newTestCluster() defer cluster.cancel() c1 := testutil.NewConfig() c1.ReplicaCount = 2 rt1, err := cluster.addNode(c1) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } if !rt1.IsBootstrapped() { t.Fatalf("The coordinator node cannot be bootstrapped") } c2 := testutil.NewConfig() c2.ReplicaCount = 2 rt2, err := cluster.addNode(c2) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } err = testutil.TryWithInterval(10, 100*time.Millisecond, func() error { if !rt2.IsBootstrapped() { return errors.New("the second node cannot be bootstrapped") } return nil }) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } err = rt1.Shutdown(context.Background()) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } c3 := testutil.NewConfig() c3.ReplicaCount = 2 rt3, err := cluster.addNode(c3) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } rt2.UpdateEagerly() for partID := uint64(0); partID < c3.PartitionCount; partID++ { part := rt3.backup.PartitionByID(partID) if part.OwnerCount() != 1 { t.Fatalf("Expected backup owners count: 1. Got: %d", part.OwnerCount()) } for _, owner := range part.Owners() { if owner.CompareByID(rt1.This()) { t.Fatalf("Dead node still a replica owner: %v", rt1.This()) } } } err = cluster.shutdown() if err != nil { t.Fatalf("Expected nil. Got: %v", err) } } ================================================ FILE: internal/cluster/routingtable/events.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package routingtable import ( "time" "github.com/olric-data/olric/events" "github.com/olric-data/olric/internal/discovery" ) func (r *RoutingTable) publishNodeJoinEvent(m *discovery.Member) { defer r.wg.Done() rc := r.client.Get(r.this.String()) message := events.NodeJoinEvent{ Kind: events.KindNodeJoinEvent, Source: r.this.String(), NodeJoin: m.String(), Timestamp: time.Now().UnixNano(), } data, err := message.Encode() if err != nil { r.log.V(3).Printf("[ERROR] Failed to encode NodeJoinEvent: %v", err) return } err = rc.Publish(r.ctx, events.ClusterEventsChannel, data).Err() if err != nil { r.log.V(3).Printf("[ERROR] Failed to publish NodeJoinEvent to %s: %v", events.ClusterEventsChannel, err) } } func (r *RoutingTable) publishNodeLeftEvent(m *discovery.Member) { defer r.wg.Done() rc := r.client.Get(r.this.String()) message := events.NodeLeftEvent{ Kind: events.KindNodeLeftEvent, Source: r.this.String(), NodeLeft: m.String(), Timestamp: time.Now().UnixNano(), } data, err := message.Encode() if err != nil { r.log.V(3).Printf("[ERROR] Failed to encode NodeLeftEvent: %v", err) return } err = rc.Publish(r.ctx, events.ClusterEventsChannel, data).Err() if err != nil { r.log.V(3).Printf("[ERROR] Failed to publish NodeLeftEvent to %s: %v", events.ClusterEventsChannel, err) } } ================================================ FILE: internal/cluster/routingtable/events_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package routingtable import ( "context" "encoding/json" "testing" "time" "github.com/olric-data/olric/events" "github.com/olric-data/olric/internal/discovery" "github.com/olric-data/olric/internal/protocol" "github.com/olric-data/olric/internal/testutil" "github.com/stretchr/testify/require" "github.com/tidwall/redcon" ) func TestRoutingTable_publishNodeJoinEvent(t *testing.T) { cluster := newTestCluster() defer cluster.cancel() c := testutil.NewConfig() rt, err := cluster.addNode(c) require.NoError(t, err) ctx, cancel := context.WithTimeout(context.Background(), time.Second) rt.server.ServeMux().HandleFunc(protocol.PubSub.Publish, func(conn redcon.Conn, cmd redcon.Command) { defer cancel() publishCmd, err := protocol.ParsePublishCommand(cmd) require.NoError(t, err) require.Equal(t, events.ClusterEventsChannel, publishCmd.Channel) v := events.NodeJoinEvent{} err = json.Unmarshal([]byte(publishCmd.Message), &v) require.NoError(t, err) require.Equal(t, events.KindNodeJoinEvent, v.Kind) require.Equal(t, rt.this.String(), v.Source) require.Equal(t, rt.this.String(), v.NodeJoin) conn.WriteInt(1) }) m := discovery.NewMember(c) rt.wg.Add(1) go rt.publishNodeJoinEvent(&m) <-ctx.Done() require.ErrorIs(t, context.Canceled, ctx.Err()) } func TestRoutingTable_publishNodeLeftEvent(t *testing.T) { cluster := newTestCluster() defer cluster.cancel() c := testutil.NewConfig() rt, err := cluster.addNode(c) require.NoError(t, err) ctx, cancel := context.WithTimeout(context.Background(), time.Second) rt.server.ServeMux().HandleFunc(protocol.PubSub.Publish, func(conn redcon.Conn, cmd redcon.Command) { defer cancel() publishCmd, err := protocol.ParsePublishCommand(cmd) require.NoError(t, err) require.Equal(t, events.ClusterEventsChannel, publishCmd.Channel) v := events.NodeLeftEvent{} err = json.Unmarshal([]byte(publishCmd.Message), &v) require.NoError(t, err) require.Equal(t, events.KindNodeLeftEvent, v.Kind) require.Equal(t, rt.this.String(), v.Source) require.Equal(t, rt.this.String(), v.NodeLeft) conn.WriteInt(1) }) m := discovery.NewMember(c) rt.wg.Add(1) go rt.publishNodeLeftEvent(&m) <-ctx.Done() require.ErrorIs(t, context.Canceled, ctx.Err()) } ================================================ FILE: internal/cluster/routingtable/handlers.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package routingtable import ( "github.com/olric-data/olric/internal/protocol" ) func (r *RoutingTable) RegisterHandlers() { r.server.ServeMux().HandleFunc(protocol.Internal.UpdateRouting, r.updateRoutingCommandHandler) r.server.ServeMux().HandleFunc(protocol.Internal.LengthOfPart, r.lengthOfPartCommandHandler) } ================================================ FILE: internal/cluster/routingtable/left_over_data.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package routingtable import ( "github.com/olric-data/olric/internal/cluster/partitions" "github.com/olric-data/olric/internal/discovery" ) func (r *RoutingTable) processLeftOverDataReports(reports map[discovery.Member]*leftOverDataReport) { check := func(member discovery.Member, owners []discovery.Member) bool { for _, owner := range owners { if member.CompareByID(owner) { return true } } return false } ensureOwnership := func(member discovery.Member, partID uint64, part *partitions.Partition) { owners := part.Owners() if check(member, owners) { return } // This section is protected by routingMtx against parallel writers. // // Copy owners and append the member to head newOwners := make([]discovery.Member, len(owners)) copy(newOwners, owners) // Prepend newOwners = append([]discovery.Member{member}, newOwners...) part.SetOwners(newOwners) r.log.V(2).Printf("[INFO] %s still have some data for PartID (kind: %s): %d", member, part.Kind(), partID) } // data structures in this function is guarded by routingMtx for member, report := range reports { for _, partID := range report.Partitions { part := r.primary.PartitionByID(partID) ensureOwnership(member, partID, part) } for _, partID := range report.Backups { part := r.backup.PartitionByID(partID) ensureOwnership(member, partID, part) } } } ================================================ FILE: internal/cluster/routingtable/left_over_data_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package routingtable import ( "errors" "testing" "time" "github.com/olric-data/olric/internal/testutil" "github.com/olric-data/olric/internal/testutil/mockfragment" "github.com/stretchr/testify/require" ) func TestRoutingTable_LeftOverData(t *testing.T) { cluster := newTestCluster() defer cluster.cancel() c1 := testutil.NewConfig() rt1, err := cluster.addNode(c1) require.NoError(t, err) if !rt1.IsBootstrapped() { t.Fatalf("The coordinator node cannot be bootstrapped") } for partID := uint64(0); partID < c1.PartitionCount; partID++ { part := rt1.primary.PartitionByID(partID) ts := mockfragment.New() ts.Fill() part.Map().Store("test-data", ts) } c2 := testutil.NewConfig() rt2, err := cluster.addNode(c2) require.NoError(t, err) err = testutil.TryWithInterval(10, 100*time.Millisecond, func() error { if !rt2.IsBootstrapped() { return errors.New("the second node cannot be bootstrapped") } return nil }) require.NoError(t, err) for partID := uint64(0); partID < c2.PartitionCount; partID++ { part := rt2.primary.PartitionByID(partID) ts := mockfragment.New() ts.Fill() part.Map().Store("test-data", ts) } rt1.UpdateEagerly() for partID := uint64(0); partID < c1.PartitionCount; partID++ { part := rt1.primary.PartitionByID(partID) if len(part.Owners()) != 2 { t.Fatalf("Expected partition owners count: 2. Got: %d, PartID: %d", part.OwnerCount(), partID) } } } ================================================ FILE: internal/cluster/routingtable/members.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package routingtable import ( "fmt" "sync" "github.com/olric-data/olric/internal/discovery" ) type Members struct { sync.RWMutex m map[uint64]discovery.Member } func newMembers() *Members { return &Members{ m: map[uint64]discovery.Member{}, } } func (m *Members) Add(member discovery.Member) { m.m[member.ID] = member } func (m *Members) Get(id uint64) (discovery.Member, error) { member, ok := m.m[id] if !ok { return discovery.Member{}, fmt.Errorf("member not found with id: %d", id) } return member, nil } func (m *Members) Delete(id uint64) { delete(m.m, id) } func (m *Members) DeleteByName(other discovery.Member) { for id, member := range m.m { if member.CompareByName(other) { delete(m.m, id) } } } func (m *Members) Length() int { return len(m.m) } func (m *Members) Range(f func(id uint64, member discovery.Member) bool) { for id, member := range m.m { if !f(id, member) { break } } } ================================================ FILE: internal/cluster/routingtable/members_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package routingtable import ( "reflect" "testing" "github.com/olric-data/olric/internal/discovery" ) func TestMembers_Get(t *testing.T) { m := newMembers() member := discovery.Member{ Name: "localhost:3320", ID: 6054057, } m.Add(member) r, err := m.Get(member.ID) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } if !reflect.DeepEqual(r, member) { t.Fatalf("Retrived member is different") } } func TestMembers_Delete(t *testing.T) { m := newMembers() member := discovery.Member{ Name: "localhost:3320", ID: 6054057, } m.Add(member) m.Delete(member.ID) _, err := m.Get(member.ID) if err == nil { t.Fatalf("Expected and error. Got: %v", err) } } func TestMembers_DeleteByName(t *testing.T) { m := newMembers() member := discovery.Member{ Name: "localhost:3320", ID: 6054057, } m.Add(member) m.DeleteByName(member) _, err := m.Get(member.ID) if err == nil { t.Fatalf("Expected and error. Got: %v", err) } } func TestMembers_Length(t *testing.T) { m := newMembers() member := discovery.Member{ Name: "localhost:3320", ID: 6054057, } m.Add(member) if m.Length() != 1 { t.Fatalf("Expected length: 1. Got: %d", m.Length()) } } func TestMembers_Range(t *testing.T) { m := newMembers() member := discovery.Member{ Name: "localhost:3320", ID: 6054057, } m.Add(member) m.Range(func(id uint64, m discovery.Member) bool { if id != member.ID { t.Fatalf("Expected id: %d. Got: %d", member.ID, id) } if member.Name != m.Name { t.Fatalf("Expected Name: %s. Got: %s", member.Name, m.Name) } return true }) } ================================================ FILE: internal/cluster/routingtable/operations.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package routingtable import ( "fmt" "github.com/cespare/xxhash/v2" "github.com/olric-data/olric/internal/cluster/partitions" "github.com/olric-data/olric/internal/protocol" "github.com/tidwall/redcon" "github.com/vmihailenco/msgpack/v5" ) func (r *RoutingTable) lengthOfPartCommandHandler(conn redcon.Conn, cmd redcon.Command) { // The command handlers of the routing table service should wait for the cluster join event. <-r.joined lengthOfPartCmd, err := protocol.ParseLengthOfPartCommand(cmd) if err != nil { protocol.WriteError(conn, err) return } var part *partitions.Partition if lengthOfPartCmd.Replica { part = r.backup.PartitionByID(lengthOfPartCmd.PartID) } else { part = r.primary.PartitionByID(lengthOfPartCmd.PartID) } conn.WriteInt(part.Length()) } func (r *RoutingTable) verifyRoutingTable(id uint64, table map[uint64]*route) error { // Check the coordinator coordinator, err := r.discovery.FindMemberByID(id) if err != nil { return err } myCoordinator := r.discovery.GetCoordinator() if !coordinator.CompareByID(myCoordinator) { return fmt.Errorf("unrecognized cluster coordinator: %s: %s", coordinator, myCoordinator) } // Compare partition counts to catch a possible inconsistencies in configuration if r.config.PartitionCount != uint64(len(table)) { return fmt.Errorf("invalid partition count: %d", len(table)) } return nil } func (r *RoutingTable) updateRoutingCommandHandler(conn redcon.Conn, cmd redcon.Command) { // The command handlers of the routing table service should wait for the cluster join event. <-r.joined r.updateRoutingMtx.Lock() defer r.updateRoutingMtx.Unlock() updateRoutingCmd, err := protocol.ParseUpdateRoutingCommand(cmd) if err != nil { protocol.WriteError(conn, err) return } table := make(map[uint64]*route) err = msgpack.Unmarshal(updateRoutingCmd.Payload, &table) if err != nil { protocol.WriteError(conn, err) return } // Log this event coordinator, err := r.discovery.FindMemberByID(updateRoutingCmd.CoordinatorID) if err != nil { protocol.WriteError(conn, err) return } r.log.V(3).Printf("[INFO] Routing table has been pushed by %s", coordinator) if err = r.verifyRoutingTable(updateRoutingCmd.CoordinatorID, table); err != nil { protocol.WriteError(conn, err) return } // owners(atomic.value) is guarded by routingUpdateMtx against parallel writers. // Calculate routing signature. This is useful to control balancing tasks. r.setSignature(xxhash.Sum64(updateRoutingCmd.Payload)) for partID, data := range table { // Set partition(primary copies) owners part := r.primary.PartitionByID(partID) part.SetOwners(data.Owners) // Set backup owners bpart := r.backup.PartitionByID(partID) bpart.SetOwners(data.Backups) } // Used by the LRU implementation. r.setOwnedPartitionCount() // Bootstrapped by the coordinator. r.markBootstrapped() // Collect report value, err := r.prepareLeftOverDataReport() if err != nil { protocol.WriteError(conn, err) return } // Call balancer to distribute load evenly r.wg.Add(1) go r.runCallbacks() conn.WriteBulk(value) } ================================================ FILE: internal/cluster/routingtable/routingtable.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package routingtable import ( "context" "errors" "sync" "sync/atomic" "time" "github.com/olric-data/olric/internal/protocol" "github.com/buraksezer/consistent" "github.com/hashicorp/memberlist" "github.com/olric-data/olric/config" "github.com/olric-data/olric/internal/checkpoint" "github.com/olric-data/olric/internal/cluster/partitions" "github.com/olric-data/olric/internal/discovery" "github.com/olric-data/olric/internal/environment" "github.com/olric-data/olric/internal/server" "github.com/olric-data/olric/internal/service" "github.com/olric-data/olric/pkg/flog" ) // ErrClusterQuorum means that the cluster could not reach a healthy numbers of members to operate. var ErrClusterQuorum = errors.New("cannot be reached cluster quorum to operate") type route struct { Owners []discovery.Member Backups []discovery.Member } type RoutingTable struct { sync.RWMutex // routingMtx // Currently owned partition count. Approximate LRU implementation // uses that. ownedPartitionCount uint64 signature uint64 // numMembers is used to check cluster quorum. numMembers int32 // These values is useful to control operation status. bootstrapped int32 updateRoutingMtx sync.Mutex table map[uint64]*route consistent *consistent.Consistent this discovery.Member members *Members config *config.Config log *flog.Logger primary *partitions.Partitions backup *partitions.Partitions client *server.Client server *server.Server discovery *discovery.Discovery callbacks []func() callbackMtx sync.Mutex pushPeriod time.Duration // The command handlers of the routing table service should wait for the cluster join event. joined chan struct{} ctx context.Context cancel context.CancelFunc wg sync.WaitGroup } func registerErrors() { protocol.SetError("CLUSTERQUORUM", ErrClusterQuorum) protocol.SetError("CLUSTERJOIN", ErrClusterJoin) protocol.SetError("SERVERGONE", ErrServerGone) protocol.SetError("OPERATIONTIMEOUT", ErrOperationTimeout) } func New(e *environment.Environment) *RoutingTable { // The routing table has to be started properly before accepting connections. checkpoint.Add() c := e.Get("config").(*config.Config) log := e.Get("logger").(*flog.Logger) ctx, cancel := context.WithCancel(context.Background()) cc := consistent.Config{ Hasher: c.Hasher, PartitionCount: int(c.PartitionCount), ReplicationFactor: 20, // TODO: This also may be a configuration param. Load: c.LoadFactor, } rt := &RoutingTable{ members: newMembers(), discovery: discovery.New(log, c), config: c, log: log, consistent: consistent.New(nil, cc), primary: e.Get("primary").(*partitions.Partitions), backup: e.Get("backup").(*partitions.Partitions), client: e.Get("client").(*server.Client), server: e.Get("server").(*server.Server), pushPeriod: c.RoutingTablePushInterval, joined: make(chan struct{}), ctx: ctx, cancel: cancel, } registerErrors() rt.RegisterHandlers() return rt } func (r *RoutingTable) Discovery() *discovery.Discovery { return r.discovery } func (r *RoutingTable) This() discovery.Member { return r.this } // setNumMembers assigns the current number of members in the cluster to a variable. func (r *RoutingTable) setNumMembers() { // Calling NumMembers in every request is quite expensive. // It's rarely updated. Just call this when the membership info changed. nr := int32(r.discovery.NumMembers()) atomic.StoreInt32(&r.numMembers, nr) } func (r *RoutingTable) SetNumMembersEagerly(nr int32) { atomic.StoreInt32(&r.numMembers, nr) } func (r *RoutingTable) NumMembers() int32 { return atomic.LoadInt32(&r.numMembers) } func (r *RoutingTable) Members() *Members { return r.members } func (r *RoutingTable) setSignature(s uint64) { atomic.StoreUint64(&r.signature, s) } func (r *RoutingTable) Signature() uint64 { return atomic.LoadUint64(&r.signature) } func (r *RoutingTable) setOwnedPartitionCount() { var count uint64 for partID := uint64(0); partID < r.config.PartitionCount; partID++ { part := r.primary.PartitionByID(partID) if part.Owner().CompareByID(r.this) { count++ } } atomic.StoreUint64(&r.ownedPartitionCount, count) } func (r *RoutingTable) OwnedPartitionCount() uint64 { return atomic.LoadUint64(&r.ownedPartitionCount) } func (r *RoutingTable) CheckMemberCountQuorum() error { // This type of quorum function determines the presence of quorum based on the count of members in the cluster, // as observed by the local member’s cluster membership manager if r.config.MemberCountQuorum > r.NumMembers() { return ErrClusterQuorum } return nil } func (r *RoutingTable) markBootstrapped() { // Bootstrapped by the coordinator. atomic.StoreInt32(&r.bootstrapped, 1) } func (r *RoutingTable) IsBootstrapped() bool { // Bootstrapped by the coordinator. return atomic.LoadInt32(&r.bootstrapped) == 1 } // CheckBootstrap is called for every request and checks whether the node is bootstrapped. // It has to be very fast for a smooth operation. func (r *RoutingTable) CheckBootstrap() error { // Prevent creating expensive structures for every request, // Just check an integer value atomically. if r.IsBootstrapped() { return nil } ctx, cancel := context.WithTimeout(context.Background(), r.config.BootstrapTimeout) defer cancel() return r.tryWithInterval(ctx, 100*time.Millisecond, func() error { if r.IsBootstrapped() { return nil } // Final error return ErrOperationTimeout }) } func (r *RoutingTable) fillRoutingTable() { if r.config.ReplicaCount > int(r.NumMembers()) { r.log.V(1).Printf("[WARN] Desired replica count is %d and "+ "the cluster has %d members currently", r.config.ReplicaCount, r.NumMembers()) } table := make(map[uint64]*route) for partID := uint64(0); partID < r.config.PartitionCount; partID++ { rt := &route{ Owners: r.distributePrimaryCopies(partID), } if r.config.ReplicaCount > config.MinimumReplicaCount { rt.Backups = r.distributeBackups(partID) } table[partID] = rt } r.table = table } func (r *RoutingTable) UpdateEagerly() { r.updateRouting() } func (r *RoutingTable) updateRouting() { // This function is called by listenMemberlistEvents and updateRoutingPeriodically // So this lock prevents parallel execution. r.Lock() defer r.Unlock() // This function is only run by the cluster coordinator. if !r.discovery.IsCoordinator() { return } // This type of quorum function determines the presence of quorum based on the count of members in the cluster, // as observed by the local member’s cluster membership manager if err := r.CheckMemberCountQuorum(); err != nil { r.log.V(2).Printf("[ERROR] Impossible to calculate and update routing table: %v", err) return } r.fillRoutingTable() reports, err := r.updateRoutingTableOnCluster() if err != nil { r.log.V(2).Printf("[ERROR] Failed to update routing table on cluster: %v", err) return } r.processLeftOverDataReports(reports) } func (r *RoutingTable) processClusterEvent(event *discovery.ClusterEvent) { r.Members().Lock() defer r.Members().Unlock() member, _ := discovery.NewMemberFromMetadata(event.NodeMeta) switch event.Event { case memberlist.NodeJoin: r.Members().Add(member) r.consistent.Add(member) r.log.V(2).Printf("[INFO] Node joined: %s", member) if r.config.EnableClusterEventsChannel { r.wg.Add(1) go r.publishNodeJoinEvent(&member) } case memberlist.NodeLeave: if _, err := r.Members().Get(member.ID); err != nil { r.log.V(2).Printf("[ERROR] Unknown node left: %s: %d", event.NodeName, member.ID) return } r.Members().Delete(member.ID) r.consistent.Remove(event.NodeName) // Don't try to used closed sockets again. r.log.V(2).Printf("[INFO] Node left: %s", event.NodeName) if err := r.client.Close(event.NodeName); err != nil { r.log.V(2).Printf("[ERROR] Failed to remove the node from pool %s: %v", event.NodeName, err) } if r.config.EnableClusterEventsChannel { r.wg.Add(1) go r.publishNodeLeftEvent(&member) } case memberlist.NodeUpdate: // Node's birthdate may be changed. Close the pool and re-add to the hash ring. // This takes linear time, but member count should be too small for a decent computer! r.Members().Range(func(id uint64, item discovery.Member) bool { if member.CompareByName(item) { r.Members().Delete(id) r.consistent.Remove(event.NodeName) if err := r.client.Close(event.NodeName); err != nil { r.log.V(2).Printf("[ERROR] Failed to remove the node from pool %s: %v", event.NodeName, err) } } return true }) r.Members().Add(member) r.consistent.Add(member) r.log.V(2).Printf("[INFO] Node updated: %s", member) default: r.log.V(2).Printf("[ERROR] Unknown event received: %v", event) return } // Store the current number of members in the member list. // We need this to implement a simple split-brain protection algorithm. r.setNumMembers() } func (r *RoutingTable) listenClusterEvents(eventCh chan *discovery.ClusterEvent) { defer r.wg.Done() for { select { case <-r.ctx.Done(): return case e := <-eventCh: r.processClusterEvent(e) r.updateRouting() } } } func (r *RoutingTable) pushPeriodically() { defer r.wg.Done() ticker := time.NewTicker(r.pushPeriod) defer ticker.Stop() for { select { case <-r.ctx.Done(): return case <-ticker.C: r.updateRouting() } } } func (r *RoutingTable) Join() error { err := r.discovery.Start() if err != nil { return err } err = r.attemptToJoin() if errors.Is(err, ErrClusterJoin) { r.log.V(1).Printf("[INFO] Forming a new Olric cluster") err = nil } if err != nil { return err } this, err := r.discovery.FindMemberByName(r.config.MemberlistConfig.Name) if err != nil { r.log.V(2).Printf("[ERROR] Failed to get this node in cluster: %v", err) shutdownError := r.discovery.Shutdown() if shutdownError != nil { return shutdownError } return err } r.this = this close(r.joined) return nil } func (r *RoutingTable) Start() error { select { case <-r.joined: // It's time to start the routing table service. Otherwise, this method will return an error. default: // Not yet, or the join process has failed return ErrNotJoinedYet } // Store the current number of members in the member list. // We need this to implement a simple split-brain protection algorithm. r.setNumMembers() r.wg.Add(1) go r.listenClusterEvents(r.discovery.ClusterEvents) // 1 Hour ctx, cancel := context.WithTimeout(r.ctx, time.Hour) defer cancel() err := r.tryWithInterval(ctx, time.Second, func() error { // Check member count quorum now. If there are not enough peers to work, wait forever. err := r.CheckMemberCountQuorum() if err != nil { r.log.V(2).Printf("[ERROR] Inoperable node: %v", err) } return nil }) if err != nil { return err } r.Members().Lock() r.Members().Add(r.this) r.Members().Unlock() r.consistent.Add(r.this) if r.discovery.IsCoordinator() { err = r.bootstrapCoordinator() if err != nil { return err } } r.wg.Add(1) go r.pushPeriodically() if r.config.MemberlistInterface != "" { r.log.V(2).Printf("[INFO] Memberlist uses interface: %s", r.config.MemberlistInterface) } r.log.V(2).Printf("[INFO] Memberlist bindAddr: %s, bindPort: %d", r.config.MemberlistConfig.BindAddr, r.config.MemberlistConfig.BindPort) r.log.V(2).Printf("[INFO] Cluster coordinator: %s", r.discovery.GetCoordinator()) checkpoint.Pass() return nil } func (r *RoutingTable) Shutdown(ctx context.Context) error { select { case <-r.ctx.Done(): // already closed return nil default: } if err := r.discovery.Shutdown(); err != nil { return err } r.cancel() done := make(chan struct{}) go func() { r.wg.Wait() close(done) }() select { case <-ctx.Done(): err := ctx.Err() if err != nil { return err } case <-done: } return nil } var _ service.Service = (*RoutingTable)(nil) ================================================ FILE: internal/cluster/routingtable/routingtable_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package routingtable import ( "context" "errors" "fmt" "net" "strconv" "testing" "time" "github.com/hashicorp/memberlist" "github.com/olric-data/olric/config" "github.com/olric-data/olric/internal/cluster/partitions" "github.com/olric-data/olric/internal/discovery" "github.com/olric-data/olric/internal/environment" "github.com/olric-data/olric/internal/server" "github.com/olric-data/olric/internal/testutil" "golang.org/x/sync/errgroup" ) func newRoutingTableForTest(c *config.Config, srv *server.Server) *RoutingTable { e := environment.New() e.Set("config", c) e.Set("logger", testutil.NewFlogger(c)) e.Set("primary", partitions.New(c.PartitionCount, partitions.PRIMARY)) e.Set("backup", partitions.New(c.PartitionCount, partitions.BACKUP)) e.Set("client", server.NewClient(c.Client)) e.Set("server", srv) rt := New(e) go func() { err := srv.ListenAndServe() if err != nil { panic(fmt.Sprintf("ListenAndServe returned an error: %v", err)) } }() <-srv.StartedCtx.Done() return rt } type testCluster struct { peerPorts []int errGr errgroup.Group ctx context.Context cancel context.CancelFunc } func newTestCluster() *testCluster { ctx, cancel := context.WithCancel(context.Background()) return &testCluster{ ctx: ctx, cancel: cancel, } } func (t *testCluster) addNode(c *config.Config) (*RoutingTable, error) { if c == nil { c = testutil.NewConfig() } port, err := testutil.GetFreePort() if err != nil { return nil, err } c.MemberlistConfig.BindPort = port var peers []string for _, peerPort := range t.peerPorts { peers = append(peers, net.JoinHostPort("127.0.0.1", strconv.Itoa(peerPort))) } c.Peers = peers srv := testutil.NewServer(c) rt := newRoutingTableForTest(c, srv) err = rt.Join() if err != nil { return nil, err } err = rt.Start() if err != nil { return nil, err } t.errGr.Go(func() error { <-t.ctx.Done() return srv.Shutdown(context.Background()) }) t.errGr.Go(func() error { <-t.ctx.Done() return rt.Shutdown(context.Background()) }) t.peerPorts = append(t.peerPorts, port) return rt, err } func (t *testCluster) shutdown() error { t.cancel() return t.errGr.Wait() } func TestRoutingTable_SingleNode(t *testing.T) { cluster := newTestCluster() defer cluster.cancel() c := testutil.NewConfig() rt, err := cluster.addNode(c) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } if !rt.This().CompareByID(rt.Discovery().GetCoordinator()) { t.Fatalf("Coordinator is different") } if !rt.IsBootstrapped() { t.Fatalf("The coordinator node cannot be bootstrapped") } for partID := uint64(0); partID < c.PartitionCount; partID++ { part := rt.primary.PartitionByID(partID) if !part.Owner().CompareByID(rt.This()) { t.Fatalf("PartID: %d has a different owner", partID) } } if rt.Signature() == 0 { t.Fatalf("routingTable.signature is zero") } if rt.OwnedPartitionCount() != c.PartitionCount { t.Fatalf("Expected owned partition count: %d. Got: %d", rt.OwnedPartitionCount(), c.PartitionCount) } } func TestRoutingTable_Cluster(t *testing.T) { cluster := newTestCluster() defer cluster.cancel() c1 := testutil.NewConfig() rt1, err := cluster.addNode(c1) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } if !rt1.IsBootstrapped() { t.Fatalf("The coordinator node cannot be bootstrapped") } firstSignature := rt1.Signature() c2 := testutil.NewConfig() rt2, err := cluster.addNode(c2) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } err = testutil.TryWithInterval(10, 100*time.Millisecond, func() error { if !rt2.IsBootstrapped() { return errors.New("the second node cannot be bootstrapped") } return nil }) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } if !rt2.Discovery().GetCoordinator().CompareByID(rt1.Discovery().GetCoordinator()) { t.Fatalf("Coordinator is different") } if firstSignature == rt2.Signature() { t.Fatalf("routingTable signature did not changed after node join") } if rt1.OwnedPartitionCount() == c1.PartitionCount { t.Fatalf("rt1 has all the partitions") } if rt2.OwnedPartitionCount() == c2.PartitionCount { t.Fatalf("rt2 has all the partitions") } totalPartitionCount := rt1.OwnedPartitionCount() + rt2.OwnedPartitionCount() if totalPartitionCount != c1.PartitionCount { t.Fatalf("Total partition count is wrong: %d", totalPartitionCount) } err = cluster.shutdown() if err != nil { t.Fatalf("Expected nil. Got: %v", err) } } func TestRoutingTable_CheckPartitionOwnership(t *testing.T) { cluster := newTestCluster() defer cluster.cancel() c1 := testutil.NewConfig() rt1, err := cluster.addNode(c1) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } if !rt1.IsBootstrapped() { t.Fatalf("The coordinator node cannot be bootstrapped") } c2 := testutil.NewConfig() rt2, err := cluster.addNode(c2) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } err = testutil.TryWithInterval(10, 100*time.Millisecond, func() error { if !rt2.IsBootstrapped() { return errors.New("the second node cannot be bootstrapped") } return nil }) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } for partID := uint64(0); partID < c1.PartitionCount; partID++ { ownerOne := rt1.primary.PartitionByID(partID).Owner() ownerTwo := rt2.primary.PartitionByID(partID).Owner() if !ownerOne.CompareByID(ownerTwo) { t.Fatalf("Different partition: %d owner: %s != %s", partID, ownerOne, ownerTwo) } } err = cluster.shutdown() if err != nil { t.Fatalf("Expected nil. Got: %v", err) } } func TestRoutingTable_NodeLeave(t *testing.T) { cluster := newTestCluster() defer cluster.cancel() c1 := testutil.NewConfig() rt1, err := cluster.addNode(c1) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } if !rt1.IsBootstrapped() { t.Fatalf("The coordinator node cannot be bootstrapped") } c2 := testutil.NewConfig() rt2, err := cluster.addNode(c2) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } err = testutil.TryWithInterval(10, 100*time.Millisecond, func() error { if !rt2.IsBootstrapped() { return errors.New("the second node cannot be bootstrapped") } return nil }) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } signatureWithTwoNode := rt1.Signature() err = rt1.Shutdown(context.Background()) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } err = testutil.TryWithInterval(50, 100*time.Millisecond, func() error { if rt2.Signature() == signatureWithTwoNode { return errors.New("still has the same signature") } return nil }) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } if !rt2.Discovery().GetCoordinator().CompareByID(rt2.This()) { t.Fatalf("Coordinator is different") } for partID := uint64(0); partID < c2.PartitionCount; partID++ { part := rt2.primary.PartitionByID(partID) if !part.Owner().CompareByID(rt2.This()) { t.Fatalf("PartID: %d has a different owner", partID) } } err = cluster.shutdown() if err != nil { t.Fatalf("Expected nil. Got: %v", err) } } func TestRoutingTable_NodeUpdate(t *testing.T) { cluster := newTestCluster() defer cluster.cancel() c1 := testutil.NewConfig() rt1, err := cluster.addNode(c1) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } if !rt1.IsBootstrapped() { t.Fatalf("The coordinator node cannot be bootstrapped") } c2 := testutil.NewConfig() rt2, err := cluster.addNode(c2) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } err = testutil.TryWithInterval(10, 100*time.Millisecond, func() error { if !rt2.IsBootstrapped() { return errors.New("the second node cannot be bootstrapped") } return nil }) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } n := rt2.Discovery().LocalNode() meta, err := discovery.NewMember(c2).Encode() if err != nil { t.Fatalf("Expected nil. Got: %v", err) } n.Meta = meta event := memberlist.NodeEvent{Event: memberlist.NodeUpdate, Node: n} rt2.Discovery().ClusterEvents <- discovery.ToClusterEvent(event) err = testutil.TryWithInterval(10, 100*time.Millisecond, func() error { _, err = rt1.Members().Get(rt2.This().ID) if err == nil { // node id is updated. return errors.New("rt2 could not be updated") } return nil }) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } err = cluster.shutdown() if err != nil { t.Fatalf("Expected nil. Got: %v", err) } } ================================================ FILE: internal/cluster/routingtable/update.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package routingtable import ( "runtime" "sync" "github.com/olric-data/olric/internal/protocol" "github.com/olric-data/olric/internal/discovery" "github.com/vmihailenco/msgpack/v5" "golang.org/x/sync/errgroup" "golang.org/x/sync/semaphore" ) type leftOverDataReport struct { Partitions []uint64 Backups []uint64 } func (r *RoutingTable) prepareLeftOverDataReport() ([]byte, error) { res := leftOverDataReport{} for partID := uint64(0); partID < r.config.PartitionCount; partID++ { part := r.primary.PartitionByID(partID) if part.Length() != 0 { res.Partitions = append(res.Partitions, partID) } backup := r.backup.PartitionByID(partID) if backup.Length() != 0 { res.Backups = append(res.Backups, partID) } } return msgpack.Marshal(res) } func (r *RoutingTable) updateRoutingTableOnMember(data []byte, member discovery.Member) (*leftOverDataReport, error) { cmd := protocol.NewUpdateRouting(data, r.this.ID).Command(r.ctx) rc := r.client.Get(member.String()) err := rc.Process(r.ctx, cmd) if err != nil { return nil, err } result, err := cmd.Bytes() if err != nil { return nil, err } report := leftOverDataReport{} err = msgpack.Unmarshal(result, &report) if err != nil { r.log.V(3).Printf("[ERROR] Failed to call decode ownership report from %s: %v", member, err) return nil, err } return &report, nil } func (r *RoutingTable) updateRoutingTableOnCluster() (map[discovery.Member]*leftOverDataReport, error) { data, err := msgpack.Marshal(r.table) if err != nil { return nil, err } var mtx sync.Mutex var g errgroup.Group reports := make(map[discovery.Member]*leftOverDataReport) num := int64(runtime.NumCPU()) sem := semaphore.NewWeighted(num) r.Members().RLock() r.Members().Range(func(id uint64, tmp discovery.Member) bool { member := tmp g.Go(func() error { if err := sem.Acquire(r.ctx, 1); err != nil { r.log.V(3).Printf("[ERROR] Failed to acquire semaphore to update routing table on %s: %v", member, err) return err } defer sem.Release(1) report, err := r.updateRoutingTableOnMember(data, member) if err != nil { return err } mtx.Lock() defer mtx.Unlock() reports[member] = report return nil }) return true }) r.Members().RUnlock() if err := g.Wait(); err != nil { return nil, err } return reports, nil } ================================================ FILE: internal/discovery/delegate.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package discovery // delegate is a struct which implements memberlist.Delegate interface. type delegate struct { meta []byte } // newDelegate returns a new delegate instance. func (d *Discovery) newDelegate() (delegate, error) { data, err := d.member.Encode() if err != nil { return delegate{}, err } return delegate{ meta: data, }, nil } // NodeMeta is used to retrieve meta-data about the current node // when broadcasting an alive message. It's length is limited to // the given byte size. This metadata is available in the Node structure. func (d delegate) NodeMeta(limit int) []byte { return d.meta } // NotifyMsg is called when a user-data message is received. func (d delegate) NotifyMsg(data []byte) {} // GetBroadcasts is called when user data messages can be broadcast. func (d delegate) GetBroadcasts(overhead, limit int) [][]byte { return nil } // LocalState is used for a TCP Push/Pull. func (d delegate) LocalState(join bool) []byte { return nil } // MergeRemoteState is invoked after a TCP Push/Pull. func (d delegate) MergeRemoteState(buf []byte, join bool) {} ================================================ FILE: internal/discovery/discovery.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. /*Package discovery provides a basic memberlist integration.*/ package discovery import ( "context" "errors" "fmt" "net" "plugin" "sort" "strconv" "sync" "time" "github.com/hashicorp/memberlist" "github.com/olric-data/olric/config" "github.com/olric-data/olric/internal/stats" "github.com/olric-data/olric/pkg/flog" "github.com/olric-data/olric/pkg/service_discovery" ) const eventChanCapacity = 256 // UptimeSeconds is number of seconds since the server started. var UptimeSeconds = stats.NewInt64Counter() // ErrMemberNotFound indicates that the requested member could not be found in the member list. var ErrMemberNotFound = errors.New("member not found") // ClusterEvent is a single event related to node activity in the memberlist. // The Node member of this struct must not be directly modified. type ClusterEvent struct { Event memberlist.NodeEventType NodeName string NodeAddr net.IP NodePort uint16 NodeMeta []byte // Metadata from the delegate for this node. } func (c *ClusterEvent) MemberAddr() string { port := strconv.Itoa(int(c.NodePort)) return net.JoinHostPort(c.NodeAddr.String(), port) } // Discovery is a structure that encapsulates memberlist and // provides useful functions to utilize it. type Discovery struct { log *flog.Logger member *Member memberlist *memberlist.Memberlist config *config.Config // To manage Join/Leave/Update events clusterEventsMtx sync.RWMutex ClusterEvents chan *ClusterEvent // Try to reconnect dead members eventSubscribers []chan *ClusterEvent serviceDiscovery service_discovery.ServiceDiscovery // Flow control wg sync.WaitGroup ctx context.Context cancel context.CancelFunc } // New creates a new memberlist with a proper configuration and returns a new Discovery instance along with it. func New(log *flog.Logger, c *config.Config) *Discovery { member := NewMember(c) ctx, cancel := context.WithCancel(context.Background()) d := &Discovery{ member: &member, config: c, log: log, ctx: ctx, cancel: cancel, } return d } func (d *Discovery) loadServiceDiscoveryPlugin() error { var sd service_discovery.ServiceDiscovery if val, ok := d.config.ServiceDiscovery["plugin"]; ok { if sd, ok = val.(service_discovery.ServiceDiscovery); !ok { return fmt.Errorf("plugin type %T is not a ServiceDiscovery interface", val) } } else { pluginPath, ok := d.config.ServiceDiscovery["path"] if !ok { return fmt.Errorf("plugin path could not be found") } plug, err := plugin.Open(pluginPath.(string)) if err != nil { return fmt.Errorf("failed to open plugin: %w", err) } symDiscovery, err := plug.Lookup("ServiceDiscovery") if err != nil { return fmt.Errorf("failed to lookup serviceDiscovery symbol: %w", err) } if sd, ok = symDiscovery.(service_discovery.ServiceDiscovery); !ok { return fmt.Errorf("unable to assert type to serviceDiscovery") } } if err := sd.SetConfig(d.config.ServiceDiscovery); err != nil { return err } sd.SetLogger(d.config.Logger) if err := sd.Initialize(); err != nil { return err } d.serviceDiscovery = sd return nil } // increaseUptimeSeconds calls UptimeSeconds.Increase function every second. func (d *Discovery) increaseUptimeSeconds() { defer d.wg.Done() ticker := time.NewTicker(time.Second) defer ticker.Stop() for { select { case <-ticker.C: UptimeSeconds.Increase(1) case <-d.ctx.Done(): return } } } func (d *Discovery) Start() error { if d.config.ServiceDiscovery != nil { if err := d.loadServiceDiscoveryPlugin(); err != nil { return err } } // ClusterEvents chan is consumed by the Olric package to maintain a consistent hash ring. d.ClusterEvents = d.SubscribeNodeEvents() // Initialize a new memberlist dl, err := d.newDelegate() if err != nil { return err } eventsCh := make(chan memberlist.NodeEvent, eventChanCapacity) d.config.MemberlistConfig.Delegate = dl d.config.MemberlistConfig.Logger = d.config.Logger d.config.MemberlistConfig.Events = &memberlist.ChannelEventDelegate{ Ch: eventsCh, } list, err := memberlist.Create(d.config.MemberlistConfig) if err != nil { return err } d.memberlist = list if d.serviceDiscovery != nil { if err := d.serviceDiscovery.Register(); err != nil { return err } } d.wg.Add(1) go d.eventLoop(eventsCh) d.wg.Add(1) go d.increaseUptimeSeconds() return nil } // Join is used to take an existing Memberlist and attempt to Join a cluster // by contacting all the given hosts and performing a state sync. Initially, // the Memberlist only contains our own state, so doing this will cause remote // nodes to become aware of the existence of this node, effectively joining the cluster. func (d *Discovery) Join() (int, error) { if d.serviceDiscovery != nil { peers, err := d.serviceDiscovery.DiscoverPeers() if err != nil { return 0, err } return d.memberlist.Join(peers) } return d.memberlist.Join(d.config.Peers) } func (d *Discovery) Rejoin(peers []string) (int, error) { return d.memberlist.Join(peers) } // GetMembers returns a full list of known alive nodes. func (d *Discovery) GetMembers() []Member { var members []Member nodes := d.memberlist.Members() for _, node := range nodes { member, _ := NewMemberFromMetadata(node.Meta) members = append(members, member) } // sort members by birthdate sort.Slice(members, func(i int, j int) bool { return members[i].Birthdate < members[j].Birthdate }) return members } func (d *Discovery) NumMembers() int { return d.memberlist.NumMembers() } // FindMemberByName finds and returns an alive member. func (d *Discovery) FindMemberByName(name string) (Member, error) { members := d.GetMembers() for _, member := range members { if member.Name == name { return member, nil } } return Member{}, ErrMemberNotFound } // FindMemberByID finds and returns an alive member. func (d *Discovery) FindMemberByID(id uint64) (Member, error) { members := d.GetMembers() for _, member := range members { if member.ID == id { return member, nil } } return Member{}, ErrMemberNotFound } // GetCoordinator returns the oldest node in the memberlist. func (d *Discovery) GetCoordinator() Member { members := d.GetMembers() if len(members) == 0 { d.log.V(1).Printf("[ERROR] There is no member in memberlist") return Member{} } return members[0] } // IsCoordinator returns true if the caller is the coordinator node. func (d *Discovery) IsCoordinator() bool { return d.GetCoordinator().ID == d.member.ID } // LocalNode is used to return the local Node func (d *Discovery) LocalNode() *memberlist.Node { return d.memberlist.LocalNode() } // Shutdown will stop any background maintenance of network activity // for this memberlist, causing it to appear "dead". A leave message // will not be broadcasted prior, so the cluster being left will have // to detect this node's Shutdown using probing. If you wish to more // gracefully exit the cluster, call Leave prior to shutting down. // // This method is safe to call multiple times. func (d *Discovery) Shutdown() error { select { case <-d.ctx.Done(): return nil default: } d.cancel() // We don't do that in a goroutine with a timeout mechanism // because this mechanism may cause goroutine leak. d.wg.Wait() if d.memberlist != nil { // Leave will broadcast a leave message but will not shutdown the background // listeners, meaning the node will continue participating in gossip and state // updates. d.log.V(2).Printf("[INFO] Broadcasting a leave message") if err := d.memberlist.Leave(d.config.LeaveTimeout); err != nil { d.log.V(3).Printf("[WARN] memberlist.Leave returned an error: %v", err) } } if d.serviceDiscovery != nil { defer func(serviceDiscovery service_discovery.ServiceDiscovery) { err := serviceDiscovery.Close() if err != nil { d.log.V(3).Printf("[ERROR] ServiceDiscovery.Close returned an error: %v", err) } }(d.serviceDiscovery) if err := d.serviceDiscovery.Deregister(); err != nil { d.log.V(3).Printf("[ERROR] ServiceDiscovery.Deregister returned an error: %v", err) } } if d.memberlist != nil { return d.memberlist.Shutdown() } return nil } ================================================ FILE: internal/discovery/discovery_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package discovery import ( "fmt" "log" "net" "strconv" "sync" "testing" "time" "github.com/hashicorp/memberlist" "github.com/olric-data/olric/pkg/service_discovery" "github.com/olric-data/olric/internal/testutil" "github.com/stretchr/testify/require" ) type testCluster struct { mtx sync.RWMutex instances []*Discovery members []string } func newTestCluster(t *testing.T) *testCluster { tc := &testCluster{} t.Cleanup(func() { tc.mtx.Lock() defer tc.mtx.Unlock() for _, instance := range tc.instances { require.NoError(t, instance.Shutdown()) } }) return tc } func (tc *testCluster) addNewMember(t *testing.T) *Discovery { tc.mtx.Lock() defer tc.mtx.Unlock() cfg := testutil.NewConfig() for _, peer := range tc.members { cfg.Peers = append(cfg.Peers, peer) } flogger := testutil.NewFlogger(cfg) d := New(flogger, cfg) err := d.Start() require.NoError(t, err) _, err = d.Join() require.NoError(t, err) tc.instances = append(tc.instances, d) addr := net.JoinHostPort( d.config.MemberlistConfig.BindAddr, strconv.Itoa(d.config.MemberlistConfig.BindPort), ) tc.members = append(tc.members, addr) return d } func TestDiscovery_GetCoordinator(t *testing.T) { c := newTestCluster(t) d1 := c.addNewMember(t) d2 := c.addNewMember(t) require.Equal(t, d1.GetCoordinator(), d2.GetCoordinator()) } func TestDiscovery_GetMembers(t *testing.T) { c := newTestCluster(t) d1 := c.addNewMember(t) c.addNewMember(t) c.addNewMember(t) require.Len(t, d1.GetMembers(), 3) } func TestDiscovery_IsCoordinator(t *testing.T) { c := newTestCluster(t) d1 := c.addNewMember(t) <-time.After(100 * time.Millisecond) d2 := c.addNewMember(t) <-time.After(100 * time.Millisecond) d3 := c.addNewMember(t) require.True(t, d1.IsCoordinator()) require.False(t, d2.IsCoordinator()) require.False(t, d3.IsCoordinator()) } func TestDiscovery_NumMembers(t *testing.T) { c := newTestCluster(t) d1 := c.addNewMember(t) c.addNewMember(t) c.addNewMember(t) require.Equal(t, d1.NumMembers(), 3) } func TestDiscovery_LocalNode(t *testing.T) { c := newTestCluster(t) d1 := c.addNewMember(t) require.Equal(t, d1.LocalNode().Name, d1.config.MemberlistConfig.Name) } func TestDiscovery_FindMemberByID(t *testing.T) { c := newTestCluster(t) c.addNewMember(t) c.addNewMember(t) c.addNewMember(t) for i, instance := range c.instances { m, err := instance.FindMemberByID(c.instances[i].member.ID) require.NoError(t, err) require.Equal(t, m.Name, instance.config.MemberlistConfig.Name) } } func TestDiscovery_FindMemberByName(t *testing.T) { c := newTestCluster(t) c.addNewMember(t) c.addNewMember(t) c.addNewMember(t) for i, instance := range c.instances { m, err := instance.FindMemberByName(c.instances[i].member.Name) require.NoError(t, err) require.Equal(t, m.Name, instance.config.MemberlistConfig.Name) } } func TestDiscovery_increaseUptimeSeconds(t *testing.T) { c := newTestCluster(t) c.addNewMember(t) <-time.After(2 * time.Second) require.Greater(t, UptimeSeconds.Read(), int64(0)) } type dummyServiceDiscovery struct { mtx sync.Mutex initialized bool closed bool setLogger bool setConfig bool register bool discoverPeers bool deregister bool log *log.Logger } func (d *dummyServiceDiscovery) Initialize() error { d.mtx.Lock() defer d.mtx.Unlock() d.initialized = true return nil } func (d *dummyServiceDiscovery) SetConfig(_ map[string]interface{}) error { d.mtx.Lock() defer d.mtx.Unlock() d.setConfig = true return nil } func (d *dummyServiceDiscovery) SetLogger(_ *log.Logger) { d.mtx.Lock() defer d.mtx.Unlock() d.setLogger = true } func (d *dummyServiceDiscovery) Register() error { d.mtx.Lock() defer d.mtx.Unlock() d.register = true return nil } func (d *dummyServiceDiscovery) Deregister() error { d.mtx.Lock() defer d.mtx.Unlock() d.deregister = true return nil } func (d *dummyServiceDiscovery) DiscoverPeers() ([]string, error) { d.mtx.Lock() defer d.mtx.Unlock() d.discoverPeers = true return []string{}, nil } func (d *dummyServiceDiscovery) Close() error { d.mtx.Lock() defer d.mtx.Unlock() d.closed = true return nil } var _ service_discovery.ServiceDiscovery = (*dummyServiceDiscovery)(nil) func TestDiscovery_loadServiceDiscoveryPlugin(t *testing.T) { c := testutil.NewConfig() sd := &dummyServiceDiscovery{} c.ServiceDiscovery = map[string]interface{}{ "plugin": sd, "provider": "dummy", "args": fmt.Sprintf("namespace=%s label_selector=\"%s\"", "foo_namespace", "foo_label_selector"), } f := testutil.NewFlogger(c) d := New(f, c) err := d.Start() require.NoError(t, err) _, err = d.Join() require.NoError(t, err) require.True(t, sd.initialized) require.True(t, sd.setConfig) require.True(t, sd.setLogger) require.True(t, sd.register) require.True(t, sd.discoverPeers) } func TestDiscovery_ClusterEvents(t *testing.T) { c := newTestCluster(t) d1 := c.addNewMember(t) d2 := c.addNewMember(t) d3 := c.addNewMember(t) var members []string loop: for { select { case e := <-d1.ClusterEvents: require.Equal(t, memberlist.NodeJoin, e.Event) members = append(members, e.MemberAddr()) if len(members) == 2 { break loop } case <-time.After(2 * time.Second): break loop } } require.Contains(t, members, net.JoinHostPort(d2.config.MemberlistConfig.BindAddr, strconv.Itoa(d2.config.MemberlistConfig.BindPort))) require.Contains(t, members, net.JoinHostPort(d3.config.MemberlistConfig.BindAddr, strconv.Itoa(d3.config.MemberlistConfig.BindPort))) } ================================================ FILE: internal/discovery/events.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package discovery import "github.com/hashicorp/memberlist" func ToClusterEvent(e memberlist.NodeEvent) *ClusterEvent { return &ClusterEvent{ Event: e.Event, NodeName: e.Node.Name, NodeAddr: e.Node.Addr, NodePort: e.Node.Port, NodeMeta: e.Node.Meta, } } func (d *Discovery) handleEvent(event memberlist.NodeEvent) { d.clusterEventsMtx.RLock() defer d.clusterEventsMtx.RUnlock() for _, ch := range d.eventSubscribers { if event.Node.Name == d.member.Name { continue } ch <- ToClusterEvent(event) } } // eventLoop awaits for messages from memberlist and broadcasts them to event listeners. func (d *Discovery) eventLoop(eventsCh chan memberlist.NodeEvent) { defer d.wg.Done() for { select { case e := <-eventsCh: d.handleEvent(e) case <-d.ctx.Done(): return } } } func (d *Discovery) SubscribeNodeEvents() chan *ClusterEvent { d.clusterEventsMtx.Lock() defer d.clusterEventsMtx.Unlock() ch := make(chan *ClusterEvent, eventChanCapacity) d.eventSubscribers = append(d.eventSubscribers, ch) return ch } ================================================ FILE: internal/discovery/member.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package discovery import ( "encoding/binary" "time" "github.com/cespare/xxhash/v2" "github.com/olric-data/olric/config" "github.com/vmihailenco/msgpack/v5" ) // Member represents a node in the cluster. type Member struct { Name string NameHash uint64 ID uint64 Birthdate int64 } // CompareByID returns true if two members denote the same member in the cluster. func (m Member) CompareByID(other Member) bool { // ID variable is calculated by combining member's name and birthdate return m.ID == other.ID } // CompareByName returns true if the two members has the same name in the cluster. // This function is intended to redirect the requests to the partition owner. func (m Member) CompareByName(other Member) bool { return m.NameHash == other.NameHash } func (m Member) String() string { return m.Name } func (m Member) Encode() ([]byte, error) { return msgpack.Marshal(m) } func NewMemberFromMetadata(metadata []byte) (Member, error) { res := &Member{} err := msgpack.Unmarshal(metadata, res) return *res, err } func MemberID(name string, birthdate int64) uint64 { // Calculate member's identity. It's useful to compare hosts. buf := make([]byte, 8+len(name)) binary.BigEndian.PutUint64(buf, uint64(birthdate)) buf = append(buf, []byte(name)...) return xxhash.Sum64(buf) } func NewMember(c *config.Config) Member { birthdate := time.Now().UnixNano() nameHash := xxhash.Sum64([]byte(c.MemberlistConfig.Name)) return Member{ Name: c.MemberlistConfig.Name, NameHash: nameHash, ID: MemberID(c.MemberlistConfig.Name, birthdate), Birthdate: birthdate, } } ================================================ FILE: internal/discovery/member_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package discovery import ( "testing" "github.com/olric-data/olric/internal/testutil" ) func TestMembers(t *testing.T) { c1 := testutil.NewConfig() member1 := NewMember(c1) c2 := testutil.NewConfig() member2 := NewMember(c2) t.Run("Name", func(t *testing.T) { if member1.String() != c1.MemberlistConfig.Name { t.Fatalf("Expected member name: %s. Got: %s", c1.MemberlistConfig.Name, member1.Name) } }) t.Run("CompareByID", func(t *testing.T) { if !member1.CompareByID(member1) { t.Fatalf("members were the same") } if member1.CompareByID(member2) { t.Fatalf("members were different") } }) t.Run("CompareByName", func(t *testing.T) { if !member1.CompareByName(member1) { t.Fatalf("members were the same") } if member1.CompareByName(member2) { t.Fatalf("members were different") } }) t.Run("Encode/Decode", func(t *testing.T) { data, err := member1.Encode() if err != nil { t.Fatalf("Expected nil. Got: %v", err) } decoded, err := NewMemberFromMetadata(data) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } if !member1.CompareByID(decoded) { t.Fatalf("Decoded member is different") } if !member1.CompareByName(decoded) { t.Fatalf("Decoded member is different") } }) } ================================================ FILE: internal/dmap/atomic.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "context" "errors" "fmt" "time" "github.com/olric-data/olric/internal/protocol" "github.com/olric-data/olric/internal/resp" "github.com/olric-data/olric/internal/util" "github.com/olric-data/olric/pkg/storage" ) func (dm *DMap) loadCurrentAtomicInt(e *env) (int, int64, error) { entry, err := dm.Get(e.ctx, e.key) if errors.Is(err, ErrKeyNotFound) { return 0, 0, nil } if err != nil { return 0, 0, err } if entry == nil { return 0, 0, nil } nr, err := util.ParseInt(entry.Value(), 10, 64) if err != nil { return 0, 0, nil } return int(nr), entry.TTL(), nil } func (dm *DMap) atomicIncrDecr(cmd string, e *env, delta int) (int, error) { atomicKey := e.dmap + e.key dm.s.locker.Lock(atomicKey) defer func() { err := dm.s.locker.Unlock(atomicKey) if err != nil { dm.s.log.V(3).Printf("[ERROR] Failed to release the fine grained lock for key: %s on DMap: %s: %v", e.key, e.dmap, err) } }() current, ttl, err := dm.loadCurrentAtomicInt(e) if err != nil { return 0, err } var updated int switch cmd { case protocol.DMap.Incr: updated = current + delta case protocol.DMap.Decr: updated = current - delta default: return 0, fmt.Errorf("invalid operation") } valueBuf := pool.Get() defer pool.Put(valueBuf) enc := resp.New(valueBuf) err = enc.Encode(updated) if err != nil { return 0, err } e.value = make([]byte, valueBuf.Len()) copy(e.value, valueBuf.Bytes()) if ttl != 0 { e.putConfig.HasPX = true e.putConfig.PX = time.Until(time.UnixMilli(ttl)) } err = dm.put(e) if err != nil { return 0, err } return updated, nil } // Incr atomically increments key by delta. The return value is the new value after being incremented or an error. func (dm *DMap) Incr(ctx context.Context, key string, delta int) (int, error) { e := newEnv(ctx) e.dmap = dm.name e.key = key return dm.atomicIncrDecr(protocol.DMap.Incr, e, delta) } // Decr atomically decrements key by delta. The return value is the new value after being decremented or an error. func (dm *DMap) Decr(ctx context.Context, key string, delta int) (int, error) { e := newEnv(ctx) e.dmap = dm.name e.key = key return dm.atomicIncrDecr(protocol.DMap.Decr, e, delta) } func (dm *DMap) getPut(e *env) (storage.Entry, error) { atomicKey := e.dmap + e.key dm.s.locker.Lock(atomicKey) defer func() { err := dm.s.locker.Unlock(atomicKey) if err != nil { dm.s.log.V(3).Printf("[ERROR] Failed to release the lock for key: %s on DMap: %s: %v", e.key, e.dmap, err) } }() entry, err := dm.Get(e.ctx, e.key) if errors.Is(err, ErrKeyNotFound) { err = nil } if err != nil { return nil, err } err = dm.put(e) if err != nil { return nil, err } if entry == nil { // The value is nil. return nil, nil } return entry, nil } // GetPut atomically sets key to value and returns the old value stored at key. func (dm *DMap) GetPut(ctx context.Context, key string, value interface{}) (storage.Entry, error) { if value == nil { value = struct{}{} } valueBuf := pool.Get() defer pool.Put(valueBuf) enc := resp.New(valueBuf) err := enc.Encode(value) if err != nil { return nil, err } e := newEnv(ctx) e.dmap = dm.name e.key = key e.value = make([]byte, valueBuf.Len()) copy(e.value, valueBuf.Bytes()) raw, err := dm.getPut(e) if err != nil { return nil, err } if raw == nil { return nil, nil } return raw, nil } func (dm *DMap) atomicIncrByFloat(e *env, delta float64) (float64, error) { atomicKey := e.dmap + e.key dm.s.locker.Lock(atomicKey) defer func() { err := dm.s.locker.Unlock(atomicKey) if err != nil { dm.s.log.V(3).Printf("[ERROR] Failed to release the fine grained lock for key: %s on DMap: %s: %v", e.key, e.dmap, err) } }() var current float64 entry, err := dm.Get(e.ctx, e.key) if errors.Is(err, ErrKeyNotFound) { err = nil } if err != nil { return 0, err } if entry != nil { current, err = util.ParseFloat(entry.Value(), 64) if err != nil { return 0, err } } latest := current + delta if err != nil { return 0, err } valueBuf := pool.Get() defer pool.Put(valueBuf) enc := resp.New(valueBuf) err = enc.Encode(latest) if err != nil { return 0, err } e.value = valueBuf.Bytes() e.value = make([]byte, valueBuf.Len()) copy(e.value, valueBuf.Bytes()) err = dm.put(e) if err != nil { return 0, err } return latest, nil } // IncrByFloat atomically increments key by delta. The return value is the new value after being incremented or an error. func (dm *DMap) IncrByFloat(ctx context.Context, key string, delta float64) (float64, error) { e := newEnv(ctx) e.dmap = dm.name e.key = key return dm.atomicIncrByFloat(e, delta) } ================================================ FILE: internal/dmap/atomic_handlers.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "strconv" "github.com/olric-data/olric/internal/protocol" "github.com/tidwall/redcon" ) func (s *Service) incrDecrCommon(cmd, dmap, key string, delta int) (int, error) { dm, err := s.getOrCreateDMap(dmap) if err != nil { return 0, err } e := newEnv(s.ctx) e.dmap = dm.name e.key = key return dm.atomicIncrDecr(cmd, e, delta) } func (s *Service) incrCommandHandler(conn redcon.Conn, cmd redcon.Command) { incrCmd, err := protocol.ParseIncrCommand(cmd) if err != nil { protocol.WriteError(conn, err) return } latest, err := s.incrDecrCommon(protocol.DMap.Incr, incrCmd.DMap, incrCmd.Key, incrCmd.Delta) if err != nil { protocol.WriteError(conn, err) return } conn.WriteInt(latest) } func (s *Service) decrCommandHandler(conn redcon.Conn, cmd redcon.Command) { decrCmd, err := protocol.ParseDecrCommand(cmd) if err != nil { protocol.WriteError(conn, err) return } latest, err := s.incrDecrCommon(protocol.DMap.Decr, decrCmd.DMap, decrCmd.Key, decrCmd.Delta) if err != nil { protocol.WriteError(conn, err) return } conn.WriteInt(latest) } func (s *Service) getPutCommandHandler(conn redcon.Conn, cmd redcon.Command) { getPutCmd, err := protocol.ParseGetPutCommand(cmd) if err != nil { protocol.WriteError(conn, err) return } dm, err := s.getOrCreateDMap(getPutCmd.DMap) if err != nil { protocol.WriteError(conn, err) return } e := newEnv(s.ctx) e.dmap = getPutCmd.DMap e.key = getPutCmd.Key e.value = getPutCmd.Value old, err := dm.getPut(e) if err != nil { protocol.WriteError(conn, err) return } if old == nil { conn.WriteNull() return } if getPutCmd.Raw { conn.WriteBulk(old.Encode()) return } conn.WriteBulk(old.Value()) } func (s *Service) incrByFloatCommandHandler(conn redcon.Conn, cmd redcon.Command) { incrCmd, err := protocol.ParseIncrByFloatCommand(cmd) if err != nil { protocol.WriteError(conn, err) return } dm, err := s.getOrCreateDMap(incrCmd.DMap) if err != nil { protocol.WriteError(conn, err) return } e := newEnv(s.ctx) e.dmap = dm.name e.key = incrCmd.Key latest, err := dm.atomicIncrByFloat(e, incrCmd.Delta) if err != nil { protocol.WriteError(conn, err) return } conn.WriteBulkString(strconv.FormatFloat(latest, 'f', -1, 64)) } ================================================ FILE: internal/dmap/atomic_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "bytes" "context" "sync" "sync/atomic" "testing" "time" "github.com/olric-data/olric/internal/protocol" "github.com/olric-data/olric/internal/resp" "github.com/olric-data/olric/internal/testcluster" "github.com/redis/go-redis/v9" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" ) func TestDMap_loadCurrentAtomicInt(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() ctx := context.Background() key := "incr" ttlDuration := time.Second * 5 s.config.DMaps.TTLDuration = time.Second * 5 dm, err := s.NewDMap("atomic_test") require.NoError(t, err) _, err = dm.Incr(ctx, key, 1) if err != nil { s.log.V(2).Printf("[ERROR] Failed to call Incr: %v", err) return } e := newEnv(ctx) e.dmap = dm.name e.key = key _, ttl, err := dm.loadCurrentAtomicInt(e) require.NoError(t, err) <-time.After(time.Millisecond * 500) require.WithinDuration(t, time.UnixMilli(ttl), time.Now(), ttlDuration) } func TestDMap_Atomic_Incr(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() var wg sync.WaitGroup var start chan struct{} key := "incr" ctx := context.Background() incr := func(dm *DMap) { <-start defer wg.Done() _, err := dm.Incr(ctx, key, 1) if err != nil { s.log.V(2).Printf("[ERROR] Failed to call Incr: %v", err) return } } dm, err := s.NewDMap("atomic_test") require.NoError(t, err) start = make(chan struct{}) for i := 0; i < 100; i++ { wg.Add(1) go incr(dm) } close(start) wg.Wait() gr, err := dm.Get(ctx, key) require.NoError(t, err) var res int err = resp.Scan(gr.Value(), &res) require.NoError(t, err) require.Equal(t, 100, res) } func TestDMap_Atomic_Decr(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() var wg sync.WaitGroup var start chan struct{} key := "decr" ctx := context.Background() decr := func(dm *DMap) { <-start defer wg.Done() _, err := dm.Decr(ctx, key, 1) if err != nil { s.log.V(2).Printf("[ERROR] Failed to call Decr: %v", err) return } } dm, err := s.NewDMap("atomic_test") require.NoError(t, err) start = make(chan struct{}) for i := 0; i < 100; i++ { wg.Add(1) go decr(dm) } close(start) wg.Wait() res, err := dm.Get(context.Background(), key) require.NoError(t, err) var value int err = resp.Scan(res.Value(), &value) require.NoError(t, err) require.Equal(t, -100, value) } func TestDMap_Atomic_GetPut(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() var total int64 var wg sync.WaitGroup var start chan struct{} key := "getput" getput := func(dm *DMap, i int) { <-start defer wg.Done() gr, err := dm.GetPut(context.Background(), key, i) if err != nil { s.log.V(2).Printf("[ERROR] Failed to call Decr: %v", err) return } if gr != nil { var oldval int err = resp.Scan(gr.Value(), &oldval) require.NoError(t, err) atomic.AddInt64(&total, int64(oldval)) } } dm, err := s.NewDMap("atomic_test") require.NoError(t, err) start = make(chan struct{}) var final int64 for i := 0; i < 100; i++ { wg.Add(1) go getput(dm, i) final += int64(i) } close(start) wg.Wait() gr, err := dm.Get(context.Background(), key) require.NoError(t, err) var last int err = resp.Scan(gr.Value(), &last) require.NoError(t, err) atomic.AddInt64(&total, int64(last)) require.Equal(t, final, atomic.LoadInt64(&total)) } func TestDMap_Atomic_IncrByFloat(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() var wg sync.WaitGroup var start chan struct{} key := "incrbyfloat" ctx := context.Background() incrByFloat := func(dm *DMap) { <-start defer wg.Done() _, err := dm.IncrByFloat(ctx, key, 1.2) if err != nil { s.log.V(2).Printf("[ERROR] Failed to call IncrByFloat: %v", err) return } } dm, err := s.NewDMap("atomic_test") require.NoError(t, err) start = make(chan struct{}) for i := 0; i < 100; i++ { wg.Add(1) go incrByFloat(dm) } close(start) wg.Wait() gr, err := dm.Get(ctx, key) require.NoError(t, err) var res float64 err = resp.Scan(gr.Value(), &res) require.NoError(t, err) require.Equal(t, 120.0000000000002, res) } func TestDMap_incrCommandHandler(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() var errGr errgroup.Group for i := 0; i < 100; i++ { errGr.Go(func() error { cmd := protocol.NewIncr("mydmap", "mykey", 1).Command(context.Background()) rc := s.client.Get(s.rt.This().String()) err := rc.Process(context.Background(), cmd) if err != nil { return err } _, err = cmd.Result() return err }) } require.NoError(t, errGr.Wait()) cmd := protocol.NewGet("mydmap", "mykey").Command(context.Background()) rc := s.client.Get(s.rt.This().String()) err := rc.Process(context.Background(), cmd) require.NoError(t, err) value, err := cmd.Bytes() require.NoError(t, err) v := new(int) err = resp.Scan(value, v) require.NoError(t, err) require.Equal(t, 100, *v) } func TestDMap_incrCommandHandler_Single_Request(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() cmd := protocol.NewIncr("mydmap", "mykey", 100).Command(context.Background()) rc := s.client.Get(s.rt.This().String()) err := rc.Process(context.Background(), cmd) require.NoError(t, err) value, err := cmd.Result() require.NoError(t, err) require.Equal(t, 100, int(value)) } func TestDMap_decrCommandHandler(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() var errGr errgroup.Group for i := 0; i < 100; i++ { errGr.Go(func() error { cmd := protocol.NewDecr("mydmap", "mykey", 1).Command(context.Background()) rc := s.client.Get(s.rt.This().String()) err := rc.Process(context.Background(), cmd) if err != nil { return err } _, err = cmd.Result() return err }) } require.NoError(t, errGr.Wait()) cmd := protocol.NewGet("mydmap", "mykey").Command(context.Background()) rc := s.client.Get(s.rt.This().String()) err := rc.Process(context.Background(), cmd) require.NoError(t, err) value, err := cmd.Bytes() require.NoError(t, err) v := new(int) err = resp.Scan(value, v) require.NoError(t, err) require.Equal(t, -100, *v) } func TestDMap_decrCommandHandler_Single_Request(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() cmd := protocol.NewDecr("mydmap", "mykey", 100).Command(context.Background()) rc := s.client.Get(s.rt.This().String()) err := rc.Process(context.Background(), cmd) require.NoError(t, err) value, err := cmd.Result() require.NoError(t, err) require.Equal(t, -100, int(value)) } func TestDMap_exGetPutOperation(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() var total int64 var final int64 start := make(chan struct{}) getPut := func(i int) error { <-start buf := bytes.NewBuffer(nil) enc := resp.New(buf) err := enc.Encode(i) if err != nil { return err } cmd := protocol.NewGetPut("mydmap", "mykey", buf.Bytes()).Command(context.Background()) rc := s.client.Get(s.rt.This().String()) err = rc.Process(context.Background(), cmd) if err == redis.Nil { return nil } if err != nil { return err } val, err := cmd.Bytes() if err != nil { return err } if len(val) != 0 { oldval := new(int) err = resp.Scan(val, oldval) if err != nil { return err } atomic.AddInt64(&total, int64(*oldval)) } return nil } var errGr errgroup.Group for i := 0; i < 100; i++ { num := i errGr.Go(func() error { return getPut(num) }) final += int64(i) } close(start) require.NoError(t, errGr.Wait()) dm, err := s.NewDMap("mydmap") require.NoError(t, err) gr, err := dm.Get(context.Background(), "mykey") require.NoError(t, err) var last int err = resp.Scan(gr.Value(), &last) require.NoError(t, err) atomic.AddInt64(&total, int64(last)) require.Equal(t, final, atomic.LoadInt64(&total)) } func TestDMap_incrByFloatCommandHandler(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() var errGr errgroup.Group for i := 0; i < 100; i++ { errGr.Go(func() error { cmd := protocol.NewIncrByFloat("mydmap", "mykey", 1.2).Command(context.Background()) rc := s.client.Get(s.rt.This().String()) err := rc.Process(context.Background(), cmd) if err != nil { return err } _, err = cmd.Result() return err }) } require.NoError(t, errGr.Wait()) cmd := protocol.NewGet("mydmap", "mykey").Command(context.Background()) rc := s.client.Get(s.rt.This().String()) err := rc.Process(context.Background(), cmd) require.NoError(t, err) value, err := cmd.Bytes() require.NoError(t, err) v := new(float64) err = resp.Scan(value, v) require.NoError(t, err) require.Equal(t, 120.0000000000002, *v) } ================================================ FILE: internal/dmap/balance.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "errors" "fmt" "time" "github.com/olric-data/olric/events" "github.com/olric-data/olric/internal/cluster/partitions" "github.com/olric-data/olric/internal/protocol" "github.com/olric-data/olric/pkg/neterrors" "github.com/olric-data/olric/pkg/storage" "github.com/tidwall/redcon" "github.com/vmihailenco/msgpack/v5" ) type fragmentPack struct { PartID uint64 Kind partitions.Kind Name string Payload []byte } func (dm *DMap) fragmentMergeFunction(f *fragment, hkey uint64, entry storage.Entry) error { current, err := f.storage.Get(hkey) if errors.Is(err, storage.ErrKeyNotFound) { return f.storage.Put(hkey, entry) } if err != nil { return err } versions := []*version{{entry: current}, {entry: entry}} versions = dm.sortVersions(versions) winner := versions[0].entry if winner == current { // No need to insert the winner return nil } return f.storage.Put(hkey, winner) } func (dm *DMap) mergeFragments(part *partitions.Partition, fp *fragmentPack) error { f, err := dm.loadOrCreateFragment(part) if err != nil { return err } // Acquire fragment's lock. No one should work on it. f.Lock() defer f.Unlock() return f.storage.Import(fp.Payload, func(hkey uint64, entry storage.Entry) error { return dm.fragmentMergeFunction(f, hkey, entry) }) } func (s *Service) checkOwnership(part *partitions.Partition) bool { owners := part.Owners() for _, owner := range owners { if owner.CompareByID(s.rt.This()) { return true } } return false } func (s *Service) validateFragmentPack(fp *fragmentPack) error { if fp.PartID >= s.config.PartitionCount { return fmt.Errorf("invalid partition id: %d", fp.PartID) } var part *partitions.Partition if fp.Kind == partitions.PRIMARY { part = s.primary.PartitionByID(fp.PartID) } else { part = s.backup.PartitionByID(fp.PartID) } // Check ownership before merging. This is useful to prevent data corruption in network partitioning case. if !s.checkOwnership(part) { return fmt.Errorf("%w: %s", neterrors.ErrInvalidArgument, fmt.Sprintf("partID: %d (kind: %s) doesn't belong to %s", fp.PartID, fp.Kind, s.rt.This())) } return nil } func (s *Service) moveFragmentCommandHandler(conn redcon.Conn, cmd redcon.Command) { moveFragmentCmd, err := protocol.ParseMoveFragmentCommand(cmd) if err != nil { protocol.WriteError(conn, err) return } fp := &fragmentPack{} err = msgpack.Unmarshal(moveFragmentCmd.Payload, fp) if err != nil { s.log.V(2).Printf("[ERROR] Failed to unmarshal DMap: %v", err) protocol.WriteError(conn, err) return } if err = s.validateFragmentPack(fp); err != nil { protocol.WriteError(conn, err) return } var part *partitions.Partition if fp.Kind == partitions.PRIMARY { part = s.primary.PartitionByID(fp.PartID) } else { part = s.backup.PartitionByID(fp.PartID) } s.log.V(2).Printf("[INFO] Received DMap (kind: %s): %s on PartID: %d", fp.Kind, fp.Name, fp.PartID) dm, err := s.NewDMap(fp.Name) if err != nil { protocol.WriteError(conn, err) return } err = dm.mergeFragments(part, fp) if err != nil { s.log.V(2).Printf("[ERROR] Failed to merge Received DMap (kind: %s): %s on PartID: %d: %v", fp.Kind, fp.Name, fp.PartID, err) protocol.WriteError(conn, err) return } if s.config.EnableClusterEventsChannel { e := &events.FragmentReceivedEvent{ Kind: events.KindFragmentReceivedEvent, Source: s.rt.This().String(), DataStructure: "dmap", PartitionID: part.ID(), Identifier: fp.Name, Length: len(moveFragmentCmd.Payload), IsBackup: part.Kind() == partitions.BACKUP, Timestamp: time.Now().UnixNano(), } s.wg.Add(1) go s.publishEvent(e) } conn.WriteString(protocol.StatusOK) } ================================================ FILE: internal/dmap/balance_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "context" "encoding/json" "strconv" "testing" "time" "github.com/olric-data/olric/events" "github.com/olric-data/olric/internal/cluster/partitions" "github.com/olric-data/olric/internal/protocol" "github.com/olric-data/olric/internal/testcluster" "github.com/olric-data/olric/internal/testutil" "github.com/stretchr/testify/require" "github.com/tidwall/redcon" ) func TestDMap_Balance_Invalid_PartID(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() fp := &fragmentPack{ PartID: 12312, Kind: partitions.PRIMARY, Name: "foobar", Payload: nil, } err := s.validateFragmentPack(fp) require.Error(t, err) } func TestDMap_Balance_FragmentMergeFunction(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() dm, err := s.NewDMap("mymap") require.NoError(t, err) err = dm.Put(context.Background(), "mykey", "myval", nil) require.NoError(t, err) hkey := partitions.HKey("mymap", "mykey") part := dm.getPartitionByHKey(hkey, partitions.PRIMARY) f, err := dm.loadFragment(part) require.NoError(t, err) currentValue := []byte("current-value") e := dm.engine.NewEntry() e.SetKey("mykey") e.SetTimestamp(time.Now().UnixNano()) e.SetValue(currentValue) err = dm.fragmentMergeFunction(f, hkey, e) require.NoError(t, err) winner, err := f.storage.Get(hkey) require.NoError(t, err) require.Equal(t, currentValue, winner.Value()) } func TestDMap_Balancer_JoinNewNode(t *testing.T) { cluster := testcluster.New(NewService) db1 := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() dm, err := db1.NewDMap("mymap") require.NoError(t, err) ctx := context.Background() var totalKeys = 1000 for i := 0; i < totalKeys; i++ { key := "balancer-test." + strconv.Itoa(i) err = dm.Put(ctx, key, testutil.ToVal(i), nil) require.NoError(t, err) } // This is an integration test. Here we try to observe the behavior of // balancer with the DMap service. db2 := cluster.AddMember(nil).(*Service) // This automatically syncs the cluster. var db1TotalKeys int for partID := uint64(0); partID < db1.config.PartitionCount; partID++ { part := db1.primary.PartitionByID(partID) db1TotalKeys += part.Length() } require.Less(t, db1TotalKeys, totalKeys) var db2TotalKeys int for partID := uint64(0); partID < db2.config.PartitionCount; partID++ { part := db2.primary.PartitionByID(partID) db2TotalKeys += part.Length() } require.Less(t, db2TotalKeys, totalKeys) require.Equal(t, totalKeys, db1TotalKeys+db2TotalKeys) } func TestDMap_Balancer_WrongOwnership(t *testing.T) { cluster := testcluster.New(NewService) db1 := cluster.AddMember(nil).(*Service) db2 := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() var id uint64 for partID := uint64(0); partID < db2.config.PartitionCount; partID++ { part := db2.primary.PartitionByID(partID) if part.Owner().CompareByID(db2.rt.This()) { id = part.ID() break } } fp := &fragmentPack{ PartID: id, Kind: partitions.PRIMARY, } // invalid argument: partID: 1 (kind: Primary) doesn't belong to 127.0.0.1:62096 require.Error(t, db1.validateFragmentPack(fp)) } func TestDMap_Balancer_ClusterEvents(t *testing.T) { c1 := testutil.NewConfig() c1.TriggerBalancerInterval = time.Millisecond c1.EnableClusterEventsChannel = true e1 := testcluster.NewEnvironment(c1) cluster := testcluster.New(NewService) db1 := cluster.AddMember(e1).(*Service) defer cluster.Shutdown() result := make(chan string, 1) db1.server.ServeMux().HandleFunc(protocol.PubSub.Publish, func(conn redcon.Conn, cmd redcon.Command) { publishCmd, err := protocol.ParsePublishCommand(cmd) require.NoError(t, err) require.Equal(t, events.ClusterEventsChannel, publishCmd.Channel) result <- publishCmd.Message conn.WriteInt(1) }) dm, err := db1.NewDMap("mymap") require.NoError(t, err) var totalKeys = 1000 for i := 0; i < totalKeys; i++ { key := "balancer-test." + strconv.Itoa(i) err = dm.Put(context.Background(), key, testutil.ToVal(i), nil) require.NoError(t, err) } go func() { c2 := testutil.NewConfig() c1.TriggerBalancerInterval = time.Millisecond c2.EnableClusterEventsChannel = true e2 := testcluster.NewEnvironment(c2) s2 := testutil.NewServer(c2) s2.ServeMux().HandleFunc(protocol.PubSub.Publish, func(conn redcon.Conn, cmd redcon.Command) { publishCmd, err := protocol.ParsePublishCommand(cmd) require.NoError(t, err) require.Equal(t, events.ClusterEventsChannel, publishCmd.Channel) result <- publishCmd.Message conn.WriteInt(1) }) e2.Set("server", s2) cluster.AddMember(e2) }() fragmentEvents := make(map[uint64]map[string]struct{}) L: for { select { case msg := <-result: value := make(map[string]interface{}) err = json.Unmarshal([]byte(msg), &value) require.NoError(t, err) kind := value["kind"].(string) if kind == events.KindFragmentMigrationEvent || kind == events.KindFragmentReceivedEvent { partID := uint64(value["partition_id"].(float64)) ev, ok := fragmentEvents[partID] if ok { ev[kind] = struct{}{} } else { fragmentEvents[partID] = map[string]struct{}{kind: {}} } } case <-time.After(time.Second): break L } } for partID, data := range fragmentEvents { require.Len(t, data, 2) part := db1.primary.PartitionByID(partID) // Transferred to db2 require.NotEqual(t, part.Owner().ID, db1.rt.This().ID) } } ================================================ FILE: internal/dmap/compaction.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "context" "runtime" "strings" "sync" "time" "github.com/olric-data/olric/internal/cluster/partitions" "golang.org/x/sync/semaphore" ) func (s *Service) callCompactionOnFragment(f *fragment) bool { for { f.Lock() done, err := f.Compaction() if err != nil { f.Unlock() // Continue return true } f.Unlock() if done { return true } select { case <-s.ctx.Done(): // Break return false case <-time.After(time.Millisecond): } } } func (s *Service) doCompaction(partID uint64) { compaction := func(part *partitions.Partition) { part.Map().Range(func(name, tmp interface{}) bool { if !strings.HasPrefix(name.(string), "dmap.") { // Continue. This fragment belongs to a different data structure. return true } f := tmp.(*fragment) return s.callCompactionOnFragment(f) }) } part := s.primary.PartitionByID(partID) compaction(part) backup := s.backup.PartitionByID(partID) compaction(backup) } func (s *Service) triggerCompaction() { var wg sync.WaitGroup // NumCPU returns the number of logical CPUs usable by the current process. // // The set of available CPUs is checked by querying the operating system // at process startup. Changes to operating system CPU allocation after // process startup are not reflected. numWorkers := runtime.NumCPU() sem := semaphore.NewWeighted(int64(numWorkers)) for partID := uint64(0); partID < s.config.PartitionCount; partID++ { select { case <-s.ctx.Done(): break default: } if err := sem.Acquire(s.ctx, 1); err != nil { if err != context.Canceled { s.log.V(3).Printf("[ERROR] Failed to acquire semaphore for DMap compaction: %v", err) } continue } wg.Add(1) go func(id uint64) { defer wg.Done() defer sem.Release(1) s.doCompaction(id) }(partID) } wg.Wait() } func (s *Service) compactionWorker() { defer s.wg.Done() timer := time.NewTimer(s.config.DMaps.TriggerCompactionInterval) defer timer.Stop() for { timer.Reset(s.config.DMaps.TriggerCompactionInterval) select { case <-timer.C: s.triggerCompaction() case <-s.ctx.Done(): return } } } ================================================ FILE: internal/dmap/compaction_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "context" "fmt" "testing" "time" "github.com/olric-data/olric/internal/ramblock" "github.com/olric-data/olric/pkg/storage" "github.com/olric-data/olric/config" "github.com/olric-data/olric/internal/testcluster" "github.com/olric-data/olric/internal/testutil" "github.com/stretchr/testify/require" ) func TestDMap_Compaction(t *testing.T) { cluster := testcluster.New(NewService) c := testutil.NewConfig() c.DMaps.TriggerCompactionInterval = time.Millisecond c.DMaps.Engine.Name = config.DefaultStorageEngine c.DMaps.Engine.Config = map[string]interface{}{ "tableSize": uint64(2048), // overwrite tableSize to trigger compaction. "maxIdleTableTimeout": time.Millisecond, } kv, err := ramblock.New(storage.NewConfig(c.DMaps.Engine.Config)) require.NoError(t, err) c.DMaps.Engine.Implementation = kv e := testcluster.NewEnvironment(c) s := cluster.AddMember(e).(*Service) defer cluster.Shutdown() checkStorageStats := func() (allocated int) { for partID := uint64(0); partID < s.config.PartitionCount; partID++ { part := s.primary.PartitionByID(partID) tmp, ok := part.Map().Load(s.fragmentName("mymap")) if !ok { continue } f := tmp.(*fragment) f.RLock() s := f.storage.Stats() allocated += s.Allocated f.RUnlock() } return } dm, err := s.NewDMap("mymap") require.NoError(t, err) ctx := context.Background() for i := 0; i < 10000; i++ { err = dm.Put(ctx, testutil.ToKey(i), testutil.ToVal(i), nil) require.NoError(t, err) } initialAllocated := checkStorageStats() for i := 0; i < 10000; i++ { if i%2 != 0 { continue } _, err = dm.Delete(ctx, testutil.ToKey(i)) require.NoError(t, err) } err = testutil.TryWithInterval(50, 100*time.Millisecond, func() error { allocated := checkStorageStats() if initialAllocated <= allocated { return fmt.Errorf("initial allocation is still greater than or equal the current allocation") } return nil }) require.NoError(t, err) } ================================================ FILE: internal/dmap/config.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "fmt" "time" "github.com/olric-data/olric/config" ) // dmapConfig keeps DMap config control parameters and access-log for keys in a dmap. type dmapConfig struct { engine *config.Engine maxIdleDuration time.Duration ttlDuration time.Duration maxKeys int maxInuse int lruSamples int evictionPolicy config.EvictionPolicy } func (c *dmapConfig) load(dc *config.DMaps, name string) error { // Try to set config configuration for this dmap. c.maxIdleDuration = dc.MaxIdleDuration c.ttlDuration = dc.TTLDuration c.maxKeys = dc.MaxKeys c.maxInuse = dc.MaxInuse c.lruSamples = dc.LRUSamples c.evictionPolicy = dc.EvictionPolicy c.engine = dc.Engine if dc.Custom != nil { // config.DMap struct can be used for fine-grained control. cs, ok := dc.Custom[name] if ok { if c.maxIdleDuration != cs.MaxIdleDuration { c.maxIdleDuration = cs.MaxIdleDuration } if c.ttlDuration != cs.TTLDuration { c.ttlDuration = cs.TTLDuration } if c.evictionPolicy != cs.EvictionPolicy { c.evictionPolicy = cs.EvictionPolicy } if c.maxKeys != cs.MaxKeys { c.maxKeys = cs.MaxKeys } if c.maxInuse != cs.MaxInuse { c.maxInuse = cs.MaxInuse } if c.lruSamples != cs.LRUSamples { c.lruSamples = cs.LRUSamples } if c.evictionPolicy != cs.EvictionPolicy { c.evictionPolicy = cs.EvictionPolicy } if c.engine == nil { c.engine = cs.Engine } } } //TODO: Create a new function to verify config. if c.evictionPolicy == config.LRUEviction { if c.maxInuse <= 0 && c.maxKeys <= 0 { return fmt.Errorf("maxInuse or maxKeys have to be greater than zero") } // set the default value. if c.lruSamples == 0 { c.lruSamples = config.DefaultLRUSamples } } return nil } ================================================ FILE: internal/dmap/config_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "testing" "time" "github.com/olric-data/olric/config" "github.com/olric-data/olric/internal/testutil" "github.com/stretchr/testify/require" ) func TestDMap_Config(t *testing.T) { c := config.New("local") // Config for all new DMaps c.DMaps.NumEvictionWorkers = 1 c.DMaps.TTLDuration = 100 * time.Second c.DMaps.MaxKeys = 100000 c.DMaps.MaxInuse = 1000000 c.DMaps.LRUSamples = 10 c.DMaps.EvictionPolicy = config.LRUEviction c.DMaps.Engine = testutil.NewEngineConfig(t) // Config for specified DMaps c.DMaps.Custom = map[string]config.DMap{"foobar": { MaxIdleDuration: 60 * time.Second, TTLDuration: 300 * time.Second, MaxKeys: 500000, LRUSamples: 20, EvictionPolicy: "NONE", Engine: &config.Engine{ Name: "ramblock", Config: map[string]interface{}{ "maxIdleTableTimeout": 15 * time.Minute, "tableSize": uint64(1048576), }, }, }} dc := dmapConfig{} err := dc.load(c.DMaps, "mydmap") require.NoError(t, err) require.Equal(t, c.DMaps.TTLDuration, dc.ttlDuration) require.Equal(t, c.DMaps.MaxKeys, dc.maxKeys) require.Equal(t, c.DMaps.MaxInuse, dc.maxInuse) require.Equal(t, c.DMaps.LRUSamples, dc.lruSamples) require.Equal(t, c.DMaps.EvictionPolicy, dc.evictionPolicy) require.Equal(t, c.DMaps.Engine.Name, dc.engine.Name) t.Run("Custom config", func(t *testing.T) { dcc := dmapConfig{} err := dcc.load(c.DMaps, "foobar") require.NoError(t, err) require.Equal(t, c.DMaps.Custom["foobar"].TTLDuration, dcc.ttlDuration) require.Equal(t, c.DMaps.Custom["foobar"].MaxKeys, dcc.maxKeys) require.Equal(t, c.DMaps.Custom["foobar"].MaxInuse, dcc.maxInuse) require.Equal(t, c.DMaps.Custom["foobar"].LRUSamples, dcc.lruSamples) require.Equal(t, c.DMaps.Custom["foobar"].EvictionPolicy, dcc.evictionPolicy) c.DMaps.Custom["foobar"].Engine.Implementation = nil dcc.engine.Implementation = nil require.Equal(t, c.DMaps.Custom["foobar"].Engine, dcc.engine) }) } ================================================ FILE: internal/dmap/delete.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "context" "errors" "github.com/olric-data/olric/internal/cluster/partitions" "github.com/olric-data/olric/internal/discovery" "github.com/olric-data/olric/internal/protocol" "github.com/olric-data/olric/internal/stats" "golang.org/x/sync/errgroup" ) var ( // DeleteHits is the number of deletion requests resulting in an item being removed. DeleteHits = stats.NewInt64Counter() // DeleteMisses is the number of deletion requests for missing keys. DeleteMisses = stats.NewInt64Counter() ) func (dm *DMap) deleteFromFragment(key string, kind partitions.Kind) error { hkey := partitions.HKey(dm.name, key) part := dm.getPartitionByHKey(hkey, kind) f, err := dm.loadFragment(part) if errors.Is(err, errFragmentNotFound) { // key doesn't exist return nil } if err != nil { return err } f.Lock() defer f.Unlock() return f.storage.Delete(hkey) } func (dm *DMap) deleteFromPreviousOwners(key string, owners []discovery.Member) error { // Traverse in reverse order. Except from the latest host, this one. for i := len(owners) - 2; i >= 0; i-- { owner := owners[i] cmd := protocol.NewDelEntry(dm.name, key).Command(dm.s.ctx) rc := dm.s.client.Get(owner.String()) err := rc.Process(dm.s.ctx, cmd) if err != nil { return protocol.ConvertError(err) } err = cmd.Err() if err != nil { return protocol.ConvertError(err) } } return nil } func (dm *DMap) deleteBackupOnCluster(hkey uint64, key string) error { owners := dm.s.backup.PartitionOwnersByHKey(hkey) var g errgroup.Group for _, owner := range owners { mem := owner g.Go(func() error { cmd := protocol.NewDelEntry(dm.name, key).SetReplica().Command(dm.s.ctx) rc := dm.s.client.Get(mem.String()) err := rc.Process(dm.s.ctx, cmd) if err != nil { dm.s.log.V(3).Printf("[ERROR] Failed to delete replica key/value on %s: %s", dm.name, err) return protocol.ConvertError(err) } return protocol.ConvertError(cmd.Err()) }) } return g.Wait() } // deleteOnCluster is not a thread-safe function func (dm *DMap) deleteOnCluster(hkey uint64, key string, f *fragment) error { owners := dm.s.primary.PartitionOwnersByHKey(hkey) if len(owners) == 0 { panic("partition owners list cannot be empty") } err := dm.deleteFromPreviousOwners(key, owners) if err != nil { return err } if dm.s.config.ReplicaCount != 0 { err := dm.deleteBackupOnCluster(hkey, key) if err != nil { return err } } err = f.storage.Delete(hkey) if err != nil { return err } // DeleteHits is the number of deletion reqs resulting in an item being removed. DeleteHits.Increase(1) return nil } func (dm *DMap) deleteKey(key string) error { hkey := partitions.HKey(dm.name, key) part := dm.getPartitionByHKey(hkey, partitions.PRIMARY) f, err := dm.loadOrCreateFragment(part) if err != nil { return err } f.Lock() defer f.Unlock() // Check the HKey before trying to delete it. if !f.storage.Check(hkey) { // DeleteMisses is the number of deletions reqs for missing keys DeleteMisses.Increase(1) return nil } return dm.deleteOnCluster(hkey, key, f) } func (dm *DMap) deleteKeys(ctx context.Context, keys ...string) (int, error) { members := make(map[discovery.Member][]string) for _, key := range keys { hkey := partitions.HKey(dm.name, key) member := dm.s.primary.PartitionByHKey(hkey).Owner() members[member] = append(members[member], key) } for member, distributedKeys := range members { if member.CompareByName(dm.s.rt.This()) { for _, key := range distributedKeys { if err := dm.deleteKey(key); err != nil { return 0, err } } } else { cmd := protocol.NewDel(dm.name, distributedKeys...).Command(dm.s.ctx) rc := dm.s.client.Get(member.String()) err := rc.Process(ctx, cmd) if err != nil { return 0, protocol.ConvertError(err) } return 0, protocol.ConvertError(cmd.Err()) } } return len(keys), nil } // Delete deletes the value for the given key. Delete will not return error if key doesn't exist. It's thread-safe. // It is safe to modify the contents of the argument after Delete returns. func (dm *DMap) Delete(ctx context.Context, keys ...string) (int, error) { return dm.deleteKeys(ctx, keys...) } ================================================ FILE: internal/dmap/delete_handlers.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "github.com/olric-data/olric/internal/cluster/partitions" "github.com/olric-data/olric/internal/protocol" "github.com/tidwall/redcon" ) func (s *Service) delCommandHandler(conn redcon.Conn, cmd redcon.Command) { delCmd, err := protocol.ParseDelCommand(cmd) if err != nil { protocol.WriteError(conn, err) return } dm, err := s.getOrCreateDMap(delCmd.DMap) if err != nil { protocol.WriteError(conn, err) return } count, err := dm.deleteKeys(s.ctx, delCmd.Keys...) if err != nil { protocol.WriteError(conn, err) return } conn.WriteInt(count) } func (s *Service) delEntryCommandHandler(conn redcon.Conn, cmd redcon.Command) { delCmd, err := protocol.ParseDelEntryCommand(cmd) if err != nil { protocol.WriteError(conn, err) return } dm, err := s.getOrCreateDMap(delCmd.Del.DMap) if err != nil { protocol.WriteError(conn, err) return } var kind = partitions.PRIMARY if delCmd.Replica { kind = partitions.BACKUP } for _, key := range delCmd.Del.Keys { err = dm.deleteFromFragment(key, kind) if err != nil { protocol.WriteError(conn, err) return } } conn.WriteInt(len(delCmd.Del.Keys)) } ================================================ FILE: internal/dmap/delete_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "context" "errors" "fmt" "testing" "time" "github.com/olric-data/olric/config" "github.com/olric-data/olric/internal/cluster/partitions" "github.com/olric-data/olric/internal/discovery" "github.com/olric-data/olric/internal/protocol" "github.com/olric-data/olric/internal/ramblock" "github.com/olric-data/olric/internal/testcluster" "github.com/olric-data/olric/internal/testutil" "github.com/olric-data/olric/pkg/storage" "github.com/stretchr/testify/require" ) func checkEmptyStorageEngine(t *testing.T, s *Service) { maximum := 50 check := func(current int) (bool, error) { for partID := uint64(0); partID < s.config.PartitionCount; partID++ { part := s.primary.PartitionByID(partID) tmp, ok := part.Map().Load("dmap.mymap") if !ok { continue } f := tmp.(*fragment) f.RLock() numTables := f.storage.Stats().NumTables f.RUnlock() if numTables != 1 && current < maximum-1 { return false, nil } if numTables != 1 && current >= maximum-1 { return false, fmt.Errorf("numTables=%d PartID: %d", numTables, partID) } } return true, nil } for i := 0; i < maximum; i++ { done, err := check(i) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } if done { return } <-time.After(100 * time.Millisecond) } t.Fatalf("Failed to control compaction status") } func TestDMap_Delete_Cluster(t *testing.T) { cluster := testcluster.New(NewService) s1 := cluster.AddMember(nil).(*Service) s2 := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() dm1, err := s1.NewDMap("mymap") require.NoError(t, err) ctx := context.Background() for i := 0; i < 10; i++ { err = dm1.Put(ctx, testutil.ToKey(i), testutil.ToVal(i), nil) require.NoError(t, err) } dm2, err := s2.NewDMap("mymap") require.NoError(t, err) for i := 0; i < 10; i++ { _, err = dm2.Delete(ctx, testutil.ToKey(i)) require.NoError(t, err) _, err = dm2.Get(ctx, testutil.ToKey(i)) require.ErrorIs(t, err, ErrKeyNotFound) } } func TestDMap_Delete_Lookup(t *testing.T) { cluster := testcluster.New(NewService) s1 := cluster.AddMember(nil).(*Service) cluster.AddMember(nil) defer cluster.Shutdown() dm1, err := s1.NewDMap("mymap") require.NoError(t, err) ctx := context.Background() for i := 0; i < 10; i++ { err = dm1.Put(ctx, testutil.ToKey(i), testutil.ToVal(i), nil) require.NoError(t, err) } s3 := cluster.AddMember(nil).(*Service) dm2, err := s3.NewDMap("mymap") require.NoError(t, err) for i := 0; i < 10; i++ { _, err = dm2.Delete(ctx, testutil.ToKey(i)) require.NoError(t, err) _, err = dm2.Get(ctx, testutil.ToKey(i)) require.ErrorIs(t, err, ErrKeyNotFound) } } func TestDMap_Delete_StaleFragments(t *testing.T) { cluster := testcluster.New(NewService) c1 := testutil.NewConfig() c1.DMaps.CheckEmptyFragmentsInterval = time.Millisecond e1 := testcluster.NewEnvironment(c1) s1 := cluster.AddMember(e1).(*Service) c2 := testutil.NewConfig() c2.DMaps.CheckEmptyFragmentsInterval = time.Millisecond e2 := testcluster.NewEnvironment(c2) s2 := cluster.AddMember(e2).(*Service) defer cluster.Shutdown() dm1, err := s1.NewDMap("mymap") if err != nil { t.Fatalf("Expected nil. Got: %v", err) } ctx := context.Background() for i := 0; i < 100; i++ { err = dm1.Put(ctx, testutil.ToKey(i), testutil.ToVal(i), nil) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } } dm2, err := s2.NewDMap("mymap") if err != nil { t.Fatalf("Expected nil. Got: %v", err) } for i := 0; i < 100; i++ { _, err = dm2.Delete(ctx, testutil.ToKey(i)) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } _, err = dm2.Get(ctx, testutil.ToKey(i)) if !errors.Is(err, ErrKeyNotFound) { t.Fatalf("Expected ErrKeyNotFound. Got: %v", err) } } s1.wg.Add(1) go s1.janitorWorker() s2.wg.Add(1) go s2.janitorWorker() var dc int32 for i := 0; i < 1000; i++ { dc = 0 for partID := uint64(0); partID < s1.config.PartitionCount; partID++ { for _, instance := range []*Service{s1, s2} { part := instance.primary.PartitionByID(partID) part.Map().Range(func(name, dm interface{}) bool { dc++; return true }) bpart := instance.backup.PartitionByID(partID) bpart.Map().Range(func(name, dm interface{}) bool { dc++; return true }) } } if dc == 0 { break } time.Sleep(100 * time.Millisecond) } if dc != 0 { t.Fatalf("Expected dmap count is 0. Got: %d", dc) } } func TestDMap_Delete_PreviousOwner(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() dm, err := s.NewDMap("mydmap") if err != nil { t.Fatalf("Expected nil. Got: %v", err) } err = dm.Put(context.Background(), "mykey", "myvalue", nil) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } cmd := protocol.NewDelEntry("mydmap", "mykey").Command(context.Background()) rc := s.client.Get(s.rt.This().String()) err = rc.Process(context.Background(), cmd) require.NoError(t, err) require.NoError(t, cmd.Err()) _, err = dm.Get(context.Background(), "mykey") require.ErrorIs(t, err, ErrKeyNotFound) } func TestDMap_Delete_DeleteKeyValFromPreviousOwners(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) cluster.AddMember(nil) defer cluster.Shutdown() dm, err := s.NewDMap("mydmap") if err != nil { t.Fatalf("Expected nil. Got: %v", err) } err = dm.Put(context.Background(), "mykey", "myvalue", nil) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } // Prepare fragmented partition owners list hkey := partitions.HKey("mydmap", "mykey") owners := s.primary.PartitionOwnersByHKey(hkey) owner := owners[len(owners)-1] var data []discovery.Member for _, member := range s.rt.Discovery().GetMembers() { if member.CompareByID(owner) { continue } data = append(data, member) } // this has to be the last one data = append(data, owner) err = dm.deleteFromPreviousOwners("mykey", data) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } } func TestDMap_Delete_Backup(t *testing.T) { cluster := testcluster.New(NewService) c1 := testutil.NewConfig() c1.ReadRepair = true c1.ReplicaCount = 2 e1 := testcluster.NewEnvironment(c1) s1 := cluster.AddMember(e1).(*Service) c2 := testutil.NewConfig() c2.ReadRepair = true c2.ReplicaCount = 2 e2 := testcluster.NewEnvironment(c2) s2 := cluster.AddMember(e2).(*Service) defer cluster.Shutdown() dm1, err := s1.NewDMap("mymap") if err != nil { t.Fatalf("Expected nil. Got: %v", err) } ctx := context.Background() for i := 0; i < 10; i++ { err = dm1.Put(ctx, testutil.ToKey(i), testutil.ToVal(i), nil) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } } dm2, err := s2.NewDMap("mymap") if err != nil { t.Fatalf("Expected nil. Got: %v", err) } for i := 0; i < 10; i++ { _, err = dm2.Delete(ctx, testutil.ToKey(i)) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } _, err = dm2.Get(ctx, testutil.ToKey(i)) if err != ErrKeyNotFound { t.Fatalf("Expected ErrKeyNotFound. Got: %v", err) } } } func TestDMap_Delete_Compaction(t *testing.T) { cluster := testcluster.New(NewService) c := testutil.NewConfig() c.ReadRepair = true c.ReplicaCount = 2 c.DMaps.TriggerCompactionInterval = time.Millisecond c.DMaps.Engine.Name = config.DefaultStorageEngine c.DMaps.Engine.Config = map[string]interface{}{ "tableSize": uint64(100), // overwrite tableSize to trigger compaction. "maxIdleTableTimeout": time.Millisecond, } kv, err := ramblock.New(storage.NewConfig(c.DMaps.Engine.Config)) require.NoError(t, err) c.DMaps.Engine.Implementation = kv e := testcluster.NewEnvironment(c) s := cluster.AddMember(e).(*Service) defer cluster.Shutdown() dm, err := s.NewDMap("mymap") require.NoError(t, err) ctx := context.Background() for i := 0; i < 100; i++ { err = dm.Put(ctx, testutil.ToKey(i), testutil.ToVal(i), nil) require.NoError(t, err) } for i := 0; i < 100; i++ { _, err = dm.Delete(ctx, testutil.ToKey(i)) require.NoError(t, err) _, err = dm.Get(ctx, testutil.ToKey(i)) require.ErrorIs(t, err, ErrKeyNotFound) } checkEmptyStorageEngine(t, s) } ================================================ FILE: internal/dmap/destroy.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "context" "runtime" "github.com/olric-data/olric/internal/discovery" "github.com/olric-data/olric/internal/protocol" "golang.org/x/sync/errgroup" "golang.org/x/sync/semaphore" ) func (dm *DMap) destroyOnCluster(ctx context.Context) error { num := int64(runtime.NumCPU()) sem := semaphore.NewWeighted(num) var g errgroup.Group // Don't block routing table to destroy a DMap on the cluster. // Just get a copy of members and run Destroy. var members []discovery.Member m := dm.s.rt.Members() m.RLock() m.Range(func(_ uint64, member discovery.Member) bool { members = append(members, member) return true }) m.RUnlock() for _, item := range members { addr := item.String() g.Go(func() error { if err := sem.Acquire(dm.s.ctx, 1); err != nil { dm.s.log.V(3). Printf("[ERROR] Failed to acquire semaphore to call Destroy command on %s for %s: %v", addr, dm.name, err) return err } defer sem.Release(1) dm.s.log.V(6).Printf("[DEBUG] Calling DM.DESTROY command on %s for %s", addr, dm.name) cmd := protocol.NewDestroy(dm.name).SetLocal().Command(dm.s.ctx) rc := dm.s.client.Get(addr) err := rc.Process(ctx, cmd) if err != nil { dm.s.log.V(3).Printf("[ERROR] DM.DESTROY returned an error: %v", err) return err } return cmd.Err() }) } return g.Wait() } // Destroy flushes the given DMap on the cluster. You should know that there // is no global lock on DMaps. So if you call Put, Put with EX and Destroy methods // concurrently on the cluster, Put and Put with EX calls may set new values to the DMap. func (dm *DMap) Destroy(ctx context.Context) error { return dm.destroyOnCluster(ctx) } ================================================ FILE: internal/dmap/destroy_handlers.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "errors" "github.com/olric-data/olric/config" "github.com/olric-data/olric/internal/cluster/partitions" "github.com/olric-data/olric/internal/protocol" "github.com/tidwall/redcon" ) func (dm *DMap) destroyFragmentOnPartition(part *partitions.Partition) error { f, err := dm.loadFragment(part) if errors.Is(err, errFragmentNotFound) { // not exists return nil } if err != nil { return err } return wipeOutFragment(part, dm.fragmentName, f) } func (s *Service) destroyLocalDMap(name string) error { // This is very similar with rm -rf. Destroys given dmap on the cluster for partID := uint64(0); partID < s.config.PartitionCount; partID++ { dm, err := s.getDMap(name) if errors.Is(err, ErrDMapNotFound) { continue } if err != nil { return err } part := dm.s.primary.PartitionByID(partID) err = dm.destroyFragmentOnPartition(part) if err != nil { return err } // Destroy on replicas if s.config.ReplicaCount > config.MinimumReplicaCount { backup := dm.s.backup.PartitionByID(partID) err = dm.destroyFragmentOnPartition(backup) if err != nil { return err } } } s.Lock() delete(s.dmaps, name) s.Unlock() return nil } func (s *Service) destroyCommandHandler(conn redcon.Conn, cmd redcon.Command) { destroyCmd, err := protocol.ParseDestroyCommand(cmd) if err != nil { protocol.WriteError(conn, err) return } dm, err := s.getOrCreateDMap(destroyCmd.DMap) if err != nil { protocol.WriteError(conn, err) return } if destroyCmd.Local { err = s.destroyLocalDMap(destroyCmd.DMap) } else { err = dm.destroyOnCluster(s.ctx) } if err != nil { protocol.WriteError(conn, err) return } conn.WriteString(protocol.StatusOK) } ================================================ FILE: internal/dmap/destroy_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "context" "testing" "github.com/olric-data/olric/internal/protocol" "github.com/olric-data/olric/internal/testcluster" "github.com/olric-data/olric/internal/testutil" "github.com/stretchr/testify/require" ) func TestDMap_Destroy_Standalone(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) cluster.AddMember(nil) defer cluster.Shutdown() dm, err := s.NewDMap("mymap") if err != nil { t.Fatalf("Expected nil. Got: %v", err) } ctx := context.Background() for i := 0; i < 100; i++ { err = dm.Put(ctx, testutil.ToKey(i), testutil.ToVal(i), nil) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } } err = dm.Destroy(ctx) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } for i := 0; i < 100; i++ { _, err = dm.Get(ctx, testutil.ToKey(i)) if err != ErrKeyNotFound { t.Fatalf("Expected ErrKeyNotFound. Got: %v", err) } } } func TestDMap_Destroy_Cluster(t *testing.T) { cluster := testcluster.New(NewService) c1 := testutil.NewConfig() c1.ReplicaCount = 2 e1 := testcluster.NewEnvironment(c1) s := cluster.AddMember(e1).(*Service) c2 := testutil.NewConfig() c2.ReplicaCount = 2 e2 := testcluster.NewEnvironment(c2) cluster.AddMember(e2) defer cluster.Shutdown() dm, err := s.NewDMap("mymap") if err != nil { t.Fatalf("Expected nil. Got: %v", err) } ctx := context.Background() for i := 0; i < 100; i++ { err = dm.Put(ctx, testutil.ToKey(i), testutil.ToVal(i), nil) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } } err = dm.Destroy(ctx) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } for i := 0; i < 100; i++ { _, err = dm.Get(ctx, testutil.ToKey(i)) if err != ErrKeyNotFound { t.Fatalf("Expected ErrKeyNotFound. Got: %v", err) } } } func TestDMap_Destroy_destroyOperation(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) cluster.AddMember(nil) defer cluster.Shutdown() dm, err := s.NewDMap("mydmap") if err != nil { t.Fatalf("Expected nil. Got: %v", err) } ctx := context.Background() for i := 0; i < 100; i++ { err = dm.Put(ctx, testutil.ToKey(i), testutil.ToVal(i), nil) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } } cmd := protocol.NewDestroy("mydmap").Command(s.ctx) rc := s.client.Get(s.rt.This().String()) err = rc.Process(s.ctx, cmd) require.NoError(t, err) require.NoError(t, cmd.Err()) for i := 0; i < 100; i++ { _, err = dm.Get(ctx, testutil.ToKey(i)) require.ErrorIs(t, err, ErrKeyNotFound) } } ================================================ FILE: internal/dmap/dmap.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "errors" "fmt" "time" "github.com/olric-data/olric/internal/cluster/partitions" "github.com/olric-data/olric/pkg/storage" ) const nilTimeout = 0 * time.Second var ( // ErrKeyNotFound is returned when a key could not be found. ErrKeyNotFound = errors.New("key not found") ErrDMapNotFound = errors.New("dmap not found") ErrServerGone = errors.New("server is gone") ) // DMap implements a single-hop distributed hash table. type DMap struct { name string fragmentName string s *Service engine storage.Engine config *dmapConfig } // Name exposes name of the DMap. func (dm *DMap) Name() string { return dm.name } // getDMap returns an initialized DMap instance, otherwise it returns ErrDMapNotFound. func (s *Service) getDMap(name string) (*DMap, error) { s.RLock() defer s.RUnlock() dm, ok := s.dmaps[name] if !ok { return nil, ErrDMapNotFound } return dm, nil } func (s *Service) fragmentName(name string) string { return fmt.Sprintf("dmap.%s", name) } // NewDMap creates and returns a new DMap instance. It checks member count quorum // and bootstrapping status before creating a new DMap. func (s *Service) NewDMap(name string) (*DMap, error) { // Check operation status first: // // * Checks member count in the cluster, returns ErrClusterQuorum if // the quorum value cannot be satisfied, // * Checks bootstrapping status and awaits for a short period before // returning ErrRequest timeout. if err := s.rt.CheckMemberCountQuorum(); err != nil { return nil, err } // An Olric node has to be bootstrapped to function properly. if err := s.rt.CheckBootstrap(); err != nil { return nil, err } s.Lock() defer s.Unlock() dm, ok := s.dmaps[name] if ok { return dm, nil } dm = &DMap{ config: &dmapConfig{}, name: name, fragmentName: s.fragmentName(name), s: s, } if err := dm.config.load(s.config.DMaps, name); err != nil { return nil, err } // It's a shortcut. dm.engine = dm.config.engine.Implementation s.dmaps[name] = dm return dm, nil } // getOrCreate is a shortcut function to create a new DMap or get an already initialized DMap instance. func (s *Service) getOrCreateDMap(name string) (*DMap, error) { dm, err := s.getDMap(name) if errors.Is(err, ErrDMapNotFound) { return s.NewDMap(name) } return dm, err } func (dm *DMap) getPartitionByHKey(hkey uint64, kind partitions.Kind) *partitions.Partition { var part *partitions.Partition switch { case kind == partitions.PRIMARY: part = dm.s.primary.PartitionByHKey(hkey) case kind == partitions.BACKUP: part = dm.s.backup.PartitionByHKey(hkey) default: panic("unknown partition kind") } return part } func isKeyExpired(ttl int64) bool { if ttl == 0 { return false } // convert nanoseconds to milliseconds res := (time.Now().UnixNano() / 1000000) >= ttl if res { // number of valid items removed from cache to free memory for new items. EvictedTotal.Increase(1) } return res } ================================================ FILE: internal/dmap/dmap_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "testing" "github.com/stretchr/testify/require" "github.com/olric-data/olric/internal/testcluster" ) func TestDMap_Name(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() dm, err := s.NewDMap("mydmap") require.NoError(t, err) require.Equal(t, "mydmap", dm.Name()) } ================================================ FILE: internal/dmap/env.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "context" "time" "github.com/olric-data/olric/internal/cluster/partitions" ) type env struct { ctx context.Context putConfig *PutConfig hkey uint64 timestamp int64 dmap string key string value []byte timeout time.Duration kind partitions.Kind fragment *fragment } func newEnv(ctx context.Context) *env { if ctx == nil { ctx = context.Background() } return &env{ ctx: ctx, putConfig: &PutConfig{}, timestamp: time.Now().UnixNano(), kind: partitions.PRIMARY, } } ================================================ FILE: internal/dmap/eviction.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "errors" "fmt" "math/rand" "runtime" "sort" "strings" "time" "github.com/olric-data/olric/internal/cluster/partitions" "github.com/olric-data/olric/pkg/storage" "golang.org/x/sync/semaphore" ) // isKeyIdleOnFragment is not a thread-safe function. It accesses underlying fragment for the given hkey. func (dm *DMap) isKeyIdleOnFragment(hkey uint64, f *fragment) bool { if dm.config == nil { return false } if dm.config.maxIdleDuration.Nanoseconds() == 0 { return false } // Maximum time in seconds for each entry to stay idle in the map. // It limits the lifetime of the entries relative to the time of the last // read or write access performed on them. The entries whose idle period // exceeds this limit are expired and evicted automatically. lastAccess, err := f.storage.GetLastAccess(hkey) if errors.Is(err, storage.ErrKeyNotFound) { return false } //TODO: Handle other errors. ttl := (dm.config.maxIdleDuration.Nanoseconds() + lastAccess) / 1000000 return isKeyExpired(ttl) } func (dm *DMap) isKeyIdle(hkey uint64) bool { part := dm.getPartitionByHKey(hkey, partitions.PRIMARY) f, err := dm.loadFragment(part) if errors.Is(err, errFragmentNotFound) { // it's no possible to know whether the key is idle or not. return false } if err != nil { // This could be a programming error and should never be happened on production systems. panic(fmt.Sprintf("failed to get primary partition for: %d: %v", hkey, err)) } f.Lock() defer f.Unlock() return dm.isKeyIdleOnFragment(hkey, f) } func (s *Service) evictKeysAtBackground() { defer s.wg.Done() num := int64(runtime.NumCPU()) if s.config.DMaps != nil && s.config.DMaps.NumEvictionWorkers != 0 { num = s.config.DMaps.NumEvictionWorkers } sem := semaphore.NewWeighted(num) for { if !s.isAlive() { return } if err := sem.Acquire(s.ctx, 1); err != nil { s.log.V(3).Printf("[ERROR] Failed to acquire semaphore: %v", err) return } s.wg.Add(1) go func() { defer s.wg.Done() defer sem.Release(1) // Good for developing tests. s.evictKeys() select { case <-time.After(100 * time.Millisecond): case <-s.ctx.Done(): return } }() } } func (s *Service) evictKeys() { partID := uint64(rand.Intn(int(s.config.PartitionCount))) part := s.primary.PartitionByID(partID) part.Map().Range(func(name, tmp interface{}) bool { dmapName := strings.TrimPrefix(name.(string), "dmap.") f := tmp.(*fragment) s.scanFragmentForEviction(partID, dmapName, f) // this breaks the loop, we only scan one dmap instance per call return false }) } func (s *Service) scanFragmentForEviction(partID uint64, name string, f *fragment) { /* From Redis Docs: 1- Test 20 random keys from the set of keys with an associated expire. 2- Delete all the keys found expired. 3- If more than 25% of keys were expired, start again from step 1. */ // We need limits to prevent CPU starvation. deleteOnCluster does some network operation // to delete keys from the backup nodes and the previous owners. var maxKeyCount = 20 var maxTotalCount = 100 var totalCount = 0 dm, err := s.getOrCreateDMap(name) if err != nil { s.log.V(3).Printf("[ERROR] Failed to load DMap: %s: %v", name, err) return } janitor := func() bool { if totalCount > maxTotalCount { // Release the lock. Eviction will be triggered again. return false } f.Lock() defer f.Unlock() count, keyCount := 0, 0 f.storage.RangeHKey(func(hkey uint64) bool { keyCount++ if keyCount >= maxKeyCount { // this means 'break'. return false } ttl, err := f.storage.GetTTL(hkey) if err != nil { dm.s.log.V(3).Printf("[ERROR] Failed to get TTL for: %d", hkey) return true // continue } key, err := f.storage.GetKey(hkey) if err != nil { dm.s.log.V(3).Printf("[ERROR] Failed to get key for: %d", hkey) return true // continue } if isKeyExpired(ttl) || dm.isKeyIdleOnFragment(hkey, f) { err = dm.deleteOnCluster(hkey, key, f) if err != nil { // It will be tried again. dm.s.log.V(3).Printf("[ERROR] Failed to delete expired key: %s on DMap: %s: %v", key, dm.name, err) return true } // number of valid items removed from cache to free memory for new items. EvictedTotal.Increase(1) } return true }) totalCount += count return count >= maxKeyCount/4 } defer func() { if totalCount > 0 { if s.log.V(6).Ok() { s.log.V(6).Printf("[DEBUG] Evicted key count is %d on PartID: %d", totalCount, partID) } } }() for { select { case <-f.ctx.Done(): // the fragment is closed. return case <-s.ctx.Done(): // The server has gone. return default: } // Call janitorWorker again until it returns false. if !janitor() { return } } } type lruItem struct { HKey uint64 LastAccess int64 } func (dm *DMap) evictKeyWithLRU(e *env) error { var idx = 1 var items []lruItem // Warning: fragment is already locked by DMap.Put. Be sure about that before editing this function. // Pick random items from the distributed map and sort them by accessedAt. e.fragment.storage.Range(func(hkey uint64, e storage.Entry) bool { if idx >= dm.config.lruSamples { return false } idx++ i := lruItem{ HKey: hkey, LastAccess: e.LastAccess(), } items = append(items, i) return true }) if len(items) == 0 { return fmt.Errorf("nothing found to expire with LRU") } sort.Slice(items, func(i, j int) bool { return items[i].LastAccess < items[j].LastAccess }) // Pick the first item to delete. It's the least recently used item in the sample. item := items[0] key, err := e.fragment.storage.GetKey(item.HKey) if err != nil { if errors.Is(err, storage.ErrKeyNotFound) { err = ErrKeyNotFound GetMisses.Increase(1) } return err } // Here we have a key/value pair to evict for making room for a new pair. if dm.s.log.V(6).Ok() { dm.s.log.V(6).Printf("[DEBUG] Evicted item on DMap: %s, key: %s with LRU", e.dmap, key) } err = dm.deleteOnCluster(item.HKey, key, e.fragment) if err != nil { return err } // number of valid items removed from cache to free memory for new items. EvictedTotal.Increase(1) return nil } ================================================ FILE: internal/dmap/eviction_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "context" "testing" "time" "github.com/olric-data/olric/config" "github.com/olric-data/olric/internal/testcluster" "github.com/olric-data/olric/internal/testutil" "github.com/stretchr/testify/require" ) func TestDMap_Eviction_TTL(t *testing.T) { cluster := testcluster.New(NewService) s1 := cluster.AddMember(nil).(*Service) s2 := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() dm, err := s1.NewDMap("mydmap") require.NoError(t, err) ctx := context.Background() pc := &PutConfig{ HasEX: true, EX: time.Millisecond, } for i := 0; i < 100; i++ { err = dm.Put(ctx, testutil.ToKey(i), testutil.ToVal(i), pc) require.NoError(t, err) } <-time.After(time.Millisecond) for i := 0; i < 100; i++ { s1.evictKeys() s2.evictKeys() } length := 0 for _, ins := range []*Service{s1, s2} { for partID := uint64(0); partID < s1.config.PartitionCount; partID++ { part := ins.primary.PartitionByID(partID) part.Map().Range(func(k, v interface{}) bool { f := v.(*fragment) length += f.storage.Stats().Length return true }) } } require.NotEqual(t, 100, length) } func TestDMap_Eviction_Config_TTLDuration(t *testing.T) { cluster := testcluster.New(NewService) c := testutil.NewConfig() c.DMaps = &config.DMaps{ TTLDuration: time.Duration(0.1 * float64(time.Second)), Engine: config.NewEngine(), } require.NoError(t, c.DMaps.Engine.Sanitize()) e := testcluster.NewEnvironment(c) s := cluster.AddMember(e).(*Service) defer cluster.Shutdown() dm, err := s.NewDMap("mydmap") require.NoError(t, err) ctx := context.Background() for i := 0; i < 100; i++ { err = dm.Put(ctx, testutil.ToKey(i), testutil.ToVal(i), nil) require.NoError(t, err) } <-time.After(200 * time.Millisecond) for i := 0; i < 100; i++ { s.evictKeys() } length := 0 for partID := uint64(0); partID < s.config.PartitionCount; partID++ { part := s.primary.PartitionByID(partID) part.Map().Range(func(k, v interface{}) bool { f := v.(*fragment) length += f.storage.Stats().Length return true }) } require.NotEqual(t, 100, length) } func TestDMap_Eviction_Config_MaxIdleDuration(t *testing.T) { cluster := testcluster.New(NewService) c := testutil.NewConfig() c.DMaps = &config.DMaps{ MaxIdleDuration: 100 * time.Millisecond, Engine: config.NewEngine(), } require.NoError(t, c.DMaps.Engine.Sanitize()) e := testcluster.NewEnvironment(c) s := cluster.AddMember(e).(*Service) defer cluster.Shutdown() dm, err := s.NewDMap("mydmap") require.NoError(t, err) ctx := context.Background() for i := 0; i < 100; i++ { err = dm.Put(ctx, testutil.ToKey(i), testutil.ToVal(i), nil) require.NoError(t, err) } <-time.After(150 * time.Millisecond) for i := 0; i < 100; i++ { s.evictKeys() } length := 0 for partID := uint64(0); partID < s.config.PartitionCount; partID++ { part := s.primary.PartitionByID(partID) part.Map().Range(func(k, v interface{}) bool { f := v.(*fragment) length += f.storage.Stats().Length return true }) } require.NotEqual(t, 100, length) } func TestDMap_Eviction_LRU_Config_MaxKeys(t *testing.T) { cluster := testcluster.New(NewService) c := testutil.NewConfig() c.DMaps = &config.DMaps{ MaxKeys: 70, EvictionPolicy: config.LRUEviction, Engine: config.NewEngine(), } require.NoError(t, c.DMaps.Engine.Sanitize()) e := testcluster.NewEnvironment(c) s := cluster.AddMember(e).(*Service) defer cluster.Shutdown() dm, err := s.NewDMap("mydmap") require.NoError(t, err) ctx := context.Background() for i := 0; i < 100; i++ { err = dm.Put(ctx, testutil.ToKey(i), testutil.ToVal(i), nil) require.NoError(t, err) } length := 0 for partID := uint64(0); partID < s.config.PartitionCount; partID++ { part := s.primary.PartitionByID(partID) part.Map().Range(func(k, v interface{}) bool { f := v.(*fragment) length += f.storage.Stats().Length return true }) } require.NotEqual(t, 100, length) } func TestDMap_Eviction_LRU_Config_MaxInuse(t *testing.T) { cluster := testcluster.New(NewService) c := testutil.NewConfig() c.DMaps = &config.DMaps{ MaxInuse: 2048, EvictionPolicy: config.LRUEviction, Engine: testutil.NewEngineConfig(t), } e := testcluster.NewEnvironment(c) s := cluster.AddMember(e).(*Service) defer cluster.Shutdown() dm, err := s.NewDMap("mydmap") require.NoError(t, err) ctx := context.Background() for i := 0; i < 100; i++ { err = dm.Put(ctx, testutil.ToKey(i), testutil.ToVal(i), nil) require.NoError(t, err) } length := 0 for partID := uint64(0); partID < s.config.PartitionCount; partID++ { part := s.primary.PartitionByID(partID) part.Map().Range(func(k, v interface{}) bool { f := v.(*fragment) length += f.storage.Stats().Length return true }) } require.NotEqual(t, 100, length) } ================================================ FILE: internal/dmap/expire.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "context" "time" ) // Expire updates the expiry for the given key. It returns ErrKeyNotFound if the // DB does not contain the key. It's thread-safe. func (dm *DMap) Expire(ctx context.Context, key string, timeout time.Duration) error { pc := &PutConfig{ OnlyUpdateTTL: true, } e := newEnv(ctx) e.putConfig = pc e.dmap = dm.name e.key = key e.timeout = timeout return dm.put(e) } ================================================ FILE: internal/dmap/expire_handlers.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "github.com/olric-data/olric/internal/protocol" "github.com/tidwall/redcon" ) func (s *Service) expireCommandHandler(conn redcon.Conn, cmd redcon.Command) { expireCmd, err := protocol.ParseExpireCommand(cmd) if err != nil { protocol.WriteError(conn, err) return } dm, err := s.getOrCreateDMap(expireCmd.DMap) if err != nil { protocol.WriteError(conn, err) return } pc := &PutConfig{ OnlyUpdateTTL: true, } e := newEnv(s.ctx) e.putConfig = pc e.dmap = expireCmd.DMap e.key = expireCmd.Key e.timeout = expireCmd.Seconds err = dm.put(e) if err != nil { protocol.WriteError(conn, err) return } conn.WriteString(protocol.StatusOK) } func (s *Service) pexpireCommandHandler(conn redcon.Conn, cmd redcon.Command) { pexpireCmd, err := protocol.ParsePExpireCommand(cmd) if err != nil { protocol.WriteError(conn, err) return } dm, err := s.getOrCreateDMap(pexpireCmd.DMap) if err != nil { protocol.WriteError(conn, err) return } pc := &PutConfig{ OnlyUpdateTTL: true, } e := newEnv(s.ctx) e.putConfig = pc e.dmap = pexpireCmd.DMap e.key = pexpireCmd.Key e.timeout = pexpireCmd.Milliseconds err = dm.put(e) if err != nil { protocol.WriteError(conn, err) return } conn.WriteString(protocol.StatusOK) } ================================================ FILE: internal/dmap/expire_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "context" "testing" "time" "github.com/olric-data/olric/internal/protocol" "github.com/olric-data/olric/internal/testcluster" "github.com/stretchr/testify/require" ) func TestDMap_Expire(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() ctx := context.Background() dm, err := s.NewDMap("mydmap") require.NoError(t, err) key := "mykey" err = dm.Put(ctx, key, "myvalue", nil) require.NoError(t, err) _, err = dm.Get(ctx, key) require.NoError(t, err) err = dm.Expire(ctx, key, time.Millisecond) require.NoError(t, err) <-time.After(time.Millisecond) // Get the value and check it. _, err = dm.Get(ctx, key) require.ErrorIs(t, err, ErrKeyNotFound) } func TestDMap_Expire_ErrKeyNotFound(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() dm, err := s.NewDMap("mydmap") require.NoError(t, err) err = dm.Expire(context.Background(), "mykey", time.Millisecond) require.ErrorIs(t, err, ErrKeyNotFound) } func TestDMap_Expire_expireCommandHandler(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() ctx := context.Background() dm, err := s.NewDMap("mydmap") require.NoError(t, err) key := "mykey" err = dm.Put(ctx, key, "myvalue", nil) require.NoError(t, err) cmd := protocol.NewExpire("mydmap", "mykey", time.Duration(0.1*float64(time.Second))).Command(s.ctx) rc := s.client.Get(s.rt.This().String()) err = rc.Process(ctx, cmd) require.NoError(t, err) <-time.After(200 * time.Millisecond) // Get the value and check it. _, err = dm.Get(ctx, key) require.ErrorIs(t, err, ErrKeyNotFound) } func TestDMap_Expire_pexpireCommandHandler(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() ctx := context.Background() dm, err := s.NewDMap("mydmap") require.NoError(t, err) key := "mykey" err = dm.Put(ctx, key, "myvalue", nil) require.NoError(t, err) cmd := protocol.NewPExpire("mydmap", "mykey", time.Millisecond).Command(s.ctx) rc := s.client.Get(s.rt.This().String()) err = rc.Process(ctx, cmd) require.NoError(t, err) <-time.After(10 * time.Millisecond) // Get the value and check it. _, err = dm.Get(ctx, key) require.ErrorIs(t, err, ErrKeyNotFound) } ================================================ FILE: internal/dmap/fragment.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "context" "errors" "strings" "sync" "time" "github.com/olric-data/olric/events" "github.com/olric-data/olric/internal/cluster/partitions" "github.com/olric-data/olric/internal/discovery" "github.com/olric-data/olric/internal/protocol" "github.com/olric-data/olric/pkg/storage" "github.com/vmihailenco/msgpack/v5" ) type fragment struct { sync.RWMutex service *Service storage storage.Engine ctx context.Context cancel context.CancelFunc } func (f *fragment) Stats() storage.Stats { f.RLock() defer f.RUnlock() return f.storage.Stats() } func (f *fragment) Compaction() (bool, error) { select { case <-f.ctx.Done(): // fragment is closed or destroyed return false, nil default: } return f.storage.Compaction() } func (f *fragment) Destroy() error { select { case <-f.ctx.Done(): return f.storage.Destroy() default: } return errors.New("fragment is not closed") } func (f *fragment) Close() error { defer f.cancel() return f.storage.Close() } func (f *fragment) Name() string { return "DMap" } func (f *fragment) Move(part *partitions.Partition, name string, owners []discovery.Member) error { f.Lock() defer f.Unlock() i := f.storage.TransferIterator() if !i.Next() { return nil } payload, index, err := i.Export() if err != nil { return err } fp := &fragmentPack{ PartID: part.ID(), Kind: part.Kind(), Name: strings.TrimPrefix(name, "dmap."), Payload: payload, } value, err := msgpack.Marshal(fp) if err != nil { return err } for _, owner := range owners { if f.service.config.EnableClusterEventsChannel { e := &events.FragmentMigrationEvent{ Kind: events.KindFragmentMigrationEvent, Source: f.service.rt.This().String(), Target: owner.String(), DataStructure: "dmap", PartitionID: part.ID(), Identifier: fp.Name, Length: len(value), IsBackup: part.Kind() == partitions.BACKUP, Timestamp: time.Now().UnixNano(), } f.service.wg.Add(1) go f.service.publishEvent(e) } cmd := protocol.NewMoveFragment(value).Command(f.service.ctx) rc := f.service.client.Get(owner.String()) err = rc.Process(f.service.ctx, cmd) if err != nil { return err } if err := cmd.Err(); err != nil { return err } } return i.Drop(index) } func (dm *DMap) newFragment() (*fragment, error) { c := storage.NewConfig(dm.config.engine.Config) engine, err := dm.engine.Fork(c) if err != nil { return nil, err } engine.SetLogger(dm.s.config.Logger) err = engine.Start() if err != nil { return nil, err } ctx, cancel := context.WithCancel(context.Background()) return &fragment{ service: dm.s, storage: engine, ctx: ctx, cancel: cancel, }, nil } func (dm *DMap) loadOrCreateFragment(part *partitions.Partition) (*fragment, error) { part.Lock() defer part.Unlock() // Critical section here. It should be protected by a lock. fg, ok := part.Map().Load(dm.fragmentName) if ok { // We already have the fragment. return fg.(*fragment), nil } f, err := dm.newFragment() if err != nil { return nil, err } part.Map().Store(dm.fragmentName, f) return f, nil } func (dm *DMap) loadFragment(part *partitions.Partition) (*fragment, error) { f, ok := part.Map().Load(dm.fragmentName) if !ok { return nil, errFragmentNotFound } return f.(*fragment), nil } var _ partitions.Fragment = (*fragment)(nil) ================================================ FILE: internal/dmap/fragment_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "errors" "sync" "testing" "github.com/olric-data/olric/internal/cluster/partitions" "github.com/olric-data/olric/internal/testcluster" "github.com/olric-data/olric/internal/testutil" ) func TestDMap_Fragment(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) dm, err := s.NewDMap("mydmap") if err != nil { t.Fatalf("Expected nil. Got: %v", err) } t.Run("loadFragment", func(t *testing.T) { part := s.primary.PartitionByID(1) _, err = dm.loadFragment(part) if !errors.Is(err, errFragmentNotFound) { t.Fatalf("Expected %v. Got: %v", errFragmentNotFound, err) } }) t.Run("newFragment", func(t *testing.T) { _, err := dm.newFragment() if err != nil { t.Fatalf("Expected nil. Got: %v", err) } }) t.Run("loadFragment -- errFragmentNotFound", func(t *testing.T) { part := dm.getPartitionByHKey(123, partitions.PRIMARY) _, err := dm.loadFragment(part) if !errors.Is(err, errFragmentNotFound) { t.Fatalf("Expected %v. Got: %v", errFragmentNotFound, err) } }) t.Run("loadOrCreateFragment", func(t *testing.T) { part := dm.getPartitionByHKey(123, partitions.PRIMARY) _, err = dm.loadOrCreateFragment(part) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } _, err := dm.loadFragment(part) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } }) } func TestDMap_Fragment_Concurrent_Access(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) dm, err := s.NewDMap("mydmap") if err != nil { t.Fatalf("Expected nil. Got: %v", err) } part := dm.getPartitionByHKey(123, partitions.PRIMARY) var mtx sync.RWMutex var wg sync.WaitGroup for i := 0; i < 1000; i++ { wg.Add(1) go func(idx int) { defer wg.Done() f, err := dm.loadOrCreateFragment(part) if err != nil { t.Errorf("Expected nil. Got: %v", err) } e := f.storage.NewEntry() e.SetKey(testutil.ToKey(idx)) mtx.Lock() // storage engine is not thread-safe err = f.storage.Put(uint64(idx), e) mtx.Unlock() if err != nil { t.Errorf("Expected nil. Got: %v", err) } }(i) } wg.Wait() f, err := dm.loadFragment(part) if err != nil { t.Errorf("Expected nil. Got: %v", err) } for i := 0; i < 1000; i++ { entry, err := f.storage.Get(uint64(i)) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } if entry.Key() != testutil.ToKey(i) { t.Fatalf("Expected key: %s. Got: %s", testutil.ToKey(i), entry.Key()) } } } ================================================ FILE: internal/dmap/get.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "context" "errors" "sort" "sync" "github.com/olric-data/olric/config" "github.com/olric-data/olric/internal/cluster/partitions" "github.com/olric-data/olric/internal/discovery" "github.com/olric-data/olric/internal/protocol" "github.com/olric-data/olric/internal/stats" "github.com/olric-data/olric/pkg/storage" ) // Entry is a DMap entry with its metadata. type Entry struct { Key string Value interface{} TTL int64 Timestamp int64 } var ( // GetMisses is the number of entries that have been requested and not found GetMisses = stats.NewInt64Counter() // GetHits is the number of entries that have been requested and found present GetHits = stats.NewInt64Counter() // EvictedTotal is the number of entries removed from cache to free memory for new entries. EvictedTotal = stats.NewInt64Counter() ) // ErrReadQuorum means that read quorum cannot be reached to operate. var ErrReadQuorum = errors.New("read quorum cannot be reached") type version struct { host *discovery.Member entry storage.Entry } // getOnFragment retrieves an entry from the associated fragment based on the provided environment details. // It returns the found entry or an error if the key is not found, too large, or expired. func (dm *DMap) getOnFragment(e *env) (storage.Entry, error) { part := dm.getPartitionByHKey(e.hkey, e.kind) f, err := dm.loadFragment(part) if err != nil { return nil, err } f.RLock() defer f.RUnlock() entry, err := f.storage.Get(e.hkey) switch err { case storage.ErrKeyNotFound: err = ErrKeyNotFound case storage.ErrKeyTooLarge: err = ErrKeyTooLarge } if err != nil { return nil, err } if isKeyExpired(entry.TTL()) { return nil, ErrKeyNotFound } return entry, nil } // lookupOnPreviousOwner retrieves the version of a key from a previous owner in the cluster. // It communicates with the specified owner node and decodes the value into a version object. func (dm *DMap) lookupOnPreviousOwner(owner *discovery.Member, key string) (*version, error) { cmd := protocol.NewGetEntry(dm.name, key).Command(dm.s.ctx) rc := dm.s.client.Get(owner.String()) err := rc.Process(dm.s.ctx, cmd) if err != nil { return nil, protocol.ConvertError(err) } value, err := cmd.Bytes() if err != nil { return nil, protocol.ConvertError(err) } v := &version{host: owner} e := dm.engine.NewEntry() e.Decode(value) v.entry = e return v, nil } func (dm *DMap) valueToVersion(value storage.Entry) *version { this := dm.s.rt.This() return &version{ host: &this, entry: value, } } // lookupOnThisNode searches for a key's version on the current node, considering // only the primary partition owner. func (dm *DMap) lookupOnThisNode(hkey uint64, key string) *version { // Check on localhost, the partition owner. part := dm.getPartitionByHKey(hkey, partitions.PRIMARY) f, err := dm.loadFragment(part) if err != nil { if !errors.Is(err, errFragmentNotFound) { dm.s.log.V(3).Printf("[ERROR] Failed to get DMap fragment: %v", err) } return dm.valueToVersion(nil) } f.RLock() defer f.RUnlock() value, err := f.storage.Get(hkey) if err != nil { if !errors.Is(err, storage.ErrKeyNotFound) { // still need to use "ver". just log this error. dm.s.log.V(3).Printf("[ERROR] Failed to get key: %s on %s: %s", key, dm.name, err) } return dm.valueToVersion(nil) } // We found the key // // LRU and MaxIdleDuration eviction policies are only valid on // the partition owner. Normally, we shouldn't need to retrieve the keys // from the backup or the previous owners. When the fsck merge // a fragmented partition or recover keys from a backup, Olric // continue maintaining a reliable access log. return dm.valueToVersion(value) } // lookupOnOwners collects versions of a key/value pair on the partition owner // by including previous partition owners. func (dm *DMap) lookupOnOwners(hkey uint64, key string) []*version { owners := dm.s.primary.PartitionOwnersByHKey(hkey) if len(owners) == 0 { panic("partition owners list cannot be empty") } var ( wg sync.WaitGroup mtx sync.Mutex versions []*version ) versions = append(versions, dm.lookupOnThisNode(hkey, key)) // Run a query on the previous owners. // Traverse in reverse order. Except from the latest host, this one. for i := len(owners) - 2; i >= 0; i-- { owner := owners[i] wg.Add(1) go func(member *discovery.Member) { defer wg.Done() v, err := dm.lookupOnPreviousOwner(member, key) if err != nil { if dm.s.log.V(6).Ok() { dm.s.log.V(6).Printf("[ERROR] Failed to call get on a previous "+ "primary owner: %s: %v", member, err) } return } mtx.Lock() // Ignore failed owners. The balancer will wipe out // the data on those hosts. versions = append(versions, v) mtx.Unlock() }(&owner) } wg.Wait() return versions } func (dm *DMap) sortVersions(versions []*version) []*version { sort.Slice(versions, func(i, j int) bool { return versions[i].entry.Timestamp() >= versions[j].entry.Timestamp() }, ) // Explicit is better than implicit. return versions } // sanitizeAndSortVersions removes nil versions from the input slice and sorts // the remaining versions by recency. func (dm *DMap) sanitizeAndSortVersions(versions []*version) []*version { var sanitized []*version // We use versions slice for read-repair. Clear nil values first. for _, ver := range versions { if ver.entry != nil { sanitized = append(sanitized, ver) } } if len(sanitized) <= 1 { return sanitized } return dm.sortVersions(sanitized) } // lookupOnReplicas retrieves data from replica nodes for the given hash key and // key, returning a list of versioned entries. func (dm *DMap) lookupOnReplicas(hkey uint64, key string) []*version { // Check replicas var ( wg sync.WaitGroup mtx sync.Mutex ) replicas := dm.s.backup.PartitionOwnersByHKey(hkey) versions := make([]*version, 0, len(replicas)) for _, replica := range replicas { wg.Add(1) go func(host *discovery.Member) { defer wg.Done() cmd := protocol.NewGetEntry(dm.name, key).SetReplica().Command(dm.s.ctx) rc := dm.s.client.Get(host.String()) err := rc.Process(dm.s.ctx, cmd) err = protocol.ConvertError(err) if err != nil { if dm.s.log.V(6).Ok() { dm.s.log.V(6).Printf("[DEBUG] Failed to call get on"+ " a replica owner: %s: %v", host, err) } return } value, err := cmd.Bytes() err = protocol.ConvertError(err) if err != nil { if dm.s.log.V(6).Ok() { dm.s.log.V(6).Printf("[DEBUG] Failed to call get on"+ " a replica owner: %s: %v", host, err) } return } v := &version{host: host} e := dm.engine.NewEntry() e.Decode(value) v.entry = e mtx.Lock() versions = append(versions, v) mtx.Unlock() }(&replica) } wg.Wait() return versions } // readRepair performs synchronization of inconsistent replicas by applying the // winning version to out-of-sync nodes. func (dm *DMap) readRepair(winner *version, versions []*version) { var wg sync.WaitGroup for _, value := range versions { // Check the timestamp first, we apply the "last write wins" rule here. if value.entry != nil && winner.entry.Timestamp() == value.entry.Timestamp() { continue } wg.Add(1) go func(v *version) { defer wg.Done() // Sync tmp := *v.host if tmp.CompareByID(dm.s.rt.This()) { hkey := partitions.HKey(dm.name, winner.entry.Key()) part := dm.getPartitionByHKey(hkey, partitions.PRIMARY) f, err := dm.loadOrCreateFragment(part) if err != nil { dm.s.log.V(3).Printf("[ERROR] Failed to get or create the fragment for: %s on %s: %v", winner.entry.Key(), dm.name, err) return } f.Lock() e := newEnv(context.Background()) e.hkey = hkey e.fragment = f err = dm.putEntryOnFragment(e, winner.entry) if err != nil { dm.s.log.V(3).Printf("[ERROR] Failed to synchronize with replica: %v", err) } f.Unlock() } else { // If readRepair is enabled, this function is called by every GET request. cmd := protocol.NewPutEntry(dm.name, winner.entry.Key(), winner.entry.Encode()).Command(dm.s.ctx) rc := dm.s.client.Get(v.host.String()) err := rc.Process(dm.s.ctx, cmd) if err != nil { dm.s.log.V(3).Printf("[ERROR] Failed to synchronize replica %s: %v", v.host, err) return } err = cmd.Err() if err != nil { dm.s.log.V(3).Printf("[ERROR] Failed to synchronize replica %s: %v", v.host, err) } } }(value) } wg.Wait() } // getOnCluster retrieves the storage.Entry for a given hashed key and key string // from cluster nodes with read quorum. It ensures data consistency via read repair // and returns ErrKeyNotFound or ErrReadQuorum if conditions aren't met. func (dm *DMap) getOnCluster(hkey uint64, key string) (storage.Entry, error) { // RUnlock should not be called with a defer statement here because // the readRepair function may call putOnFragment function which needs a write // lock. Please remember calling RUnlock before returning here. versions := dm.lookupOnOwners(hkey, key) if dm.s.config.ReadQuorum >= config.MinimumReplicaCount { v := dm.lookupOnReplicas(hkey, key) versions = append(versions, v...) } if len(versions) < dm.s.config.ReadQuorum { return nil, ErrReadQuorum } sorted := dm.sanitizeAndSortVersions(versions) if len(sorted) == 0 { // We checked everywhere, it's not here. return nil, ErrKeyNotFound } if len(sorted) < dm.s.config.ReadQuorum { return nil, ErrReadQuorum } // The most up-to-date version of the values. winner := sorted[0] if isKeyExpired(winner.entry.TTL()) || dm.isKeyIdle(hkey) { return nil, ErrKeyNotFound } if dm.s.config.ReadRepair { // Parallel read operations may propagate different versions of // the same key/value pair. The rule is simple: last write wins. dm.readRepair(winner, versions) } return winner.entry, nil } // Get gets the value for the given key. It returns ErrKeyNotFound if the DB // does not contain the key. It's thread-safe. It is safe to modify the contents // of the returned value. func (dm *DMap) Get(ctx context.Context, key string) (storage.Entry, error) { hkey := partitions.HKey(dm.name, key) member := dm.s.primary.PartitionByHKey(hkey).Owner() // We are on the partition owner if member.CompareByName(dm.s.rt.This()) { entry, err := dm.getOnCluster(hkey, key) if errors.Is(err, ErrKeyNotFound) { GetMisses.Increase(1) } if err != nil { return nil, err } // number of keys that have been requested and found present GetHits.Increase(1) return entry, nil } // Redirect to the partition owner cmd := protocol.NewGet(dm.name, key).SetRaw().Command(dm.s.ctx) rc := dm.s.client.Get(member.String()) err := rc.Process(ctx, cmd) if err != nil { return nil, protocol.ConvertError(err) } value, err := cmd.Bytes() if err != nil { return nil, protocol.ConvertError(err) } // number of keys that have been requested and found present GetHits.Increase(1) entry := dm.engine.NewEntry() entry.Decode(value) return entry, nil } ================================================ FILE: internal/dmap/get_handlers.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "github.com/olric-data/olric/internal/cluster/partitions" "github.com/olric-data/olric/internal/protocol" "github.com/tidwall/redcon" ) func (s *Service) getCommandHandler(conn redcon.Conn, cmd redcon.Command) { getCmd, err := protocol.ParseGetCommand(cmd) if err != nil { protocol.WriteError(conn, err) return } dm, err := s.getOrCreateDMap(getCmd.DMap) if err != nil { protocol.WriteError(conn, err) return } raw, err := dm.Get(s.ctx, getCmd.Key) if err != nil { protocol.WriteError(conn, err) return } if getCmd.Raw { conn.WriteBulk(raw.Encode()) return } conn.WriteBulk(raw.Value()) } func (s *Service) getEntryCommandHandler(conn redcon.Conn, cmd redcon.Command) { getEntryCmd, err := protocol.ParseGetEntryCommand(cmd) if err != nil { protocol.WriteError(conn, err) return } dm, err := s.getOrCreateDMap(getEntryCmd.DMap) if err != nil { protocol.WriteError(conn, err) return } var kind = partitions.PRIMARY if getEntryCmd.Replica { kind = partitions.BACKUP } e := newEnv(s.ctx) e.dmap = getEntryCmd.DMap e.key = getEntryCmd.Key e.hkey = partitions.HKey(getEntryCmd.DMap, getEntryCmd.Key) e.kind = kind nt, err := dm.getOnFragment(e) if err == errFragmentNotFound { err = ErrKeyNotFound } if err != nil { protocol.WriteError(conn, err) return } // We found it. conn.WriteBulk(nt.Encode()) } ================================================ FILE: internal/dmap/get_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "context" "testing" "github.com/olric-data/olric/internal/cluster/routingtable" "github.com/olric-data/olric/internal/testcluster" "github.com/olric-data/olric/internal/testutil" "github.com/stretchr/testify/require" ) func TestDMap_Get_Standalone(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() ctx := context.Background() // Call DMap.Put on S1 dm, err := s.NewDMap("mydmap") require.NoError(t, err) for i := 0; i < 10; i++ { err = dm.Put(ctx, testutil.ToKey(i), testutil.ToVal(i), nil) require.NoError(t, err) } for i := 0; i < 10; i++ { gr, err := dm.Get(ctx, testutil.ToKey(i)) require.NoError(t, err) require.Equal(t, testutil.ToVal(i), gr.Value()) } } func TestDMap_Get_Cluster(t *testing.T) { cluster := testcluster.New(NewService) s1 := cluster.AddMember(nil).(*Service) s2 := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() ctx := context.Background() // Call DMap.Put on S1 dm1, err := s1.NewDMap("mydmap") require.NoError(t, err) for i := 0; i < 10; i++ { err = dm1.Put(ctx, testutil.ToKey(i), testutil.ToVal(i), nil) require.NoError(t, err) } // Call DMap.Get on S2 dm2, err := s2.NewDMap("mydmap") require.NoError(t, err) for i := 0; i < 10; i++ { res, err := dm2.Get(ctx, testutil.ToKey(i)) require.NoError(t, err) require.Equal(t, testutil.ToVal(i), res.Value()) } } func TestDMap_Get_Lookup(t *testing.T) { cluster := testcluster.New(NewService) s1 := cluster.AddMember(nil).(*Service) cluster.AddMember(nil) defer cluster.Shutdown() ctx := context.Background() // Call DMap.Put on S1 dm1, err := s1.NewDMap("mydmap") require.NoError(t, err) for i := 0; i < 10; i++ { err = dm1.Put(ctx, testutil.ToKey(i), testutil.ToVal(i), nil) require.NoError(t, err) } s3 := cluster.AddMember(nil).(*Service) // Call DMap.Get on S3 dm3, err := s3.NewDMap("mydmap") require.NoError(t, err) for i := 0; i < 10; i++ { gr, err := dm3.Get(ctx, testutil.ToKey(i)) require.NoError(t, err) require.Equal(t, testutil.ToVal(i), gr.Value()) } } func TestDMap_Get_NilValue(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() ctx := context.Background() // Call DMap.Put on S1 dm, err := s.NewDMap("mydmap") if err != nil { t.Fatalf("Expected nil. Got: %v", err) } err = dm.Put(ctx, "foobar", nil, nil) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } gr, err := dm.Get(ctx, "foobar") if err != nil { t.Fatalf("Expected nil. Got: %v", err) } require.Equal(t, []byte{}, gr.Value()) _, err = dm.Delete(ctx, "foobar") if err != nil { t.Fatalf("Expected nil. Got: %v", err) } _, err = dm.Get(ctx, "foobar") if err != ErrKeyNotFound { t.Fatalf("Expected ErrKeyNotFound. Got: %v", err) } } func TestDMap_Get_NilValue_Cluster(t *testing.T) { cluster := testcluster.New(NewService) s1 := cluster.AddMember(nil).(*Service) s2 := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() ctx := context.Background() // Call DMap.Put on S1 dm, err := s1.NewDMap("mydmap") require.NoError(t, err) err = dm.Put(ctx, "foobar", nil, nil) require.NoError(t, err) dm2, err := s2.NewDMap("mydmap") require.NoError(t, err) gr, err := dm2.Get(ctx, "foobar") require.NoError(t, err) require.Equal(t, []byte{}, gr.Value()) _, err = dm2.Delete(ctx, "foobar") require.NoError(t, err) _, err = dm2.Get(ctx, "foobar") require.ErrorIs(t, err, ErrKeyNotFound) } func TestDMap_Put_ReadQuorum(t *testing.T) { cluster := testcluster.New(NewService) // Create DMap services with custom configuration c := testutil.NewConfig() c.ReplicaCount = 2 c.ReadQuorum = 2 e := testcluster.NewEnvironment(c) s := cluster.AddMember(e).(*Service) defer cluster.Shutdown() ctx := context.Background() dm, err := s.NewDMap("mydmap") if err != nil { t.Fatalf("Expected nil. Got: %v", err) } _, err = dm.Get(ctx, testutil.ToKey(1)) if err != ErrReadQuorum { t.Fatalf("Expected ErrReadQuorum. Got: %v", err) } } func TestDMap_Get_ReadRepair(t *testing.T) { cluster := testcluster.New(NewService) c1 := testutil.NewConfig() c1.ReadRepair = true c1.ReplicaCount = 2 e1 := testcluster.NewEnvironment(c1) s1 := cluster.AddMember(e1).(*Service) c2 := testutil.NewConfig() c2.ReadRepair = true c2.ReplicaCount = 2 e2 := testcluster.NewEnvironment(c2) s2 := cluster.AddMember(e2).(*Service) defer cluster.Shutdown() // Call DMap.Put on S1 dm1, err := s1.NewDMap("mydmap") require.NoError(t, err) ctx := context.Background() for i := 0; i < 10; i++ { err = dm1.Put(ctx, testutil.ToKey(i), testutil.ToVal(i), nil) require.NoError(t, err) } err = s2.Shutdown(context.Background()) require.NoError(t, err) rt := e2.Get("routingtable").(*routingtable.RoutingTable) err = rt.Shutdown(context.Background()) require.NoError(t, err) c3 := testutil.NewConfig() c3.ReadRepair = true c3.ReplicaCount = 2 e3 := testcluster.NewEnvironment(c3) s3 := cluster.AddMember(e3).(*Service) // Call DMap.Get on S2 dm2, err := s3.NewDMap("mydmap") require.NoError(t, err) for i := 0; i < 10; i++ { gr, err := dm2.Get(ctx, testutil.ToKey(i)) require.NoError(t, err) require.Equal(t, testutil.ToVal(i), gr.Value()) } } ================================================ FILE: internal/dmap/handlers.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "github.com/olric-data/olric/internal/protocol" ) func (s *Service) RegisterHandlers() { s.server.ServeMux().HandleFunc(protocol.DMap.Put, s.putCommandHandler) s.server.ServeMux().HandleFunc(protocol.DMap.Get, s.getCommandHandler) s.server.ServeMux().HandleFunc(protocol.DMap.Del, s.delCommandHandler) s.server.ServeMux().HandleFunc(protocol.DMap.DelEntry, s.delEntryCommandHandler) s.server.ServeMux().HandleFunc(protocol.DMap.GetEntry, s.getEntryCommandHandler) s.server.ServeMux().HandleFunc(protocol.DMap.PutEntry, s.putEntryCommandHandler) s.server.ServeMux().HandleFunc(protocol.DMap.Expire, s.expireCommandHandler) s.server.ServeMux().HandleFunc(protocol.DMap.PExpire, s.pexpireCommandHandler) s.server.ServeMux().HandleFunc(protocol.DMap.Destroy, s.destroyCommandHandler) s.server.ServeMux().HandleFunc(protocol.DMap.Scan, s.scanCommandHandler) s.server.ServeMux().HandleFunc(protocol.DMap.Incr, s.incrCommandHandler) s.server.ServeMux().HandleFunc(protocol.DMap.Decr, s.decrCommandHandler) s.server.ServeMux().HandleFunc(protocol.DMap.GetPut, s.getPutCommandHandler) s.server.ServeMux().HandleFunc(protocol.DMap.IncrByFloat, s.incrByFloatCommandHandler) s.server.ServeMux().HandleFunc(protocol.DMap.Lock, s.lockCommandHandler) s.server.ServeMux().HandleFunc(protocol.DMap.Unlock, s.unlockCommandHandler) s.server.ServeMux().HandleFunc(protocol.DMap.LockLease, s.lockLeaseCommandHandler) s.server.ServeMux().HandleFunc(protocol.DMap.PLockLease, s.plockLeaseCommandHandler) s.server.ServeMux().HandleFunc(protocol.Internal.MoveFragment, s.moveFragmentCommandHandler) } ================================================ FILE: internal/dmap/janitor.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "strings" "time" "github.com/olric-data/olric/internal/cluster/partitions" ) func wipeOutFragment(part *partitions.Partition, name string, f *fragment) error { // Stop background services if there is any. err := f.Close() if err != nil { return err } // Destroy data on-disk or in-memory. err = f.Destroy() if err != nil { return err } // Delete the fragment from partition. part.Map().Delete(name) return nil } func (s *Service) janitor(part *partitions.Partition) { part.Map().Range(func(name, tmp interface{}) bool { if !strings.HasPrefix(name.(string), "dmap.") { // This fragment belongs to a different data structure. return true } f := tmp.(*fragment) f.Lock() defer f.Unlock() if f.storage.Stats().Length != 0 { // It's not empty. Continue scanning. return true } err := wipeOutFragment(part, name.(string), f) if err != nil { s.log.V(3).Printf("[ERROR] Failed to delete empty DMap fragment (kind: %s): %s on PartID: %d", part.Kind(), name, part.ID()) // continue scanning return true } s.log.V(4).Printf("[INFO] Empty DMap fragment (kind: %s) has been deleted: %s on PartID: %d", part.Kind(), name, part.ID()) return true }) } func (s *Service) deleteEmptyFragments() { for partID := uint64(0); partID < s.config.PartitionCount; partID++ { // Clean stale DMap fragments on partition table part := s.primary.PartitionByID(partID) s.janitor(part) // Clean stale DMap fragments on backup partition table backup := s.backup.PartitionByID(partID) s.janitor(backup) } } func (s *Service) janitorWorker() { defer s.wg.Done() timer := time.NewTimer(s.config.DMaps.CheckEmptyFragmentsInterval) defer timer.Stop() for { timer.Reset(s.config.DMaps.CheckEmptyFragmentsInterval) select { case <-timer.C: s.deleteEmptyFragments() case <-s.ctx.Done(): return } } } ================================================ FILE: internal/dmap/lock.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "bytes" "context" "crypto/rand" "encoding/hex" "errors" "fmt" "time" "github.com/olric-data/olric/internal/cluster/partitions" "github.com/olric-data/olric/internal/protocol" ) var ( // ErrLockNotAcquired is returned when the requested lock could not be acquired ErrLockNotAcquired = errors.New("lock not acquired") // ErrNoSuchLock is returned when the requested lock does not exist ErrNoSuchLock = errors.New("no such lock") ) // unlockKey tries to unlock the lock by verifying the lock with token. func (dm *DMap) unlockKey(ctx context.Context, key string, token []byte) error { lkey := dm.name + key // Only one unlockKey should work for a given key. dm.s.locker.Lock(lkey) defer func() { err := dm.s.locker.Unlock(lkey) if err != nil { dm.s.log.V(3).Printf("[ERROR] Failed to release the fine grained lock for key: %s on DMap: %s: %v", key, dm.name, err) } }() // get the key to check its value entry, err := dm.Get(ctx, key) if errors.Is(err, ErrKeyNotFound) { return ErrNoSuchLock } if err != nil { return err } // the lock is released by the node(timeout) or the user if !bytes.Equal(entry.Value(), token) { return ErrNoSuchLock } // release it. _, err = dm.deleteKeys(ctx, key) if err != nil { return fmt.Errorf("unlock failed because of delete: %w", err) } return nil } // Unlock takes key and token and tries to unlock the key. // It redirects the request to the partition owner, if required. func (dm *DMap) Unlock(ctx context.Context, key string, token []byte) error { hkey := partitions.HKey(dm.name, key) member := dm.s.primary.PartitionByHKey(hkey).Owner() if member.CompareByName(dm.s.rt.This()) { return dm.unlockKey(ctx, key, token) } cmd := protocol.NewUnlock(dm.name, key, hex.EncodeToString(token)).Command(dm.s.ctx) rc := dm.s.client.Get(member.String()) err := rc.Process(ctx, cmd) if err != nil { return protocol.ConvertError(err) } return protocol.ConvertError(cmd.Err()) } // tryLock takes a deadline and env and sets a key-value pair by using // Put with NX and PX commands. It tries to acquire the lock 100 times per second // if the lock is already acquired. It returns ErrLockNotAcquired if the deadline exceeds. func (dm *DMap) tryLock(e *env, deadline time.Duration) error { err := dm.put(e) if err == nil { return nil } // If it returns ErrKeyFound, the lock is already acquired. if !errors.Is(err, ErrKeyFound) { // something went wrong return err } ctx, cancel := context.WithTimeout(e.ctx, deadline) defer cancel() timer := time.NewTimer(10 * time.Millisecond) defer timer.Stop() // Try to acquire lock. LOOP: for { timer.Reset(10 * time.Millisecond) select { case <-timer.C: err = dm.put(e) if errors.Is(err, ErrKeyFound) { // not released by the other process/goroutine. try again. continue } if err != nil { // something went wrong. return err } // Acquired! Quit without error. break LOOP case <-ctx.Done(): // Deadline exceeded. Quit with an error. return ErrLockNotAcquired case <-dm.s.ctx.Done(): return fmt.Errorf("server is gone") } } return nil } // Lock prepares a token and env, then calls tryLock func (dm *DMap) Lock(ctx context.Context, key string, timeout, deadline time.Duration) ([]byte, error) { token := make([]byte, 16) _, err := rand.Read(token) if err != nil { return nil, err } var pc PutConfig pc.HasNX = true if timeout.Milliseconds() != 0 { pc.HasPX = true pc.PX = timeout } e := newEnv(ctx) e.putConfig = &pc e.dmap = dm.name e.key = key e.value = token err = dm.tryLock(e, deadline) if err != nil { return nil, err } return token, nil } // leaseKey tries to update the expiry of the key by verifying token. func (dm *DMap) leaseKey(ctx context.Context, key string, token []byte, timeout time.Duration) error { lkey := dm.name + key // Only one unlockKey should work for a given key. dm.s.locker.Lock(lkey) defer func() { err := dm.s.locker.Unlock(lkey) if err != nil { dm.s.log.V(3).Printf("[ERROR] Failed to release the fine grained lock for key: %s on DMap: %s: %v", key, dm.name, err) } }() // get the key to check its value e, err := dm.Get(ctx, key) if errors.Is(err, ErrKeyNotFound) { return ErrNoSuchLock } if err != nil { return err } // the lock is released by the node(timeout) or the user if !bytes.Equal(e.Value(), token) { return ErrNoSuchLock } ttl := e.TTL() if ttl > 0 && (time.Now().UnixNano()/1000000) >= ttl { // already expired return ErrNoSuchLock } // update err = dm.Expire(ctx, key, timeout) if err != nil { return fmt.Errorf("lease failed: %w", err) } return nil } // Lease takes key and token and tries to update the expiry with duration. // It redirects the request to the partition owner, if required. func (dm *DMap) Lease(ctx context.Context, key string, token []byte, timeout time.Duration) error { hkey := partitions.HKey(dm.name, key) member := dm.s.primary.PartitionByHKey(hkey).Owner() if member.CompareByName(dm.s.rt.This()) { return dm.leaseKey(ctx, key, token, timeout) } cmd := protocol.NewLockLease(dm.name, key, hex.EncodeToString(token), timeout.Seconds()).Command(dm.s.ctx) rc := dm.s.client.Get(member.String()) err := rc.Process(ctx, cmd) if err != nil { return protocol.ConvertError(err) } return protocol.ConvertError(cmd.Err()) } ================================================ FILE: internal/dmap/lock_handlers.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "encoding/hex" "time" "github.com/olric-data/olric/internal/protocol" "github.com/tidwall/redcon" ) func (s *Service) unlockCommandHandler(conn redcon.Conn, cmd redcon.Command) { unlockCmd, err := protocol.ParseUnlockCommand(cmd) if err != nil { protocol.WriteError(conn, err) return } dm, err := s.getOrCreateDMap(unlockCmd.DMap) if err != nil { protocol.WriteError(conn, err) return } token, err := hex.DecodeString(unlockCmd.Token) if err != nil { protocol.WriteError(conn, err) return } err = dm.Unlock(s.ctx, unlockCmd.Key, token) if err != nil { protocol.WriteError(conn, err) return } conn.WriteString(protocol.StatusOK) } func (s *Service) lockCommandHandler(conn redcon.Conn, cmd redcon.Command) { lockCmd, err := protocol.ParseLockCommand(cmd) if err != nil { protocol.WriteError(conn, err) return } dm, err := s.getOrCreateDMap(lockCmd.DMap) if err != nil { protocol.WriteError(conn, err) return } var timeout = nilTimeout switch { case lockCmd.EX != 0: timeout = time.Duration(lockCmd.EX * float64(time.Second)) case lockCmd.PX != 0: timeout = time.Duration(lockCmd.PX * int64(time.Millisecond)) } var deadline = time.Duration(lockCmd.Deadline * float64(time.Second)) token, err := dm.Lock(s.ctx, lockCmd.Key, timeout, deadline) if err != nil { protocol.WriteError(conn, err) return } conn.WriteString(hex.EncodeToString(token)) } func (s *Service) lockLeaseCommandHandler(conn redcon.Conn, cmd redcon.Command) { lockLeaseCmd, err := protocol.ParseLockLeaseCommand(cmd) if err != nil { protocol.WriteError(conn, err) return } dm, err := s.getOrCreateDMap(lockLeaseCmd.DMap) if err != nil { protocol.WriteError(conn, err) return } timeout := time.Duration(lockLeaseCmd.Timeout * float64(time.Second)) token, err := hex.DecodeString(lockLeaseCmd.Token) if err != nil { protocol.WriteError(conn, err) return } err = dm.Lease(s.ctx, lockLeaseCmd.Key, token, timeout) if err != nil { protocol.WriteError(conn, err) return } conn.WriteString(protocol.StatusOK) } func (s *Service) plockLeaseCommandHandler(conn redcon.Conn, cmd redcon.Command) { plockLeaseCmd, err := protocol.ParsePLockLeaseCommand(cmd) if err != nil { protocol.WriteError(conn, err) return } dm, err := s.getOrCreateDMap(plockLeaseCmd.DMap) if err != nil { protocol.WriteError(conn, err) return } timeout := time.Duration(plockLeaseCmd.Timeout * int64(time.Millisecond)) token, err := hex.DecodeString(plockLeaseCmd.Token) if err != nil { protocol.WriteError(conn, err) return } err = dm.Lease(s.ctx, plockLeaseCmd.Key, token, timeout) if err != nil { protocol.WriteError(conn, err) return } conn.WriteString(protocol.StatusOK) } ================================================ FILE: internal/dmap/lock_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "context" "encoding/hex" "strconv" "testing" "time" "github.com/olric-data/olric/internal/protocol" "github.com/olric-data/olric/internal/testcluster" "github.com/stretchr/testify/require" ) func TestDMap_Lock_With_Timeout_Standalone(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() key := "lock.test.foo" dm, err := s.NewDMap("lock.test") require.NoError(t, err) ctx := context.Background() token, err := dm.Lock(ctx, key, time.Second, time.Second) require.NoError(t, err) err = dm.Unlock(ctx, key, token) require.NoError(t, err) } func TestDMap_Unlock_After_Timeout_Standalone(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() key := "lock.test.foo" dm, err := s.NewDMap("lock.test") require.NoError(t, err) ctx := context.Background() token, err := dm.Lock(context.Background(), key, time.Millisecond, time.Second) require.NoError(t, err) <-time.After(10 * time.Millisecond) err = dm.Unlock(ctx, key, token) require.ErrorIs(t, err, ErrNoSuchLock) } func TestDMap_Lock_With_Timeout_ErrLockNotAcquired_Standalone(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() key := "lock.test.foo" dm, err := s.NewDMap("lock.test") require.NoError(t, err) ctx := context.Background() _, err = dm.Lock(ctx, key, time.Second, time.Second) require.NoError(t, err) _, err = dm.Lock(context.Background(), key, time.Second, time.Millisecond) require.ErrorIs(t, err, ErrLockNotAcquired) } func TestDMap_LockLease_Standalone(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() key := "lock.test.foo" dm, err := s.NewDMap("lock.test") require.NoError(t, err) ctx := context.Background() token, err := dm.Lock(context.Background(), key, time.Second, time.Second) require.NoError(t, err) err = dm.Lease(ctx, key, token, 2*time.Second) require.NoError(t, err) e, err := dm.Get(ctx, key) require.NoError(t, err) if e.TTL()-(time.Now().UnixNano()/1000000) <= 1900 { t.Fatalf("Expected >=1900. Got: %v", e.TTL()-(time.Now().UnixNano()/1000000)) } <-time.After(3 * time.Second) err = dm.Lease(ctx, key, token, 3*time.Second) require.ErrorIs(t, err, ErrNoSuchLock) } func TestDMap_Lock_Standalone(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() ctx := context.Background() key := "lock.test.foo" dm, err := s.NewDMap("lock.test") require.NoError(t, err) token, err := dm.Lock(ctx, key, nilTimeout, time.Second) require.NoError(t, err) err = dm.Unlock(ctx, key, token) require.NoError(t, err) } func TestDMap_Lock_ErrLockNotAcquired_Standalone(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() ctx := context.Background() key := "lock.test.foo" dm, err := s.NewDMap("lock.test") require.NoError(t, err) _, err = dm.Lock(ctx, key, nilTimeout, time.Second) require.NoError(t, err) _, err = dm.Lock(ctx, key, nilTimeout, time.Millisecond) require.ErrorIs(t, err, ErrLockNotAcquired) } func TestDMap_LockWithTimeout_Cluster(t *testing.T) { cluster := testcluster.New(NewService) s1 := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() dm, err := s1.NewDMap("lock.test") require.NoError(t, err) ctx := context.Background() tokens := make(map[string][]byte) for i := 0; i < 100; i++ { key := "lock.test.foo." + strconv.Itoa(i) token, err := dm.Lock(ctx, key, time.Hour, time.Second) require.NoError(t, err) tokens[key] = token } cluster.AddMember(nil) for key, token := range tokens { err = dm.Unlock(ctx, key, token) require.NoError(t, err) } } func TestDMap_LockLease_Cluster(t *testing.T) { cluster := testcluster.New(NewService) s1 := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() dm, err := s1.NewDMap("lock.test") require.NoError(t, err) ctx := context.Background() tokens := make(map[string][]byte) for i := 0; i < 100; i++ { key := "lock.test.foo." + strconv.Itoa(i) token, err := dm.Lock(ctx, key, 5*time.Second, 10*time.Millisecond) require.NoError(t, err) tokens[key] = token } cluster.AddMember(nil) for key, token := range tokens { err = dm.Lease(ctx, key, token, 10*time.Second) require.NoError(t, err) } } func TestDMap_Lock_Cluster(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() dm, err := s.NewDMap("lock.test") require.NoError(t, err) ctx := context.Background() tokens := make(map[string][]byte) for i := 0; i < 100; i++ { key := "lock.test.foo." + strconv.Itoa(i) token, err := dm.Lock(ctx, key, nilTimeout, time.Second) require.NoError(t, err) tokens[key] = token } cluster.AddMember(nil) for key, token := range tokens { err = dm.Unlock(ctx, key, token) require.NoError(t, err) } } func TestDMap_LockWithTimeout_ErrLockNotAcquired_Cluster(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() dm, err := s.NewDMap("lock.test") require.NoError(t, err) ctx := context.Background() for i := 0; i < 100; i++ { key := "lock.test.foo." + strconv.Itoa(i) _, err := dm.Lock(ctx, key, time.Second, time.Second) require.NoError(t, err) } cluster.AddMember(nil) for i := 0; i < 100; i++ { key := "lock.test.foo." + strconv.Itoa(i) _, err = dm.Lock(ctx, key, time.Second, time.Millisecond) require.ErrorIs(t, err, ErrLockNotAcquired) } } func TestDMap_Lock_After_Lock_With_Timeout_Cluster(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() dm, err := s.NewDMap("lock.test") require.NoError(t, err) ctx := context.Background() for i := 0; i < 100; i++ { key := "lock.test.foo." + strconv.Itoa(i) _, err = dm.Lock(ctx, key, time.Millisecond, time.Second) require.NoError(t, err) } cluster.AddMember(nil) for i := 0; i < 100; i++ { key := "lock.test.foo." + strconv.Itoa(i) _, err = dm.Lock(ctx, key, nilTimeout, time.Second) require.NoError(t, err) } } func TestDMap_tryLock(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() key := "lock.test.foo" dm, err := s.NewDMap("lock.test") require.NoError(t, err) _, err = dm.Lock(context.Background(), key, time.Second, time.Second) require.NoError(t, err) var i int var acquired bool for i <= 10 { i++ _, err := dm.Lock(context.Background(), key, nilTimeout, 100*time.Millisecond) if err == ErrLockNotAcquired { // already acquired continue } require.NoError(t, err) // Acquired acquired = true break } if !acquired { t.Fatal("Failed to acquire lock") } } func TestDMap_lockCommandHandler(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() cmd := protocol.NewLock("lock.test", "lock.test.foo", 1).Command(s.ctx) rc := s.client.Get(s.rt.This().String()) err := rc.Process(s.ctx, cmd) require.NoError(t, err) token, err := cmd.Bytes() require.NoError(t, err) cmdUnlock := protocol.NewUnlock("lock.test", "lock.test.foo", string(token)).Command(s.ctx) err = rc.Process(s.ctx, cmdUnlock) require.NoError(t, err) err = cmdUnlock.Err() require.NoError(t, err) } func TestDMap_lockCommandHandler_EX(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() cmd := protocol.NewLock("lock.test", "lock.test.foo", 1).SetEX(1).Command(s.ctx) rc := s.client.Get(s.rt.This().String()) err := rc.Process(s.ctx, cmd) require.NoError(t, err) token, err := cmd.Bytes() require.NoError(t, err) <-time.After(2 * time.Second) cmdUnlock := protocol.NewUnlock("lock.test", "lock.test.foo", string(token)).Command(s.ctx) err = rc.Process(s.ctx, cmdUnlock) err = protocol.ConvertError(err) require.ErrorIs(t, err, ErrNoSuchLock) } func TestDMap_lockCommandHandler_PX(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() cmd := protocol.NewLock("lock.test", "lock.test.foo", 1). SetPX((10 * time.Millisecond).Milliseconds()).Command(s.ctx) rc := s.client.Get(s.rt.This().String()) err := rc.Process(s.ctx, cmd) require.NoError(t, err) token, err := cmd.Bytes() require.NoError(t, err) <-time.After(20 * time.Millisecond) cmdUnlock := protocol.NewUnlock("lock.test", "lock.test.foo", string(token)).Command(s.ctx) err = rc.Process(s.ctx, cmdUnlock) err = protocol.ConvertError(err) require.ErrorIs(t, err, ErrNoSuchLock) } func TestDMap_lockLeaseCommandHandler(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() key := "lock.test.foo" dm, err := s.NewDMap("lock.test") require.NoError(t, err) ctx := context.Background() token, err := dm.Lock(ctx, key, time.Second, time.Second) require.NoError(t, err) // Update the timeout etoken := hex.EncodeToString(token) cmd := protocol.NewLockLease("lock.test", key, etoken, 10).Command(s.ctx) rc := s.client.Get(s.rt.This().String()) err = rc.Process(s.ctx, cmd) require.NoError(t, err) <-time.After(2 * time.Second) err = dm.Unlock(ctx, key, token) require.NoError(t, err) } func TestDMap_plockLeaseCommandHandler(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() key := "lock.test.foo" dm, err := s.NewDMap("lock.test") require.NoError(t, err) ctx := context.Background() token, err := dm.Lock(ctx, key, 250*time.Millisecond, time.Second) require.NoError(t, err) // Update the timeout etoken := hex.EncodeToString(token) cmd := protocol.NewPLockLease("lock.test", key, etoken, 2000).Command(s.ctx) rc := s.client.Get(s.rt.This().String()) err = rc.Process(s.ctx, cmd) require.NoError(t, err) <-time.After(500 * time.Millisecond) err = dm.Unlock(ctx, key, token) require.NoError(t, err) } ================================================ FILE: internal/dmap/put.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "context" "errors" "fmt" "time" "github.com/olric-data/olric/config" "github.com/olric-data/olric/internal/bufpool" "github.com/olric-data/olric/internal/cluster/partitions" "github.com/olric-data/olric/internal/discovery" "github.com/olric-data/olric/internal/protocol" "github.com/olric-data/olric/internal/resp" "github.com/olric-data/olric/internal/stats" "github.com/olric-data/olric/pkg/storage" "github.com/redis/go-redis/v9" ) var pool = bufpool.New() // EntriesTotal is the total number of entries(including replicas) // stored during the life of this instance. var EntriesTotal = stats.NewInt64Counter() var ( ErrKeyFound = errors.New("key found") ErrWriteQuorum = errors.New("write quorum cannot be reached") ErrKeyTooLarge = errors.New("key too large") ErrEntryTooLarge = errors.New("entry too large for the configured table size") ) func prepareTTL(e *env) int64 { var ttl int64 switch { case e.putConfig.HasEX: ttl = (e.putConfig.EX.Nanoseconds() + time.Now().UnixNano()) / 1000000 case e.putConfig.HasPX: ttl = (e.putConfig.PX.Nanoseconds() + time.Now().UnixNano()) / 1000000 case e.putConfig.HasEXAT: ttl = e.putConfig.EXAT.Nanoseconds() / 1000000 case e.putConfig.HasPXAT: ttl = e.putConfig.PXAT.Nanoseconds() / 1000000 default: ns := e.timeout.Nanoseconds() if ns != 0 { ttl = (ns + time.Now().UnixNano()) / 1000000 } } return ttl } // putOnFragment calls underlying storage engine's Put method to store the key/value pair. It's not thread-safe. func (dm *DMap) putEntryOnFragment(e *env, nt storage.Entry) error { if e.putConfig.OnlyUpdateTTL { err := e.fragment.storage.UpdateTTL(e.hkey, nt) if err != nil { if errors.Is(err, storage.ErrKeyNotFound) { err = ErrKeyNotFound } return err } return nil } err := e.fragment.storage.Put(e.hkey, nt) if errors.Is(err, storage.ErrKeyTooLarge) { err = ErrKeyTooLarge } if errors.Is(err, storage.ErrEntryTooLarge) { err = ErrEntryTooLarge } if err != nil { return err } // total number of entries stored during the life of this instance. EntriesTotal.Increase(1) return nil } func (dm *DMap) prepareEntry(e *env) storage.Entry { nt := e.fragment.storage.NewEntry() nt.SetKey(e.key) nt.SetValue(e.value) nt.SetTTL(prepareTTL(e)) nt.SetTimestamp(e.timestamp) return nt } func (dm *DMap) putOnReplicaFragment(e *env) error { part := dm.getPartitionByHKey(e.hkey, partitions.BACKUP) f, err := dm.loadOrCreateFragment(part) if err != nil { return err } e.fragment = f f.Lock() defer f.Unlock() err = f.storage.PutRaw(e.hkey, e.value) if errors.Is(err, storage.ErrKeyTooLarge) { err = ErrKeyTooLarge } if errors.Is(err, storage.ErrEntryTooLarge) { err = ErrEntryTooLarge } if err != nil { return err } // total number of entries stored during the life of this instance. EntriesTotal.Increase(1) return nil } func (dm *DMap) asyncPutOnBackup(e *env, data []byte, owner discovery.Member) { defer dm.s.wg.Done() rc := dm.s.client.Get(owner.String()) cmd := protocol.NewPutEntry(e.dmap, e.key, data).Command(dm.s.ctx) err := rc.Process(dm.s.ctx, cmd) if err != nil { if dm.s.log.V(3).Ok() { dm.s.log.V(3).Printf("[ERROR] Failed to create replica in async mode: %v", err) } return } err = cmd.Err() if err != nil { if dm.s.log.V(3).Ok() { dm.s.log.V(3).Printf("[ERROR] Failed to create replica in async mode: %v", err) } } } func (dm *DMap) asyncPutOnCluster(e *env, nt storage.Entry) error { err := dm.putEntryOnFragment(e, nt) if err != nil { return err } encodedEntry := nt.Encode() // Fire and forget mode. owners := dm.s.backup.PartitionOwnersByHKey(e.hkey) for _, owner := range owners { if !dm.s.isAlive() { return ErrServerGone } dm.s.wg.Add(1) go dm.asyncPutOnBackup(e, encodedEntry, owner) } return nil } func (dm *DMap) syncPutOnCluster(e *env, nt storage.Entry) error { // Quorum based replication. var successful int encodedEntry := nt.Encode() owners := dm.s.backup.PartitionOwnersByHKey(e.hkey) for _, owner := range owners { rc := dm.s.client.Get(owner.String()) cmd := protocol.NewPutEntry(dm.name, e.key, encodedEntry).Command(dm.s.ctx) err := rc.Process(dm.s.ctx, cmd) if err != nil { return protocol.ConvertError(err) } err = protocol.ConvertError(cmd.Err()) if err != nil { if dm.s.log.V(3).Ok() { dm.s.log.V(3).Printf("[ERROR] Failed to call put command on %s for DMap: %s: %v", owner, e.dmap, err) } continue } successful++ } err := dm.putEntryOnFragment(e, nt) if err != nil { if dm.s.log.V(3).Ok() { dm.s.log.V(3).Printf("[ERROR] Failed to call put command on %s for DMap: %s: %v", dm.s.rt.This(), e.dmap, err) } } else { successful++ } if successful >= dm.s.config.WriteQuorum { return nil } return ErrWriteQuorum } func (dm *DMap) setLRUEvictionStats(e *env) error { // Try to make room for the new item, if it's required. // MaxKeys and MaxInuse properties of LRU can be used in the same time. // But I think that it's good to use only one of time in a production system. // Because it should be easy to understand and debug. st := e.fragment.storage.Stats() // This works for every request if you enabled LRU. // But loading a number from memory should be very cheap. // ownedPartitionCount changes in the case of node join or leave. ownedPartitionCount := dm.s.rt.OwnedPartitionCount() if ownedPartitionCount == 0 { // Routing table is an eventually consistent data structure. In order to prevent a panic in prod, // check the owned partition count before doing math. return nil } if dm.config.maxKeys > 0 { // MaxKeys controls maximum key count owned by this node. // We need ownedPartitionCount property because every partition // manages itself independently. So if you set MaxKeys=70 and // your partition count is 7, every partition 10 keys at maximum. if st.Length > 0 && st.Length >= dm.config.maxKeys/int(ownedPartitionCount) { err := dm.evictKeyWithLRU(e) if err != nil { return err } } } if dm.config.maxInuse > 0 { // MaxInuse controls maximum in-use memory of partitions on this node. // We need ownedPartitionCount property because every partition // manages itself independently. So if you set MaxInuse=70M(in bytes) and // your partition count is 7, every partition consumes 10M in-use space at maximum. // WARNING: Actual allocated memory can be different. if st.Inuse > 0 && st.Inuse >= dm.config.maxInuse/int(ownedPartitionCount) { err := dm.evictKeyWithLRU(e) if err != nil { return err } } } return nil } func (dm *DMap) checkPutConditions(e *env) error { // Only set the key if it does not already exist. if e.putConfig.HasNX { ttl, err := e.fragment.storage.GetTTL(e.hkey) if err == nil { if !isKeyExpired(ttl) { return ErrKeyFound } } if errors.Is(err, storage.ErrKeyNotFound) { err = nil } if err != nil { return err } } // Only set the key if it already exists. if e.putConfig.HasXX && !e.fragment.storage.Check(e.hkey) { ttl, err := e.fragment.storage.GetTTL(e.hkey) if err == nil { if isKeyExpired(ttl) { return ErrKeyNotFound } } if errors.Is(err, storage.ErrKeyNotFound) { err = ErrKeyNotFound } if err != nil { return err } } return nil } func (dm *DMap) putOnCluster(e *env) error { part := dm.getPartitionByHKey(e.hkey, partitions.PRIMARY) f, err := dm.loadOrCreateFragment(part) if err != nil { return err } e.fragment = f f.Lock() defer f.Unlock() if err = dm.checkPutConditions(e); err != nil { return err } if dm.config != nil { if dm.config.ttlDuration.Seconds() != 0 && e.timeout.Seconds() == 0 { e.timeout = dm.config.ttlDuration } if dm.config.evictionPolicy == config.LRUEviction { if err = dm.setLRUEvictionStats(e); err != nil { return err } } } nt := dm.prepareEntry(e) if dm.s.config.ReplicaCount > config.MinimumReplicaCount { switch dm.s.config.ReplicationMode { case config.AsyncReplicationMode: // Fire and forget mode. Calls PutBackup command in different goroutines // and stores the key/value pair on local storage instance. return dm.asyncPutOnCluster(e, nt) case config.SyncReplicationMode: // Quorum based replication. return dm.syncPutOnCluster(e, nt) default: return fmt.Errorf("invalid replication mode: %v", dm.s.config.ReplicationMode) } } // single replica return dm.putEntryOnFragment(e, nt) } func (dm *DMap) writePutCommand(e *env) (*redis.StatusCmd, error) { cmd := protocol.NewPut(e.dmap, e.key, e.value) switch { case e.putConfig.HasEX: cmd.SetEX(e.putConfig.EX.Seconds()) case e.putConfig.HasPX: cmd.SetPX(e.putConfig.PX.Milliseconds()) case e.putConfig.HasEXAT: cmd.SetEXAT(e.putConfig.EXAT.Seconds()) case e.putConfig.HasPXAT: cmd.SetPXAT(e.putConfig.PXAT.Milliseconds()) } switch { case e.putConfig.HasNX: cmd.SetNX() case e.putConfig.HasXX: cmd.SetXX() } return cmd.Command(dm.s.ctx), nil } // put controls every write operation in Olric. It redirects the requests to its owner, // if the key belongs to another host. func (dm *DMap) put(e *env) error { e.hkey = partitions.HKey(e.dmap, e.key) member := dm.s.primary.PartitionByHKey(e.hkey).Owner() if member.CompareByName(dm.s.rt.This()) { // We are on the partition owner. return dm.putOnCluster(e) } // Redirect to the partition owner. cmd, err := dm.writePutCommand(e) if err != nil { return err } rc := dm.s.client.Get(member.String()) err = rc.Process(e.ctx, cmd) if err != nil { return protocol.ConvertError(err) } return protocol.ConvertError(cmd.Err()) } type PutConfig struct { HasEX bool EX time.Duration HasPX bool PX time.Duration HasEXAT bool EXAT time.Duration HasPXAT bool PXAT time.Duration HasNX bool HasXX bool OnlyUpdateTTL bool } // Put sets the value for the given key. It overwrites any previous value // for that key, and it's thread-safe. The key has to be a string. value type // is arbitrary. It is safe to modify the contents of the arguments after // Put returns but not before. func (dm *DMap) Put(ctx context.Context, key string, value interface{}, cfg *PutConfig) error { valueBuf := pool.Get() defer pool.Put(valueBuf) enc := resp.New(valueBuf) err := enc.Encode(value) if err != nil { return err } if cfg == nil { cfg = &PutConfig{} } e := newEnv(ctx) e.putConfig = cfg e.dmap = dm.name e.key = key e.value = make([]byte, valueBuf.Len()) copy(e.value[:], valueBuf.Bytes()) return dm.put(e) } ================================================ FILE: internal/dmap/put_handlers.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "time" "github.com/olric-data/olric/internal/cluster/partitions" "github.com/olric-data/olric/internal/protocol" "github.com/tidwall/redcon" ) func (s *Service) putCommandHandler(conn redcon.Conn, cmd redcon.Command) { putCmd, err := protocol.ParsePutCommand(cmd) if err != nil { protocol.WriteError(conn, err) return } dm, err := s.getOrCreateDMap(putCmd.DMap) if err != nil { protocol.WriteError(conn, err) return } var pc PutConfig switch { case putCmd.NX: pc.HasNX = true case putCmd.XX: pc.HasXX = true } switch { case putCmd.EX != 0: pc.HasEX = true pc.EX = time.Duration(putCmd.EX * float64(time.Second)) case putCmd.PX != 0: pc.HasPX = true pc.PX = time.Duration(putCmd.PX * int64(time.Millisecond)) case putCmd.EXAT != 0: pc.HasEXAT = true pc.EXAT = time.Duration(putCmd.EXAT * float64(time.Second)) case putCmd.PXAT != 0: pc.HasPXAT = true pc.PXAT = time.Duration(putCmd.PXAT * int64(time.Millisecond)) } e := newEnv(s.ctx) e.putConfig = &pc e.dmap = putCmd.DMap e.key = putCmd.Key e.value = putCmd.Value err = dm.put(e) if err != nil { protocol.WriteError(conn, err) return } conn.WriteString(protocol.StatusOK) } func (s *Service) putEntryCommandHandler(conn redcon.Conn, cmd redcon.Command) { putEntryCmd, err := protocol.ParsePutEntryCommand(cmd) if err != nil { protocol.WriteError(conn, err) return } dm, err := s.getOrCreateDMap(putEntryCmd.DMap) if err != nil { protocol.WriteError(conn, err) return } e := newEnv(s.ctx) e.hkey = partitions.HKey(putEntryCmd.DMap, putEntryCmd.Key) e.dmap = putEntryCmd.DMap e.key = putEntryCmd.Key e.value = putEntryCmd.Value err = dm.putOnReplicaFragment(e) if err != nil { protocol.WriteError(conn, err) return } conn.WriteString(protocol.StatusOK) } ================================================ FILE: internal/dmap/put_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "context" "encoding/hex" "errors" "math/rand" "testing" "time" "github.com/olric-data/olric/config" "github.com/olric-data/olric/internal/cluster/partitions" "github.com/olric-data/olric/internal/testcluster" "github.com/olric-data/olric/internal/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestDMap_Put_Standalone(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() ctx := context.Background() dm, err := s.NewDMap("mydmap") require.NoError(t, err) for i := 0; i < 10; i++ { err = dm.Put(ctx, testutil.ToKey(i), testutil.ToVal(i), nil) require.NoError(t, err) } for i := 0; i < 10; i++ { gr, err := dm.Get(ctx, testutil.ToKey(i)) require.NoError(t, err) require.Equal(t, testutil.ToVal(i), gr.Value()) } } func TestDMap_Put_Cluster(t *testing.T) { cluster := testcluster.New(NewService) s1 := cluster.AddMember(nil).(*Service) s2 := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() ctx := context.Background() dm1, err := s1.NewDMap("mydmap") require.NoError(t, err) for i := 0; i < 10; i++ { err = dm1.Put(ctx, testutil.ToKey(i), testutil.ToVal(i), nil) require.NoError(t, err) } dm2, err := s2.NewDMap("mydmap") require.NoError(t, err) for i := 0; i < 10; i++ { gr, err := dm2.Get(ctx, testutil.ToKey(i)) require.NoError(t, err) require.Equal(t, testutil.ToVal(i), gr.Value()) } } func TestDMap_Put_AsyncReplicationMode(t *testing.T) { cluster := testcluster.New(NewService) // Create DMap services with custom configuration c1 := testutil.NewConfig() c1.ReplicationMode = config.AsyncReplicationMode e1 := testcluster.NewEnvironment(c1) s1 := cluster.AddMember(e1).(*Service) c2 := testutil.NewConfig() c2.ReplicationMode = config.AsyncReplicationMode e2 := testcluster.NewEnvironment(c2) s2 := cluster.AddMember(e2).(*Service) defer cluster.Shutdown() ctx := context.Background() dm, err := s1.NewDMap("mydmap") require.NoError(t, err) for i := 0; i < 10; i++ { err = dm.Put(ctx, testutil.ToKey(i), testutil.ToVal(i), nil) require.NoError(t, err) } // Wait some time for async replication <-time.After(100 * time.Millisecond) dm2, err := s2.NewDMap("mydmap") require.NoError(t, err) for i := 0; i < 10; i++ { gr, err := dm2.Get(ctx, testutil.ToKey(i)) require.NoError(t, err) require.Equal(t, testutil.ToVal(i), gr.Value()) } } func TestDMap_Put_WriteQuorum(t *testing.T) { cluster := testcluster.New(NewService) // Create DMap services with custom configuration c1 := testutil.NewConfig() c1.ReplicaCount = 2 c1.WriteQuorum = 2 e1 := testcluster.NewEnvironment(c1) s1 := cluster.AddMember(e1).(*Service) defer cluster.Shutdown() ctx := context.Background() var hit bool dm, err := s1.NewDMap("mydmap") require.NoError(t, err) for i := 0; i < 10; i++ { key := testutil.ToKey(i) hkey := partitions.HKey(dm.name, key) host := dm.s.primary.PartitionByHKey(hkey).Owner() if s1.rt.This().CompareByID(host) { err = dm.Put(ctx, key, testutil.ToVal(i), nil) if err != ErrWriteQuorum { t.Fatalf("Expected ErrWriteQuorum. Got: %v", err) } hit = true } } if !hit { t.Fatalf("No keys checked on %v", s1) } } func TestDMap_Put_PX(t *testing.T) { cluster := testcluster.New(NewService) s1 := cluster.AddMember(nil).(*Service) s2 := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() ctx := context.Background() dm1, err := s1.NewDMap("mydmap") require.NoError(t, err) pc := &PutConfig{ HasPX: true, PX: time.Millisecond, } for i := 0; i < 10; i++ { err = dm1.Put(ctx, testutil.ToKey(i), testutil.ToVal(i), pc) require.NoError(t, err) } <-time.After(10 * time.Millisecond) dm2, err := s2.NewDMap("mydmap") require.NoError(t, err) for i := 0; i < 10; i++ { _, err := dm2.Get(ctx, testutil.ToKey(i)) if err != ErrKeyNotFound { t.Fatalf("Expected ErrKeyNotFound. Got: %v", err) } } } func TestDMap_Put_NX(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() ctx := context.Background() dm, err := s.NewDMap("mydmap") require.NoError(t, err) for i := 0; i < 10; i++ { err = dm.Put(ctx, testutil.ToKey(i), testutil.ToVal(i), nil) require.NoError(t, err) } pc := &PutConfig{ HasNX: true, } for i := 0; i < 10; i++ { err = dm.Put(ctx, testutil.ToKey(i), testutil.ToVal(i*2), pc) if err == ErrKeyFound { err = nil } require.NoError(t, err) } for i := 0; i < 10; i++ { gr, err := dm.Get(ctx, testutil.ToKey(i)) require.NoError(t, err) require.Equal(t, testutil.ToVal(i), gr.Value()) } } func TestDMap_Put_XX(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() ctx := context.Background() dm, err := s.NewDMap("mydmap") require.NoError(t, err) pc := &PutConfig{ HasXX: true, } for i := 0; i < 10; i++ { err = dm.Put(ctx, testutil.ToKey(i), testutil.ToVal(i*2), pc) if errors.Is(err, ErrKeyNotFound) { err = nil } require.NoError(t, err) } for i := 0; i < 10; i++ { _, err = dm.Get(ctx, testutil.ToKey(i)) if !errors.Is(err, ErrKeyNotFound) { t.Fatalf("Expected ErrKeyNotFound. Got: %v", err) } } } func TestDMap_Put_EX(t *testing.T) { cluster := testcluster.New(NewService) s1 := cluster.AddMember(nil).(*Service) s2 := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() ctx := context.Background() dm1, err := s1.NewDMap("mydmap") require.NoError(t, err) pc := &PutConfig{ HasEX: true, EX: time.Second / 4, } for i := 0; i < 10; i++ { err = dm1.Put(ctx, testutil.ToKey(i), testutil.ToVal(i), pc) require.NoError(t, err) } <-time.After(time.Second) dm2, err := s2.NewDMap("mydmap") require.NoError(t, err) for i := 0; i < 10; i++ { _, err := dm2.Get(ctx, testutil.ToKey(i)) if err != ErrKeyNotFound { t.Fatalf("Expected ErrKeyNotFound. Got: %v", err) } } } func TestDMap_Put_EXAT(t *testing.T) { cluster := testcluster.New(NewService) s1 := cluster.AddMember(nil).(*Service) s2 := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() ctx := context.Background() dm1, err := s1.NewDMap("mydmap") require.NoError(t, err) pc := &PutConfig{ HasEXAT: true, EXAT: time.Duration(time.Now().Add(time.Second).UnixNano()), } for i := 0; i < 10; i++ { err = dm1.Put(ctx, testutil.ToKey(i), testutil.ToVal(i), pc) require.NoError(t, err) } <-time.After(time.Second) dm2, err := s2.NewDMap("mydmap") require.NoError(t, err) for i := 0; i < 10; i++ { _, err := dm2.Get(ctx, testutil.ToKey(i)) require.ErrorIs(t, err, ErrKeyNotFound) } } func TestDMap_Put_PXAT(t *testing.T) { cluster := testcluster.New(NewService) s1 := cluster.AddMember(nil).(*Service) s2 := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() ctx := context.Background() dm1, err := s1.NewDMap("mydmap") require.NoError(t, err) pc := &PutConfig{ HasPXAT: true, PXAT: time.Duration(time.Now().Add(time.Millisecond).UnixNano()), } for i := 0; i < 10; i++ { err = dm1.Put(ctx, testutil.ToKey(i), testutil.ToVal(i), pc) require.NoError(t, err) } <-time.After(10 * time.Millisecond) dm2, err := s2.NewDMap("mydmap") require.NoError(t, err) for i := 0; i < 10; i++ { _, err := dm2.Get(ctx, testutil.ToKey(i)) require.ErrorIs(t, err, ErrKeyNotFound) } } func TestDMap_Put_ErrKeyTooLarge(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() ctx := context.Background() dm, err := s.NewDMap("mydmap") require.NoError(t, err) data := make([]byte, 300) _, err = rand.Read(data) require.NoError(t, err) key := hex.EncodeToString(data) err = dm.Put(ctx, key, "value", nil) require.ErrorIs(t, err, ErrKeyTooLarge) } func TestDMap_Put_ErrEntryTooLarge(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() ctx := context.Background() dm, err := s.NewDMap("mydmap") require.NoError(t, err) data := make([]byte, 1<<21) _, err = rand.Read(data) require.NoError(t, err) err = dm.Put(ctx, "key", data, nil) require.ErrorIs(t, err, ErrEntryTooLarge) } func TestDMap_Put_PX_With_NX(t *testing.T) { cluster := testcluster.New(NewService) s1 := cluster.AddMember(nil).(*Service) s2 := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() ctx := context.Background() dm1, err := s1.NewDMap("mydmap") require.NoError(t, err) pc := &PutConfig{ HasPX: true, PX: time.Minute, HasNX: true, } for i := range 10 { err = dm1.Put(ctx, testutil.ToKey(i), testutil.ToVal(i), pc) require.NoError(t, err) } <-time.After(10 * time.Millisecond) dm2, err := s2.NewDMap("mydmap") require.NoError(t, err) for i := range 10 { gr, err := dm2.Get(ctx, testutil.ToKey(i)) require.NoError(t, err) assert.NotZero(t, gr.TTL()) } } ================================================ FILE: internal/dmap/scan_handlers.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "strconv" "github.com/olric-data/olric/internal/cluster/partitions" "github.com/olric-data/olric/internal/protocol" "github.com/olric-data/olric/pkg/storage" "github.com/tidwall/redcon" ) func (dm *DMap) scanOnFragment(f *fragment, cursor uint64, sc *ScanConfig) ([]string, uint64, error) { f.Lock() defer f.Unlock() var items []string var err error if sc.HasMatch { cursor, err = f.storage.ScanRegexMatch(cursor, sc.Match, sc.Count, func(e storage.Entry) bool { items = append(items, e.Key()) return true }) if err != nil { return nil, 0, err } return items, cursor, nil } cursor, err = f.storage.Scan(cursor, sc.Count, func(e storage.Entry) bool { items = append(items, e.Key()) return true }) if err != nil { return nil, 0, err } return items, cursor, nil } func (dm *DMap) Scan(partID, cursor uint64, sc *ScanConfig) ([]string, uint64, error) { var part *partitions.Partition if sc.Replica { part = dm.s.backup.PartitionByID(partID) } else { part = dm.s.primary.PartitionByID(partID) } f, err := dm.loadFragment(part) if err == errFragmentNotFound { return nil, 0, nil } if err != nil { return nil, 0, err } return dm.scanOnFragment(f, cursor, sc) } type ScanConfig struct { HasCount bool Count int HasMatch bool Match string Replica bool } type ScanOption func(*ScanConfig) func Count(c int) ScanOption { return func(cfg *ScanConfig) { cfg.HasCount = true cfg.Count = c } } func Match(s string) ScanOption { return func(cfg *ScanConfig) { cfg.HasMatch = true cfg.Match = s } } func (s *Service) scanCommandHandler(conn redcon.Conn, cmd redcon.Command) { scanCmd, err := protocol.ParseScanCommand(cmd) if err != nil { protocol.WriteError(conn, err) return } dm, err := s.getOrCreateDMap(scanCmd.DMap) if err != nil { protocol.WriteError(conn, err) return } var sc ScanConfig var options []ScanOption options = append(options, Count(scanCmd.Count)) if scanCmd.Match != "" { options = append(options, Match(scanCmd.Match)) } for _, opt := range options { opt(&sc) } sc.Replica = scanCmd.Replica var result []string var cursor uint64 result, cursor, err = dm.Scan(scanCmd.PartID, scanCmd.Cursor, &sc) if err != nil { protocol.WriteError(conn, err) return } conn.WriteArray(2) conn.WriteBulkString(strconv.FormatUint(cursor, 10)) conn.WriteArray(len(result)) for _, i := range result { conn.WriteBulkString(i) } } ================================================ FILE: internal/dmap/scan_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "context" "fmt" "testing" "github.com/olric-data/olric/internal/protocol" "github.com/olric-data/olric/internal/testcluster" "github.com/olric-data/olric/internal/testutil" "github.com/stretchr/testify/require" ) func testScanIterator(t *testing.T, s *Service, allKeys map[string]bool, sc *ScanConfig) int { if sc == nil { sc = &ScanConfig{} } ctx := context.Background() rc := s.client.Get(s.rt.This().String()) var totalKeys int var partID, cursor uint64 for { r := protocol.NewScan(partID, "mydmap", cursor) if sc.Replica { r.SetReplica() } if sc.HasMatch { r.SetMatch(sc.Match) } cmd := r.Command(ctx) err := rc.Process(ctx, cmd) require.NoError(t, err) var keys []string keys, cursor, err = cmd.Result() require.NoError(t, err) totalKeys += len(keys) for _, key := range keys { _, ok := allKeys[key] require.True(t, ok) allKeys[key] = true } if cursor == 0 { if partID+1 < s.config.PartitionCount { partID++ continue } break } } return totalKeys } func TestDMap_scanCommandHandler_Standalone(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() ctx := context.Background() dm, err := s.NewDMap("mydmap") require.NoError(t, err) allKeys := make(map[string]bool) for i := 0; i < 100; i++ { err = dm.Put(ctx, testutil.ToKey(i), i, nil) require.NoError(t, err) allKeys[testutil.ToKey(i)] = false } totalKeys := testScanIterator(t, s, allKeys, nil) require.Equal(t, 100, totalKeys) for _, value := range allKeys { require.True(t, value) } } func TestDMap_scanCommandHandler_Cluster(t *testing.T) { cluster := testcluster.New(NewService) c1 := testutil.NewConfig() c1.ReplicaCount = 2 c1.WriteQuorum = 2 e1 := testcluster.NewEnvironment(c1) s1 := cluster.AddMember(e1).(*Service) c2 := testutil.NewConfig() c2.ReplicaCount = 2 c1.WriteQuorum = 2 e2 := testcluster.NewEnvironment(c2) s2 := cluster.AddMember(e2).(*Service) defer cluster.Shutdown() ctx := context.Background() dm, err := s1.NewDMap("mydmap") require.NoError(t, err) allKeys := make(map[string]bool) for i := 0; i < 100; i++ { err = dm.Put(ctx, testutil.ToKey(i), i, nil) require.NoError(t, err) allKeys[testutil.ToKey(i)] = false } t.Run("Scan on primary copies", func(t *testing.T) { var totalKeys int totalKeys += testScanIterator(t, s1, allKeys, nil) totalKeys += testScanIterator(t, s2, allKeys, nil) require.Equal(t, 100, totalKeys) for _, value := range allKeys { require.True(t, value) } }) t.Run("Scan on replicas", func(t *testing.T) { var totalKeys int sc := &ScanConfig{Replica: true} totalKeys += testScanIterator(t, s1, allKeys, sc) totalKeys += testScanIterator(t, s2, allKeys, sc) require.Equal(t, 100, totalKeys) for _, value := range allKeys { require.True(t, value) } }) } func TestDMap_scanCommandHandler_match(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() dm, err := s.NewDMap("mydmap") require.NoError(t, err) ctx := context.Background() evenKeys := make(map[string]bool) for i := 0; i < 100; i++ { var key string if i%2 == 0 { key = fmt.Sprintf("even:%s", testutil.ToKey(i)) evenKeys[key] = false } else { key = fmt.Sprintf("odd:%s", testutil.ToKey(i)) } err = dm.Put(ctx, key, i, nil) require.NoError(t, err) } sc := &ScanConfig{ HasMatch: true, Match: "^even:", } totalKeys := testScanIterator(t, s, evenKeys, sc) require.Equal(t, 50, totalKeys) for _, value := range evenKeys { require.True(t, value) } } func TestDMap_scanCommandHandler_count(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() dm, err := s.NewDMap("mydmap") require.NoError(t, err) ctx := context.Background() for i := 0; i < 100; i++ { err = dm.Put(ctx, testutil.ToKey(i), i, nil) require.NoError(t, err) } rc := s.client.Get(s.rt.This().String()) var partID, cursor uint64 r := protocol.NewScan(partID, "mydmap", cursor) r.SetCount(5) cmd := r.Command(ctx) err = rc.Process(ctx, cmd) require.NoError(t, err) var keys []string keys, _, err = cmd.Result() require.NoError(t, err) require.Len(t, keys, 5) } ================================================ FILE: internal/dmap/service.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "context" "errors" "reflect" "sync" "github.com/olric-data/olric/config" "github.com/olric-data/olric/events" "github.com/olric-data/olric/internal/cluster/partitions" "github.com/olric-data/olric/internal/cluster/routingtable" "github.com/olric-data/olric/internal/environment" "github.com/olric-data/olric/internal/locker" "github.com/olric-data/olric/internal/protocol" "github.com/olric-data/olric/internal/server" "github.com/olric-data/olric/internal/service" "github.com/olric-data/olric/pkg/flog" "github.com/olric-data/olric/pkg/storage" ) var errFragmentNotFound = errors.New("fragment not found") type storageMap struct { engines map[string]storage.Engine configs map[string]map[string]interface{} } type Service struct { sync.RWMutex // protects dmaps map log *flog.Logger config *config.Config client *server.Client server *server.Server rt *routingtable.RoutingTable primary *partitions.Partitions backup *partitions.Partitions locker *locker.Locker dmaps map[string]*DMap storage *storageMap wg sync.WaitGroup ctx context.Context cancel context.CancelFunc } func registerErrors() { protocol.SetError("NOSUCHLOCK", ErrNoSuchLock) protocol.SetError("LOCKNOTACQUIRED", ErrLockNotAcquired) protocol.SetError("READQUORUM", ErrReadQuorum) protocol.SetError("WRITEQUORUM", ErrWriteQuorum) protocol.SetError("DMAPNOTFOUND", ErrDMapNotFound) protocol.SetError("KEYTOOLARGE", ErrKeyTooLarge) protocol.SetError("ENTRYTOOLARGE", ErrEntryTooLarge) protocol.SetError("KEYNOTFOUND", ErrKeyNotFound) protocol.SetError("KEYFOUND", ErrKeyFound) } func NewService(e *environment.Environment) (service.Service, error) { ctx, cancel := context.WithCancel(context.Background()) s := &Service{ config: e.Get("config").(*config.Config), client: e.Get("client").(*server.Client), server: e.Get("server").(*server.Server), log: e.Get("logger").(*flog.Logger), rt: e.Get("routingtable").(*routingtable.RoutingTable), primary: e.Get("primary").(*partitions.Partitions), backup: e.Get("backup").(*partitions.Partitions), locker: e.Get("locker").(*locker.Locker), storage: &storageMap{ engines: make(map[string]storage.Engine), configs: make(map[string]map[string]interface{}), }, dmaps: make(map[string]*DMap), ctx: ctx, cancel: cancel, } registerErrors() s.RegisterHandlers() return s, nil } func (s *Service) isAlive() bool { select { case <-s.ctx.Done(): // The node is gone. return false default: } return true } func getType(data interface{}) string { t := reflect.TypeOf(data) if t.Kind() == reflect.Ptr { return t.Elem().Name() } return t.Name() } func (s *Service) publishEvent(e events.Event) { defer s.wg.Done() rc := s.client.Get(s.rt.This().String()) data, err := e.Encode() if err != nil { s.log.V(3).Printf("[ERROR] Failed to encode %s: %v", getType(e), err) return } err = rc.Publish(s.ctx, events.ClusterEventsChannel, data).Err() if err != nil { s.log.V(3).Printf("[ERROR] Failed to publish %s to %s: %v", getType(e), events.ClusterEventsChannel, err) } } // Start starts the distributed map service. func (s *Service) Start() error { s.wg.Add(1) go s.janitorWorker() s.wg.Add(1) go s.compactionWorker() s.wg.Add(1) go s.evictKeysAtBackground() return nil } func (s *Service) Shutdown(ctx context.Context) error { s.cancel() done := make(chan struct{}) go func() { s.wg.Wait() close(done) }() select { case <-ctx.Done(): err := ctx.Err() if err != nil { return err } case <-done: } return nil } var _ service.Service = (*Service)(nil) ================================================ FILE: internal/dmap/service_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "context" "testing" "github.com/olric-data/olric/internal/testcluster" ) func TestDMapService(t *testing.T) { cluster := testcluster.New(NewService) defer cluster.Shutdown() s, ok := cluster.AddMember(nil).(*Service) if !ok { t.Fatal("AddMember returned a different service.Service implementation") } err := s.Shutdown(context.Background()) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } } ================================================ FILE: internal/dmap/stats_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dmap import ( "context" "testing" "time" "github.com/olric-data/olric/internal/testcluster" "github.com/olric-data/olric/internal/testutil" "github.com/stretchr/testify/require" ) func TestDMap_Stats(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() dm, err := s.NewDMap("mymap") if err != nil { t.Fatalf("Expected nil. Got: %v", err) } ctx := context.Background() // EntriesTotal for i := 0; i < 10; i++ { err = dm.Put(ctx, testutil.ToKey(i), testutil.ToVal(i), nil) require.NoError(t, err) } //GetHits for i := 0; i < 10; i++ { _, err = dm.Get(ctx, testutil.ToKey(i)) require.NoError(t, err) } // DeleteHits for i := 0; i < 10; i++ { _, err = dm.Delete(ctx, testutil.ToKey(i)) require.NoError(t, err) } // GetMisses for i := 0; i < 10; i++ { _, err = dm.Get(ctx, testutil.ToKey(i)) require.ErrorIs(t, err, ErrKeyNotFound) } // DeleteMisses for i := 0; i < 10; i++ { _, err = dm.Delete(ctx, testutil.ToKey(i)) require.NoError(t, err) } pc := &PutConfig{ HasEX: true, EX: time.Millisecond, } // EntriesTotal, EvictedTotal for i := 0; i < 10; i++ { err = dm.Put(ctx, testutil.ToKey(i), testutil.ToVal(i), pc) require.NoError(t, err) } <-time.After(100 * time.Millisecond) // GetMisses for i := 0; i < 10; i++ { _, err = dm.Get(ctx, testutil.ToKey(i)) require.ErrorIs(t, err, ErrKeyNotFound) } stats := map[string]int64{ "EntriesTotal": EntriesTotal.Read(), "GetMisses": GetMisses.Read(), "GetHits": GetHits.Read(), "DeleteHits": DeleteHits.Read(), "DeleteMisses": DeleteMisses.Read(), "EvictedTotal": EvictedTotal.Read(), } for name, value := range stats { if value <= 0 { t.Fatalf("Expected %s has to be bigger than zero", name) } } } ================================================ FILE: internal/environment/environment.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package environment import "sync" type Environment struct { sync.RWMutex m map[string]interface{} } func New() *Environment { return &Environment{ m: make(map[string]interface{}), } } func (e *Environment) Get(key string) interface{} { e.RLock() defer e.RUnlock() value, ok := e.m[key] if ok { return value } return nil } func (e *Environment) Set(key string, value interface{}) { e.Lock() defer e.Unlock() e.m[key] = value } func (e *Environment) Clone() *Environment { e.RLock() defer e.RUnlock() f := New() for key, value := range e.m { f.Set(key, value) } return f } ================================================ FILE: internal/environment/environment_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package environment import ( "testing" "github.com/stretchr/testify/require" ) type envTest struct { number uint64 } func TestEnvironment(t *testing.T) { e := New() st := &envTest{ number: 1988, } e.Set("my-struct", st) e.Set("my-string", "value") e.Set("my-uint64", uint64(4576)) structVal := e.Get("my-struct") require.Equal(t, uint64(1988), structVal.(*envTest).number) stringVal := e.Get("my-string") require.Equal(t, "value", stringVal) stringUint64 := e.Get("my-uint64") require.Equal(t, uint64(4576), stringUint64.(uint64)) t.Run("Clone", func(t *testing.T) { clone := e.Clone() require.Equal(t, e, clone) }) } ================================================ FILE: internal/locker/locker.go ================================================ /* Package locker provides a mechanism for creating finer-grained locking to help free up more global locks to handle other tasks. The implementation looks close to a sync.Mutex, however the user must provide a reference to use to refer to the underlying lock when locking and unlocking, and unlock may generate an error. If a lock with a given name does not exist when `Lock` is called, one is created. Lock references are automatically cleaned up on `Unlock` if nothing else is waiting for the lock. */ package locker import ( "errors" "sync" "sync/atomic" ) // ErrNoSuchLock is returned when the requested lock does not exist var ErrNoSuchLock = errors.New("no such lock") // Locker provides a locking mechanism based on the passed in reference name type Locker struct { mu sync.Mutex locks map[string]*lockCtr } // lockCtr is used by Locker to represent a lock with a given name. type lockCtr struct { mu sync.Mutex // waiters is the number of waiters waiting to acquire the lock // this is int32 instead of uint32 so we can add `-1` in `dec()` waiters int32 } // inc increments the number of waiters waiting for the lock func (l *lockCtr) inc() { atomic.AddInt32(&l.waiters, 1) } // dec decrements the number of waiters waiting on the lock func (l *lockCtr) dec() { atomic.AddInt32(&l.waiters, -1) } // count gets the current number of waiters func (l *lockCtr) count() int32 { return atomic.LoadInt32(&l.waiters) } // Lock locks the mutex func (l *lockCtr) Lock() { l.mu.Lock() } // Unlock unlocks the mutex func (l *lockCtr) Unlock() { l.mu.Unlock() } // New creates a new Locker func New() *Locker { return &Locker{ locks: make(map[string]*lockCtr), } } // Lock locks a mutex with the given name. If it doesn't exist, one is created func (l *Locker) Lock(name string) { l.mu.Lock() if l.locks == nil { l.locks = make(map[string]*lockCtr) } nameLock, exists := l.locks[name] if !exists { nameLock = &lockCtr{} l.locks[name] = nameLock } // increment the nameLock waiters while inside the main mutex // this makes sure that the lock isn't deleted if `Lock` and `Unlock` are called concurrently nameLock.inc() l.mu.Unlock() // Lock the nameLock outside the main mutex so we don't block other operations // once locked then we can decrement the number of waiters for this lock nameLock.Lock() nameLock.dec() } // Unlock unlocks the mutex with the given name // If the given lock is not being waited on by any other callers, it is deleted func (l *Locker) Unlock(name string) error { l.mu.Lock() nameLock, exists := l.locks[name] if !exists { l.mu.Unlock() return ErrNoSuchLock } if nameLock.count() == 0 { delete(l.locks, name) } nameLock.Unlock() l.mu.Unlock() return nil } ================================================ FILE: internal/locker/locker_test.go ================================================ package locker import ( "math/rand" "strconv" "sync" "testing" "time" ) func TestLockCounter(t *testing.T) { l := &lockCtr{} l.inc() if l.waiters != 1 { t.Fatal("counter inc failed") } l.dec() if l.waiters != 0 { t.Fatal("counter dec failed") } } func TestLockerLock(t *testing.T) { l := New() l.Lock("test") ctr := l.locks["test"] if ctr.count() != 0 { t.Fatalf("expected waiters to be 0, got :%d", ctr.waiters) } chDone := make(chan struct{}) go func() { l.Lock("test") close(chDone) }() chWaiting := make(chan struct{}) go func() { for range time.Tick(1 * time.Millisecond) { if ctr.count() == 1 { close(chWaiting) break } } }() select { case <-chWaiting: case <-time.After(3 * time.Second): t.Fatal("timed out waiting for lock waiters to be incremented") } select { case <-chDone: t.Fatal("lock should not have returned while it was still held") default: } if err := l.Unlock("test"); err != nil { t.Fatal(err) } select { case <-chDone: case <-time.After(3 * time.Second): t.Fatalf("lock should have completed") } if ctr.count() != 0 { t.Fatalf("expected waiters to be 0, got: %d", ctr.count()) } } func TestLockerUnlock(t *testing.T) { l := New() l.Lock("test") l.Unlock("test") chDone := make(chan struct{}) go func() { l.Lock("test") close(chDone) }() select { case <-chDone: case <-time.After(3 * time.Second): t.Fatalf("lock should not be blocked") } } func TestLockerConcurrency(t *testing.T) { l := New() var wg sync.WaitGroup for i := 0; i <= 10000; i++ { wg.Add(1) go func() { l.Lock("test") // if there is a concurrency issue, will very likely panic here l.Unlock("test") wg.Done() }() } chDone := make(chan struct{}) go func() { wg.Wait() close(chDone) }() select { case <-chDone: case <-time.After(10 * time.Second): t.Fatal("timeout waiting for locks to complete") } // Since everything has unlocked this should not exist anymore if ctr, exists := l.locks["test"]; exists { t.Fatalf("lock should not exist: %v", ctr) } } func BenchmarkLocker(b *testing.B) { l := New() for i := 0; i < b.N; i++ { l.Lock("test") l.Unlock("test") } } func BenchmarkLockerParallel(b *testing.B) { l := New() b.SetParallelism(128) b.RunParallel(func(pb *testing.PB) { for pb.Next() { l.Lock("test") l.Unlock("test") } }) } func BenchmarkLockerMoreKeys(b *testing.B) { l := New() var keys []string for i := 0; i < 64; i++ { keys = append(keys, strconv.Itoa(i)) } b.SetParallelism(128) b.RunParallel(func(pb *testing.PB) { for pb.Next() { k := keys[rand.Intn(len(keys))] l.Lock(k) l.Unlock(k) } }) } ================================================ FILE: internal/protocol/cluster.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package protocol import ( "context" "github.com/redis/go-redis/v9" "github.com/tidwall/redcon" ) type ClusterRoutingTable struct{} func NewClusterRoutingTable() *ClusterRoutingTable { return &ClusterRoutingTable{} } func (c *ClusterRoutingTable) Command(ctx context.Context) *redis.Cmd { var args []interface{} args = append(args, Cluster.RoutingTable) return redis.NewCmd(ctx, args...) } func ParseClusterRoutingTable(cmd redcon.Command) (*ClusterRoutingTable, error) { if len(cmd.Args) > 1 { return nil, errWrongNumber(cmd.Args) } c := NewClusterRoutingTable() return c, nil } type ClusterMembers struct{} func NewClusterMembers() *ClusterMembers { return &ClusterMembers{} } func (c *ClusterMembers) Command(ctx context.Context) *redis.Cmd { var args []interface{} args = append(args, Cluster.Members) return redis.NewCmd(ctx, args...) } func ParseClusterMembers(cmd redcon.Command) (*ClusterMembers, error) { if len(cmd.Args) > 1 { return nil, errWrongNumber(cmd.Args) } c := NewClusterMembers() return c, nil } ================================================ FILE: internal/protocol/cluster_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package protocol import ( "context" "testing" "github.com/stretchr/testify/require" ) func TestProtocol_ClusterRoutingTable(t *testing.T) { rtCmd := NewClusterRoutingTable() cmd := stringToCommand(rtCmd.Command(context.Background()).String()) _, err := ParseClusterRoutingTable(cmd) require.NoError(t, err) t.Run("CLUSTER.ROUTINGTABLE invalid command", func(t *testing.T) { cmd := stringToCommand("cluster routing table foobar") _, err = ParseClusterRoutingTable(cmd) require.Error(t, err) }) } func TestProtocol_ClusterMembers(t *testing.T) { membersCmd := NewClusterMembers() cmd := stringToCommand(membersCmd.Command(context.Background()).String()) _, err := ParseClusterMembers(cmd) require.NoError(t, err) t.Run("CLUSTER.MEMBERS invalid command", func(t *testing.T) { cmd := stringToCommand("cluster members foobar") _, err = ParseClusterMembers(cmd) require.Error(t, err) }) } ================================================ FILE: internal/protocol/commands.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package protocol const StatusOK = "OK" type ClusterCommands struct { RoutingTable string Members string } var Cluster = &ClusterCommands{ RoutingTable: "cluster.routingtable", Members: "cluster.members", } type InternalCommands struct { MoveFragment string UpdateRouting string LengthOfPart string ClusterRoutingTable string } var Internal = &InternalCommands{ MoveFragment: "internal.node.movefragment", UpdateRouting: "internal.node.updaterouting", LengthOfPart: "internal.node.lengthofpart", } type GenericCommands struct { Ping string Stats string Auth string } var Generic = &GenericCommands{ Ping: "ping", Stats: "stats", Auth: "auth", } type DMapCommands struct { Get string GetEntry string Put string PutEntry string Del string DelEntry string Expire string PExpire string Destroy string Query string Incr string Decr string GetPut string IncrByFloat string Lock string Unlock string LockLease string PLockLease string Scan string } var DMap = &DMapCommands{ Get: "dm.get", GetEntry: "dm.getentry", Put: "dm.put", PutEntry: "dm.putentry", Del: "dm.del", DelEntry: "dm.delentry", Expire: "dm.expire", PExpire: "dm.pexpire", Destroy: "dm.destroy", Incr: "dm.incr", Decr: "dm.decr", GetPut: "dm.getput", IncrByFloat: "dm.incrbyfloat", Lock: "dm.lock", Unlock: "dm.unlock", LockLease: "dm.locklease", PLockLease: "dm.plocklease", Scan: "dm.scan", } type PubSubCommands struct { PubSub string Publish string PublishInternal string Subscribe string PSubscribe string PubSubChannels string PubSubNumpat string PubSubNumsub string } var PubSub = &PubSubCommands{ PubSub: "pubsub", Publish: "publish", PublishInternal: "publish.internal", Subscribe: "subscribe", PSubscribe: "psubscribe", PubSubChannels: "pubsub channels", PubSubNumpat: "pubsub numpat", PubSubNumsub: "pubsub numsub", } ================================================ FILE: internal/protocol/dmap.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package protocol import ( "context" "errors" "fmt" "strconv" "strings" "time" "github.com/olric-data/olric/internal/util" "github.com/redis/go-redis/v9" "github.com/tidwall/redcon" ) type Put struct { DMap string Key string Value []byte EX float64 PX int64 EXAT float64 PXAT int64 NX bool XX bool } func NewPut(dmap, key string, value []byte) *Put { return &Put{ DMap: dmap, Key: key, Value: value, } } func (p *Put) SetEX(ex float64) *Put { p.EX = ex return p } func (p *Put) SetPX(px int64) *Put { p.PX = px return p } func (p *Put) SetEXAT(exat float64) *Put { p.EXAT = exat return p } func (p *Put) SetPXAT(pxat int64) *Put { p.PXAT = pxat return p } func (p *Put) SetNX() *Put { p.NX = true return p } func (p *Put) SetXX() *Put { p.XX = true return p } func (p *Put) Command(ctx context.Context) *redis.StatusCmd { var args []interface{} args = append(args, DMap.Put) args = append(args, p.DMap) args = append(args, p.Key) args = append(args, p.Value) if p.EX != 0 { args = append(args, "EX") args = append(args, p.EX) } if p.PX != 0 { args = append(args, "PX") args = append(args, p.PX) } if p.EXAT != 0 { args = append(args, "EXAT") args = append(args, p.EXAT) } if p.PXAT != 0 { args = append(args, "PXAT") args = append(args, p.PXAT) } if p.NX { args = append(args, "NX") } if p.XX { args = append(args, "XX") } return redis.NewStatusCmd(ctx, args...) } func ParsePutCommand(cmd redcon.Command) (*Put, error) { if len(cmd.Args) < 4 { return nil, errWrongNumber(cmd.Args) } p := NewPut( util.BytesToString(cmd.Args[1]), // DMap util.BytesToString(cmd.Args[2]), // Key cmd.Args[3], // Value ) args := cmd.Args[4:] for len(args) > 0 { switch arg := strings.ToUpper(util.BytesToString(args[0])); arg { case "NX": p.SetNX() args = args[1:] continue case "XX": p.SetXX() args = args[1:] continue case "PX": px, err := strconv.ParseInt(util.BytesToString(args[1]), 10, 64) if err != nil { return nil, err } p.SetPX(px) args = args[2:] continue case "EX": ex, err := strconv.ParseFloat(util.BytesToString(args[1]), 64) if err != nil { return nil, err } p.SetEX(ex) args = args[2:] continue case "EXAT": exat, err := strconv.ParseFloat(util.BytesToString(args[1]), 64) if err != nil { return nil, err } p.SetEXAT(exat) args = args[2:] continue case "PXAT": pxat, err := strconv.ParseInt(util.BytesToString(args[1]), 10, 64) if err != nil { return nil, err } p.SetPXAT(pxat) args = args[2:] continue default: return nil, errors.New("syntax error") } } return p, nil } type PutEntry struct { DMap string Key string Value []byte } func NewPutEntry(dmap, key string, value []byte) *PutEntry { return &PutEntry{ DMap: dmap, Key: key, Value: value, } } func (p *PutEntry) Command(ctx context.Context) *redis.StatusCmd { var args []interface{} args = append(args, DMap.PutEntry) args = append(args, p.DMap) args = append(args, p.Key) args = append(args, p.Value) return redis.NewStatusCmd(ctx, args...) } func ParsePutEntryCommand(cmd redcon.Command) (*PutEntry, error) { if len(cmd.Args) < 4 { return nil, errWrongNumber(cmd.Args) } return NewPutEntry( util.BytesToString(cmd.Args[1]), util.BytesToString(cmd.Args[2]), cmd.Args[3], ), nil } type Get struct { DMap string Key string Raw bool } func NewGet(dmap, key string) *Get { return &Get{ DMap: dmap, Key: key, } } func (g *Get) SetRaw() *Get { g.Raw = true return g } func (g *Get) Command(ctx context.Context) *redis.StringCmd { var args []interface{} args = append(args, DMap.Get) args = append(args, g.DMap) args = append(args, g.Key) if g.Raw { args = append(args, "RW") } return redis.NewStringCmd(ctx, args...) } func ParseGetCommand(cmd redcon.Command) (*Get, error) { if len(cmd.Args) < 3 { return nil, errWrongNumber(cmd.Args) } g := NewGet( util.BytesToString(cmd.Args[1]), util.BytesToString(cmd.Args[2]), ) if len(cmd.Args) == 4 { arg := util.BytesToString(cmd.Args[3]) if arg == "RW" { g.SetRaw() } else { return nil, fmt.Errorf("%w: %s", ErrInvalidArgument, arg) } } return g, nil } type GetEntry struct { DMap string Key string Replica bool } func NewGetEntry(dmap, key string) *GetEntry { return &GetEntry{ DMap: dmap, Key: key, } } func (g *GetEntry) SetReplica() *GetEntry { g.Replica = true return g } func (g *GetEntry) Command(ctx context.Context) *redis.StringCmd { var args []interface{} args = append(args, DMap.GetEntry) args = append(args, g.DMap) args = append(args, g.Key) if g.Replica { args = append(args, "RC") } return redis.NewStringCmd(ctx, args...) } func ParseGetEntryCommand(cmd redcon.Command) (*GetEntry, error) { if len(cmd.Args) < 2 { return nil, errWrongNumber(cmd.Args) } g := NewGetEntry( util.BytesToString(cmd.Args[1]), // DMap util.BytesToString(cmd.Args[2]), // Key ) if len(cmd.Args) == 4 { arg := util.BytesToString(cmd.Args[3]) if arg == "RC" { g.SetReplica() } else { return nil, fmt.Errorf("%w: %s", ErrInvalidArgument, arg) } } return g, nil } type Del struct { DMap string Keys []string } func NewDel(dmap string, keys ...string) *Del { return &Del{ DMap: dmap, Keys: keys, } } func (d *Del) Command(ctx context.Context) *redis.IntCmd { var args []interface{} args = append(args, DMap.Del) args = append(args, d.DMap) for _, key := range d.Keys { args = append(args, key) } return redis.NewIntCmd(ctx, args...) } func ParseDelCommand(cmd redcon.Command) (*Del, error) { if len(cmd.Args) < 3 { return nil, errWrongNumber(cmd.Args) } d := NewDel( util.BytesToString(cmd.Args[1]), ) for _, key := range cmd.Args[2:] { d.Keys = append(d.Keys, util.BytesToString(key)) } return d, nil } type DelEntry struct { Del *Del Replica bool } func NewDelEntry(dmap, key string) *DelEntry { return &DelEntry{ Del: NewDel(dmap, key), } } func (d *DelEntry) SetReplica() *DelEntry { d.Replica = true return d } func (d *DelEntry) Command(ctx context.Context) *redis.IntCmd { cmd := d.Del.Command(ctx) args := cmd.Args() args[0] = DMap.DelEntry if d.Replica { args = append(args, "RC") } return redis.NewIntCmd(ctx, args...) } func ParseDelEntryCommand(cmd redcon.Command) (*DelEntry, error) { if len(cmd.Args) < 3 { return nil, errWrongNumber(cmd.Args) } d := NewDelEntry( util.BytesToString(cmd.Args[1]), util.BytesToString(cmd.Args[2]), ) if len(cmd.Args) == 4 { arg := util.BytesToString(cmd.Args[3]) if arg == "RC" { d.SetReplica() } else { return nil, fmt.Errorf("%w: %s", ErrInvalidArgument, arg) } } return d, nil } type PExpire struct { DMap string Key string Milliseconds time.Duration } func NewPExpire(dmap, key string, milliseconds time.Duration) *PExpire { return &PExpire{ DMap: dmap, Key: key, Milliseconds: milliseconds, } } func (p *PExpire) Command(ctx context.Context) *redis.StatusCmd { var args []interface{} args = append(args, DMap.PExpire) args = append(args, p.DMap) args = append(args, p.Key) args = append(args, p.Milliseconds.Milliseconds()) return redis.NewStatusCmd(ctx, args...) } func ParsePExpireCommand(cmd redcon.Command) (*PExpire, error) { if len(cmd.Args) < 4 { return nil, errWrongNumber(cmd.Args) } rawMilliseconds := util.BytesToString(cmd.Args[3]) milliseconds, err := strconv.ParseInt(rawMilliseconds, 10, 64) if err != nil { return nil, err } p := NewPExpire( util.BytesToString(cmd.Args[1]), // DMap util.BytesToString(cmd.Args[2]), // Key time.Duration(milliseconds*int64(time.Millisecond)), ) return p, nil } type Expire struct { DMap string Key string Seconds time.Duration } func NewExpire(dmap, key string, seconds time.Duration) *Expire { return &Expire{ DMap: dmap, Key: key, Seconds: seconds, } } func (e *Expire) Command(ctx context.Context) *redis.StatusCmd { var args []interface{} args = append(args, DMap.Expire) args = append(args, e.DMap) args = append(args, e.Key) args = append(args, e.Seconds.Seconds()) return redis.NewStatusCmd(ctx, args...) } func ParseExpireCommand(cmd redcon.Command) (*Expire, error) { if len(cmd.Args) < 4 { return nil, errWrongNumber(cmd.Args) } rawSeconds := util.BytesToString(cmd.Args[3]) seconds, err := strconv.ParseFloat(rawSeconds, 64) if err != nil { return nil, err } e := NewExpire( util.BytesToString(cmd.Args[1]), // DMap util.BytesToString(cmd.Args[2]), // Key time.Duration(seconds*float64(time.Second)), ) return e, nil } type Destroy struct { DMap string Local bool } func NewDestroy(dmap string) *Destroy { return &Destroy{ DMap: dmap, } } func (d *Destroy) SetLocal() *Destroy { d.Local = true return d } func (d *Destroy) Command(ctx context.Context) *redis.StatusCmd { var args []interface{} args = append(args, DMap.Destroy) args = append(args, d.DMap) if d.Local { args = append(args, "LC") } return redis.NewStatusCmd(ctx, args...) } func ParseDestroyCommand(cmd redcon.Command) (*Destroy, error) { if len(cmd.Args) < 2 { return nil, errWrongNumber(cmd.Args) } d := NewDestroy( util.BytesToString(cmd.Args[1]), ) if len(cmd.Args) == 3 { arg := util.BytesToString(cmd.Args[2]) if arg == "LC" { d.SetLocal() } else { return nil, fmt.Errorf("%w: %s", ErrInvalidArgument, arg) } } return d, nil } type Scan struct { PartID uint64 DMap string Cursor uint64 Count int Match string Replica bool } func NewScan(partID uint64, dmap string, cursor uint64) *Scan { return &Scan{ PartID: partID, DMap: dmap, Cursor: cursor, } } func (s *Scan) SetMatch(match string) *Scan { s.Match = match return s } func (s *Scan) SetCount(count int) *Scan { s.Count = count return s } func (s *Scan) SetReplica() *Scan { s.Replica = true return s } func (s *Scan) Command(ctx context.Context) *redis.ScanCmd { var args []interface{} args = append(args, DMap.Scan) args = append(args, s.PartID) args = append(args, s.DMap) args = append(args, s.Cursor) if s.Match != "" { args = append(args, "MATCH") args = append(args, s.Match) } if s.Count != 0 { args = append(args, "COUNT") args = append(args, s.Count) } if s.Replica { args = append(args, "RC") } return redis.NewScanCmd(ctx, nil, args...) } const DefaultScanCount = 10 func ParseScanCommand(cmd redcon.Command) (*Scan, error) { if len(cmd.Args) < 4 { return nil, errWrongNumber(cmd.Args) } rawPartID := util.BytesToString(cmd.Args[1]) partID, err := strconv.ParseUint(rawPartID, 10, 64) if err != nil { return nil, err } rawCursor := util.BytesToString(cmd.Args[3]) cursor, err := strconv.ParseUint(rawCursor, 10, 64) if err != nil { return nil, err } s := NewScan( partID, util.BytesToString(cmd.Args[2]), // DMap cursor, ) args := cmd.Args[4:] for len(args) > 0 { switch arg := strings.ToUpper(util.BytesToString(args[0])); arg { case "MATCH": s.SetMatch(util.BytesToString(args[1])) args = args[2:] continue case "COUNT": count, err := strconv.Atoi(util.BytesToString(args[1])) if err != nil { return nil, err } s.SetCount(count) args = args[2:] continue case "RC": s.SetReplica() args = args[1:] } } if s.Count == 0 { s.SetCount(DefaultScanCount) } return s, nil } type Incr struct { DMap string Key string Delta int } func NewIncr(dmap, key string, delta int) *Incr { return &Incr{ DMap: dmap, Key: key, Delta: delta, } } func (i *Incr) Command(ctx context.Context) *redis.IntCmd { var args []interface{} args = append(args, DMap.Incr) args = append(args, i.DMap) args = append(args, i.Key) args = append(args, i.Delta) return redis.NewIntCmd(ctx, args...) } func ParseIncrCommand(cmd redcon.Command) (*Incr, error) { if len(cmd.Args) < 4 { return nil, errWrongNumber(cmd.Args) } delta, err := strconv.Atoi(util.BytesToString(cmd.Args[3])) if err != nil { return nil, err } return NewIncr( util.BytesToString(cmd.Args[1]), util.BytesToString(cmd.Args[2]), delta, ), nil } type Decr struct { *Incr } func NewDecr(dmap, key string, delta int) *Decr { return &Decr{ NewIncr(dmap, key, delta), } } func (d *Decr) Command(ctx context.Context) *redis.IntCmd { cmd := d.Incr.Command(ctx) cmd.Args()[0] = DMap.Decr return cmd } func ParseDecrCommand(cmd redcon.Command) (*Decr, error) { if len(cmd.Args) < 4 { return nil, errWrongNumber(cmd.Args) } delta, err := strconv.Atoi(util.BytesToString(cmd.Args[3])) if err != nil { return nil, err } return NewDecr( util.BytesToString(cmd.Args[1]), util.BytesToString(cmd.Args[2]), delta, ), nil } type GetPut struct { DMap string Key string Value []byte Raw bool } func NewGetPut(dmap, key string, value []byte) *GetPut { return &GetPut{ DMap: dmap, Key: key, Value: value, } } func (g *GetPut) SetRaw() *GetPut { g.Raw = true return g } func (g *GetPut) Command(ctx context.Context) *redis.StringCmd { var args []interface{} args = append(args, DMap.GetPut) args = append(args, g.DMap) args = append(args, g.Key) args = append(args, g.Value) if g.Raw { args = append(args, "RW") } return redis.NewStringCmd(ctx, args...) } func ParseGetPutCommand(cmd redcon.Command) (*GetPut, error) { if len(cmd.Args) < 4 { return nil, errWrongNumber(cmd.Args) } g := NewGetPut( util.BytesToString(cmd.Args[1]), // DMap util.BytesToString(cmd.Args[2]), // Key cmd.Args[3], // Value ) if len(cmd.Args) == 5 { arg := util.BytesToString(cmd.Args[4]) if arg == "RW" { g.SetRaw() } else { return nil, fmt.Errorf("%w: %s", ErrInvalidArgument, arg) } } return g, nil } type IncrByFloat struct { DMap string Key string Delta float64 } func NewIncrByFloat(dmap, key string, delta float64) *IncrByFloat { return &IncrByFloat{ DMap: dmap, Key: key, Delta: delta, } } func (i *IncrByFloat) Command(ctx context.Context) *redis.FloatCmd { var args []interface{} args = append(args, DMap.IncrByFloat) args = append(args, i.DMap) args = append(args, i.Key) args = append(args, i.Delta) return redis.NewFloatCmd(ctx, args...) } func ParseIncrByFloatCommand(cmd redcon.Command) (*IncrByFloat, error) { if len(cmd.Args) < 4 { return nil, errWrongNumber(cmd.Args) } delta, err := strconv.ParseFloat(util.BytesToString(cmd.Args[3]), 10) if err != nil { return nil, err } return NewIncrByFloat( util.BytesToString(cmd.Args[1]), util.BytesToString(cmd.Args[2]), delta, ), nil } type Lock struct { DMap string Key string Deadline float64 EX float64 PX int64 } func NewLock(dmap, key string, deadline float64) *Lock { return &Lock{ DMap: dmap, Key: key, Deadline: deadline, } } func (l *Lock) SetEX(ex float64) *Lock { l.EX = ex return l } func (l *Lock) SetPX(px int64) *Lock { l.PX = px return l } func (l *Lock) Command(ctx context.Context) *redis.StringCmd { var args []interface{} args = append(args, DMap.Lock) args = append(args, l.DMap) args = append(args, l.Key) args = append(args, l.Deadline) // Options if l.EX != 0 { args = append(args, "EX") args = append(args, l.EX) } if l.PX != 0 { args = append(args, "PX") args = append(args, l.PX) } return redis.NewStringCmd(ctx, args...) } func ParseLockCommand(cmd redcon.Command) (*Lock, error) { if len(cmd.Args) < 4 { return nil, errWrongNumber(cmd.Args) } deadline, err := strconv.ParseFloat(util.BytesToString(cmd.Args[3]), 64) if err != nil { return nil, err } l := NewLock( util.BytesToString(cmd.Args[1]), // DMap util.BytesToString(cmd.Args[2]), // Key deadline, // Deadline ) // EX or PX are optional. if len(cmd.Args) > 4 { if len(cmd.Args) == 5 { return nil, fmt.Errorf("%w: %s needs a numerical argument", ErrInvalidArgument, util.BytesToString(cmd.Args[5])) } switch arg := strings.ToUpper(util.BytesToString(cmd.Args[4])); arg { case "PX": px, err := strconv.ParseInt(util.BytesToString(cmd.Args[5]), 10, 64) if err != nil { return nil, err } l.PX = px case "EX": ex, err := strconv.ParseFloat(util.BytesToString(cmd.Args[5]), 64) if err != nil { return nil, err } l.EX = ex default: return nil, fmt.Errorf("%w: %s", ErrInvalidArgument, arg) } } return l, nil } type Unlock struct { DMap string Key string Token string } func NewUnlock(dmap, key, token string) *Unlock { return &Unlock{ DMap: dmap, Key: key, Token: token, } } func (u *Unlock) Command(ctx context.Context) *redis.StatusCmd { var args []interface{} args = append(args, DMap.Unlock) args = append(args, u.DMap) args = append(args, u.Key) args = append(args, u.Token) return redis.NewStatusCmd(ctx, args...) } func ParseUnlockCommand(cmd redcon.Command) (*Unlock, error) { if len(cmd.Args) < 4 { return nil, errWrongNumber(cmd.Args) } return NewUnlock( util.BytesToString(cmd.Args[1]), // DMap util.BytesToString(cmd.Args[2]), // Key util.BytesToString(cmd.Args[3]), // Token ), nil } type LockLease struct { DMap string Key string Token string Timeout float64 } func NewLockLease(dmap, key, token string, timeout float64) *LockLease { return &LockLease{ DMap: dmap, Key: key, Token: token, Timeout: timeout, } } func (l *LockLease) Command(ctx context.Context) *redis.StatusCmd { var args []interface{} args = append(args, DMap.LockLease) args = append(args, l.DMap) args = append(args, l.Key) args = append(args, l.Token) args = append(args, l.Timeout) return redis.NewStatusCmd(ctx, args...) } func ParseLockLeaseCommand(cmd redcon.Command) (*LockLease, error) { if len(cmd.Args) < 5 { return nil, errWrongNumber(cmd.Args) } timeout, err := strconv.ParseFloat(util.BytesToString(cmd.Args[4]), 64) if err != nil { return nil, err } return NewLockLease( util.BytesToString(cmd.Args[1]), // DMap util.BytesToString(cmd.Args[2]), // Key util.BytesToString(cmd.Args[3]), // Token timeout, // Timeout ), nil } type PLockLease struct { DMap string Key string Token string Timeout int64 } func NewPLockLease(dmap, key, token string, timeout int64) *PLockLease { return &PLockLease{ DMap: dmap, Key: key, Token: token, Timeout: timeout, } } func (p *PLockLease) Command(ctx context.Context) *redis.StatusCmd { var args []interface{} args = append(args, DMap.PLockLease) args = append(args, p.DMap) args = append(args, p.Key) args = append(args, p.Token) args = append(args, p.Timeout) return redis.NewStatusCmd(ctx, args...) } func ParsePLockLeaseCommand(cmd redcon.Command) (*PLockLease, error) { if len(cmd.Args) < 5 { return nil, errWrongNumber(cmd.Args) } timeout, err := strconv.ParseInt(util.BytesToString(cmd.Args[4]), 10, 64) if err != nil { return nil, err } return NewPLockLease( util.BytesToString(cmd.Args[1]), // DMap util.BytesToString(cmd.Args[2]), // Key util.BytesToString(cmd.Args[3]), // Token timeout, // Timeout ), nil } ================================================ FILE: internal/protocol/dmap_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package protocol import ( "context" "strings" "testing" "time" "github.com/stretchr/testify/require" "github.com/tidwall/redcon" ) func stringToCommand(s string) redcon.Command { cmd := redcon.Command{ Raw: []byte(s), } s = strings.TrimSuffix(s, ": []") s = strings.TrimSuffix(s, ": 0") s = strings.TrimSuffix(s, ":") s = strings.TrimSuffix(s, ": ") parsed := strings.Split(s, " ") for _, arg := range parsed { cmd.Args = append(cmd.Args, []byte(arg)) } return cmd } func TestProtocol_ParsePutCommand_EX(t *testing.T) { putCmd := NewPut("my-dmap", "my-key", []byte("my-value")) putCmd.SetEX((10 * time.Second).Seconds()) cmd := stringToCommand(putCmd.Command(context.Background()).String()) parsed, err := ParsePutCommand(cmd) require.NoError(t, err) require.Equal(t, "my-dmap", parsed.DMap) require.Equal(t, "my-key", parsed.Key) require.Equal(t, []byte("my-value"), parsed.Value) require.Equal(t, float64(10), parsed.EX) } func TestProtocol_ParsePutCommand_PX(t *testing.T) { putCmd := NewPut("my-dmap", "my-key", []byte("my-value")) putCmd.SetPX((100 * time.Millisecond).Milliseconds()) cmd := stringToCommand(putCmd.Command(context.Background()).String()) parsed, err := ParsePutCommand(cmd) require.NoError(t, err) require.Equal(t, "my-dmap", parsed.DMap) require.Equal(t, "my-key", parsed.Key) require.Equal(t, []byte("my-value"), parsed.Value) require.Equal(t, int64(100), parsed.PX) } func TestProtocol_ParsePutCommand_NX(t *testing.T) { putCmd := NewPut("my-dmap", "my-key", []byte("my-value")) putCmd.SetNX() cmd := stringToCommand(putCmd.Command(context.Background()).String()) parsed, err := ParsePutCommand(cmd) require.NoError(t, err) require.Equal(t, "my-dmap", parsed.DMap) require.Equal(t, "my-key", parsed.Key) require.Equal(t, []byte("my-value"), parsed.Value) require.True(t, parsed.NX) require.False(t, parsed.XX) } func TestProtocol_ParsePutCommand_XX(t *testing.T) { putCmd := NewPut("my-dmap", "my-key", []byte("my-value")) putCmd.SetXX() cmd := stringToCommand(putCmd.Command(context.Background()).String()) parsed, err := ParsePutCommand(cmd) require.NoError(t, err) require.Equal(t, "my-dmap", parsed.DMap) require.Equal(t, "my-key", parsed.Key) require.Equal(t, []byte("my-value"), parsed.Value) require.True(t, parsed.XX) require.False(t, parsed.NX) } func TestProtocol_ParsePutCommand_EXAT(t *testing.T) { putCmd := NewPut("my-dmap", "my-key", []byte("my-value")) exat := float64(time.Now().Unix()) + 10 putCmd.SetEXAT(exat) cmd := stringToCommand(putCmd.Command(context.Background()).String()) parsed, err := ParsePutCommand(cmd) require.NoError(t, err) require.Equal(t, "my-dmap", parsed.DMap) require.Equal(t, "my-key", parsed.Key) require.Equal(t, []byte("my-value"), parsed.Value) require.Equal(t, exat, parsed.EXAT) } func TestProtocol_ParsePutCommand_PXAT(t *testing.T) { putCmd := NewPut("my-dmap", "my-key", []byte("my-value")) pxat := (time.Now().UnixNano() / 1000000) + 10 putCmd.SetPXAT(pxat) cmd := stringToCommand(putCmd.Command(context.Background()).String()) parsed, err := ParsePutCommand(cmd) require.NoError(t, err) require.Equal(t, "my-dmap", parsed.DMap) require.Equal(t, "my-key", parsed.Key) require.Equal(t, []byte("my-value"), parsed.Value) require.Equal(t, pxat, parsed.PXAT) } func TestProtocol_ParseScanCommand(t *testing.T) { scanCmd := NewScan(1, "my-dmap", 0) s := scanCmd.Command(context.Background()).String() s = strings.TrimSuffix(s, ": []") cmd := stringToCommand(s) parsed, err := ParseScanCommand(cmd) require.NoError(t, err) require.Equal(t, "my-dmap", parsed.DMap) require.Equal(t, "", parsed.Match) require.Equal(t, 10, parsed.Count) require.False(t, scanCmd.Replica) } func TestProtocol_ParseScanCommand_Replica(t *testing.T) { scanCmd := NewScan(1, "my-dmap", 0).SetReplica() s := scanCmd.Command(context.Background()).String() s = strings.TrimSuffix(s, ": []") cmd := stringToCommand(s) parsed, err := ParseScanCommand(cmd) require.NoError(t, err) require.Equal(t, "my-dmap", parsed.DMap) require.Equal(t, "", parsed.Match) require.Equal(t, 10, parsed.Count) require.True(t, scanCmd.Replica) } func TestProtocol_ParseScanCommand_Match(t *testing.T) { scanCmd := NewScan(1, "my-dmap", 0).SetMatch("^even") s := scanCmd.Command(context.Background()).String() s = strings.TrimSuffix(s, ": []") cmd := stringToCommand(s) parsed, err := ParseScanCommand(cmd) require.NoError(t, err) require.Equal(t, "my-dmap", parsed.DMap) require.Equal(t, uint64(1), parsed.PartID) require.Equal(t, "^even", parsed.Match) require.Equal(t, 10, parsed.Count) require.False(t, scanCmd.Replica) } func TestProtocol_ParseScanCommand_PartID(t *testing.T) { scanCmd := NewScan(1, "my-dmap", 0).SetCount(200) s := scanCmd.Command(context.Background()).String() s = strings.TrimSuffix(s, ": []") cmd := stringToCommand(s) parsed, err := ParseScanCommand(cmd) require.NoError(t, err) require.Equal(t, "my-dmap", parsed.DMap) require.Equal(t, uint64(1), parsed.PartID) require.Equal(t, "", parsed.Match) require.Equal(t, 200, parsed.Count) require.False(t, scanCmd.Replica) } func TestProtocol_ParseScanCommand_Match_Count(t *testing.T) { scanCmd := NewScan(1, "my-dmap", 0).SetCount(100).SetMatch("^even") s := scanCmd.Command(context.Background()).String() s = strings.TrimSuffix(s, ": []") cmd := stringToCommand(s) parsed, err := ParseScanCommand(cmd) require.NoError(t, err) require.Equal(t, "my-dmap", parsed.DMap) require.Equal(t, uint64(1), parsed.PartID) require.Equal(t, "^even", parsed.Match) require.Equal(t, 100, parsed.Count) require.False(t, scanCmd.Replica) } func TestProtocol_ParseScanCommand_Match_Count_Replica(t *testing.T) { scanCmd := NewScan(1, "my-dmap", 0). SetCount(100). SetMatch("^even"). SetReplica() s := scanCmd.Command(context.Background()).String() s = strings.TrimSuffix(s, ": []") cmd := stringToCommand(s) parsed, err := ParseScanCommand(cmd) require.NoError(t, err) require.Equal(t, "my-dmap", parsed.DMap) require.Equal(t, uint64(1), parsed.PartID) require.Equal(t, "^even", parsed.Match) require.Equal(t, 100, parsed.Count) require.True(t, scanCmd.Replica) } func TestProtocol_PutEntry(t *testing.T) { putEntryCmd := NewPutEntry("my-dmap", "my-key", []byte("my-value")) cmd := stringToCommand(putEntryCmd.Command(context.Background()).String()) parsed, err := ParsePutEntryCommand(cmd) require.NoError(t, err) require.Equal(t, "my-dmap", parsed.DMap) require.Equal(t, "my-key", parsed.Key) require.Equal(t, []byte("my-value"), parsed.Value) } func TestProtocol_Get(t *testing.T) { getCmd := NewGet("my-dmap", "my-key") cmd := stringToCommand(getCmd.Command(context.Background()).String()) parsed, err := ParseGetCommand(cmd) require.NoError(t, err) require.Equal(t, "my-dmap", parsed.DMap) require.Equal(t, "my-key", parsed.Key) require.False(t, parsed.Raw) } func TestProtocol_Get_RW(t *testing.T) { getCmd := NewGet("my-dmap", "my-key") getCmd.SetRaw() cmd := stringToCommand(getCmd.Command(context.Background()).String()) parsed, err := ParseGetCommand(cmd) require.NoError(t, err) require.Equal(t, "my-dmap", parsed.DMap) require.Equal(t, "my-key", parsed.Key) require.True(t, parsed.Raw) } func TestProtocol_GetEntry(t *testing.T) { getEntryCmd := NewGetEntry("my-dmap", "my-key") cmd := stringToCommand(getEntryCmd.Command(context.Background()).String()) parsed, err := ParseGetEntryCommand(cmd) require.NoError(t, err) require.Equal(t, "my-dmap", parsed.DMap) require.Equal(t, "my-key", parsed.Key) require.False(t, parsed.Replica) } func TestProtocol_GetEntry_RC(t *testing.T) { getEntryCmd := NewGetEntry("my-dmap", "my-key") getEntryCmd.SetReplica() cmd := stringToCommand(getEntryCmd.Command(context.Background()).String()) parsed, err := ParseGetEntryCommand(cmd) require.NoError(t, err) require.Equal(t, "my-dmap", parsed.DMap) require.Equal(t, "my-key", parsed.Key) require.True(t, parsed.Replica) } func TestProtocol_Del(t *testing.T) { delCmd := NewDel("my-dmap", "key1", "key2") cmd := stringToCommand(delCmd.Command(context.Background()).String()) parsed, err := ParseDelCommand(cmd) require.NoError(t, err) require.Equal(t, "my-dmap", parsed.DMap) require.Equal(t, []string{"key1", "key2"}, parsed.Keys) } func TestProtocol_DelEntry(t *testing.T) { delEntryCmd := NewDelEntry("my-dmap", "my-key") cmd := stringToCommand(delEntryCmd.Command(context.Background()).String()) parsed, err := ParseDelEntryCommand(cmd) require.NoError(t, err) require.Equal(t, "my-dmap", parsed.Del.DMap) require.Equal(t, []string{"my-key"}, parsed.Del.Keys) require.False(t, parsed.Replica) } func TestProtocol_DelEntry_RC(t *testing.T) { delEntryCmd := NewDelEntry("my-dmap", "my-key") delEntryCmd.SetReplica() cmd := stringToCommand(delEntryCmd.Command(context.Background()).String()) parsed, err := ParseDelEntryCommand(cmd) require.NoError(t, err) require.Equal(t, "my-dmap", parsed.Del.DMap) require.Equal(t, []string{"my-key"}, parsed.Del.Keys) require.True(t, parsed.Replica) } func TestProtocol_PExpire(t *testing.T) { pexpireCmd := NewPExpire("my-dmap", "my-key", 10*time.Millisecond) cmd := stringToCommand(pexpireCmd.Command(context.Background()).String()) parsed, err := ParsePExpireCommand(cmd) require.NoError(t, err) require.Equal(t, "my-dmap", parsed.DMap) require.Equal(t, "my-key", parsed.Key) require.Equal(t, 10*time.Millisecond, parsed.Milliseconds) } func TestProtocol_Expire(t *testing.T) { pexpireCmd := NewExpire("my-dmap", "my-key", 10*time.Second) cmd := stringToCommand(pexpireCmd.Command(context.Background()).String()) parsed, err := ParseExpireCommand(cmd) require.NoError(t, err) require.Equal(t, "my-dmap", parsed.DMap) require.Equal(t, "my-key", parsed.Key) require.Equal(t, 10*time.Second, parsed.Seconds) } func TestProtocol_Destroy(t *testing.T) { destroyCmd := NewDestroy("my-dmap") cmd := stringToCommand(destroyCmd.Command(context.Background()).String()) parsed, err := ParseDestroyCommand(cmd) require.NoError(t, err) require.Equal(t, "my-dmap", parsed.DMap) require.False(t, parsed.Local) } func TestProtocol_Destroy_Local(t *testing.T) { destroyCmd := NewDestroy("my-dmap") destroyCmd.SetLocal() cmd := stringToCommand(destroyCmd.Command(context.Background()).String()) parsed, err := ParseDestroyCommand(cmd) require.NoError(t, err) require.Equal(t, "my-dmap", parsed.DMap) require.True(t, parsed.Local) } func TestProtocol_Incr(t *testing.T) { incrCmd := NewIncr("my-dmap", "my-key", 7) cmd := stringToCommand(incrCmd.Command(context.Background()).String()) parsed, err := ParseIncrCommand(cmd) require.NoError(t, err) require.Equal(t, "my-dmap", parsed.DMap) require.Equal(t, "my-key", parsed.Key) require.Equal(t, 7, parsed.Delta) } func TestProtocol_Decr(t *testing.T) { decrCmd := NewDecr("my-dmap", "my-key", 7) cmd := stringToCommand(decrCmd.Command(context.Background()).String()) parsed, err := ParseDecrCommand(cmd) require.NoError(t, err) require.Equal(t, "my-dmap", parsed.DMap) require.Equal(t, "my-key", parsed.Key) require.Equal(t, 7, parsed.Delta) } func TestProtocol_GetPut(t *testing.T) { getputCmd := NewGetPut("my-dmap", "my-key", []byte("my-value")) cmd := stringToCommand(getputCmd.Command(context.Background()).String()) parsed, err := ParseGetPutCommand(cmd) require.NoError(t, err) require.Equal(t, "my-dmap", parsed.DMap) require.Equal(t, "my-key", parsed.Key) require.Equal(t, []byte("my-value"), parsed.Value) require.False(t, parsed.Raw) } func TestProtocol_GetPut_RW(t *testing.T) { getputCmd := NewGetPut("my-dmap", "my-key", []byte("my-value")) getputCmd.SetRaw() cmd := stringToCommand(getputCmd.Command(context.Background()).String()) parsed, err := ParseGetPutCommand(cmd) require.NoError(t, err) require.Equal(t, "my-dmap", parsed.DMap) require.Equal(t, "my-key", parsed.Key) require.Equal(t, []byte("my-value"), parsed.Value) require.True(t, parsed.Raw) } func TestProtocol_IncrByFloat(t *testing.T) { incrByFloatCmd := NewIncrByFloat("my-dmap", "my-key", 3.14159265359) cmd := stringToCommand(incrByFloatCmd.Command(context.Background()).String()) parsed, err := ParseIncrByFloatCommand(cmd) require.NoError(t, err) require.Equal(t, "my-dmap", parsed.DMap) require.Equal(t, "my-key", parsed.Key) require.Equal(t, 3.14159265359, parsed.Delta) } func TestProtocol_Lock(t *testing.T) { lockCmd := NewLock("my-dmap", "my-key", 7) cmd := stringToCommand(lockCmd.Command(context.Background()).String()) parsed, err := ParseLockCommand(cmd) require.NoError(t, err) require.Equal(t, "my-dmap", parsed.DMap) require.Equal(t, "my-key", parsed.Key) require.Equal(t, float64(7), parsed.Deadline) } func TestProtocol_Lock_EX(t *testing.T) { exDuration := (250 * time.Second).Seconds() lockCmd := NewLock("my-dmap", "my-key", 7) lockCmd.SetEX(exDuration) cmd := stringToCommand(lockCmd.Command(context.Background()).String()) parsed, err := ParseLockCommand(cmd) require.NoError(t, err) require.Equal(t, "my-dmap", parsed.DMap) require.Equal(t, "my-key", parsed.Key) require.Equal(t, float64(7), parsed.Deadline) require.Equal(t, exDuration, parsed.EX) } func TestProtocol_Lock_PX(t *testing.T) { pxDuration := (250 * time.Millisecond).Milliseconds() lockCmd := NewLock("my-dmap", "my-key", 7) lockCmd.SetPX(pxDuration) cmd := stringToCommand(lockCmd.Command(context.Background()).String()) parsed, err := ParseLockCommand(cmd) require.NoError(t, err) require.Equal(t, "my-dmap", parsed.DMap) require.Equal(t, "my-key", parsed.Key) require.Equal(t, float64(7), parsed.Deadline) require.Equal(t, pxDuration, parsed.PX) } func TestProtocol_Unlock(t *testing.T) { unlockCmd := NewUnlock("my-dmap", "my-key", "token") cmd := stringToCommand(unlockCmd.Command(context.Background()).String()) parsed, err := ParseUnlockCommand(cmd) require.NoError(t, err) require.Equal(t, "my-dmap", parsed.DMap) require.Equal(t, "my-key", parsed.Key) require.Equal(t, "token", parsed.Token) } func TestProtocol_LockLease(t *testing.T) { timeout := (7 * time.Second).Seconds() unlockCmd := NewLockLease("my-dmap", "my-key", "token", timeout) cmd := stringToCommand(unlockCmd.Command(context.Background()).String()) parsed, err := ParseLockLeaseCommand(cmd) require.NoError(t, err) require.Equal(t, "my-dmap", parsed.DMap) require.Equal(t, "my-key", parsed.Key) require.Equal(t, "token", parsed.Token) require.Equal(t, timeout, parsed.Timeout) } func TestProtocol_PLockLease(t *testing.T) { timeout := (250 * time.Millisecond).Milliseconds() plockleaseCmd := NewPLockLease("my-dmap", "my-key", "token", timeout) cmd := stringToCommand(plockleaseCmd.Command(context.Background()).String()) parsed, err := ParsePLockLeaseCommand(cmd) require.NoError(t, err) require.Equal(t, "my-dmap", parsed.DMap) require.Equal(t, "my-key", parsed.Key) require.Equal(t, "token", parsed.Token) require.Equal(t, timeout, parsed.Timeout) } func TestProtocol_Scan(t *testing.T) { scanCmd := NewScan(17, "my-dmap", 234) cmd := stringToCommand(scanCmd.Command(context.Background()).String()) parsed, err := ParseScanCommand(cmd) require.NoError(t, err) require.Equal(t, "my-dmap", parsed.DMap) require.Equal(t, uint64(17), parsed.PartID) require.Equal(t, uint64(234), parsed.Cursor) require.False(t, parsed.Replica) require.Equal(t, DefaultScanCount, parsed.Count) require.Equal(t, "", parsed.Match) } func TestProtocol_Scan_Count_Match_Replica(t *testing.T) { scanCmd := NewScan(17, "my-dmap", 234) scanCmd.SetCount(123) scanCmd.SetMatch("^even:") scanCmd.SetReplica() cmd := stringToCommand(scanCmd.Command(context.Background()).String()) parsed, err := ParseScanCommand(cmd) require.NoError(t, err) require.Equal(t, "my-dmap", parsed.DMap) require.Equal(t, uint64(17), parsed.PartID) require.Equal(t, uint64(234), parsed.Cursor) require.True(t, parsed.Replica) require.Equal(t, 123, parsed.Count) require.Equal(t, "^even:", parsed.Match) } ================================================ FILE: internal/protocol/errors.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package protocol import ( "errors" "fmt" "strings" "sync" "github.com/tidwall/redcon" ) var ErrInvalidArgument = errors.New("invalid argument") var GenericError = "ERR" var errorWithPrefix = struct { mtx sync.RWMutex prefix map[string]error err map[error]string }{ prefix: make(map[string]error), err: make(map[error]string), } func init() { SetError("INVALIDARGUMENT", ErrInvalidArgument) } func SetError(prefix string, err error) { errorWithPrefix.mtx.Lock() defer errorWithPrefix.mtx.Unlock() e, ok := errorWithPrefix.prefix[prefix] if ok && e != err { panic(fmt.Sprintf("prefix collision: %s: %v != %v", prefix, err, e)) } errorWithPrefix.err[err] = prefix errorWithPrefix.prefix[prefix] = err } func GetError(prefix string) error { errorWithPrefix.mtx.RLock() defer errorWithPrefix.mtx.RUnlock() return errorWithPrefix.prefix[prefix] } func getPrefix(err error) string { prefix, ok := errorWithPrefix.err[err] if !ok { return GenericError } return prefix } func GetPrefix(err error) string { errorWithPrefix.mtx.RLock() defer errorWithPrefix.mtx.RUnlock() prefix := getPrefix(err) if prefix == GenericError { return getPrefix(errors.Unwrap(err)) } return prefix } func ConvertError(err error) error { if err == nil { return nil } parsed := strings.SplitN(err.Error(), " ", 2) if perr := GetError(parsed[0]); perr != nil { return perr } if len(parsed) > 1 { return fmt.Errorf("%s", parsed[1]) } return err } func WriteError(conn redcon.Conn, err error) { prefix := GetPrefix(err) conn.WriteError(fmt.Sprintf("%s %s", prefix, err.Error())) } func errWrongNumber(args [][]byte) error { sb := strings.Builder{} for { arg := args[0] sb.Write(arg) args = args[1:] if len(args) == 0 { break } sb.WriteByte(0x20) } return fmt.Errorf("wrong number of arguments for '%s' command", strings.ToLower(sb.String())) } ================================================ FILE: internal/protocol/errors_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package protocol import ( "context" "errors" "fmt" "testing" "github.com/stretchr/testify/require" ) var errSomethingWentWrong = errors.New("something went wrong") func TestProtocol_errWrongNumber(t *testing.T) { getCmd := NewGet("my-dmap", "my-key").Command(context.Background()) cmd := stringToCommand(getCmd.String()) err := errWrongNumber(cmd.Args) require.Equal(t, "wrong number of arguments for 'dm.get my-dmap my-key' command", err.Error()) } func TestProtocol_GetPrefix(t *testing.T) { SetError("WRONG", errSomethingWentWrong) prefix := GetPrefix(errSomethingWentWrong) require.Equal(t, "WRONG", prefix) } func TestProtocol_GetError(t *testing.T) { SetError("WRONG", errSomethingWentWrong) err := GetError("WRONG") require.ErrorIs(t, err, errSomethingWentWrong) } func TestProtocol_ConvertError(t *testing.T) { SetError("WRONG", errSomethingWentWrong) err := fmt.Errorf("WRONG %s", errSomethingWentWrong.Error()) cerr := ConvertError(err) require.ErrorIs(t, cerr, errSomethingWentWrong) } ================================================ FILE: internal/protocol/pubsub.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package protocol import ( "context" "github.com/olric-data/olric/internal/util" "github.com/redis/go-redis/v9" "github.com/tidwall/redcon" ) type Publish struct { Channel string Message string } func NewPublish(channel, message string) *Publish { return &Publish{ Channel: channel, Message: message, } } func (p *Publish) Command(ctx context.Context) *redis.IntCmd { var args []interface{} args = append(args, PubSub.Publish) args = append(args, p.Channel) args = append(args, p.Message) return redis.NewIntCmd(ctx, args...) } func ParsePublishCommand(cmd redcon.Command) (*Publish, error) { if len(cmd.Args) < 3 { return nil, errWrongNumber(cmd.Args) } return NewPublish( util.BytesToString(cmd.Args[1]), // Channel util.BytesToString(cmd.Args[2]), // Message ), nil } type PublishInternal struct { Channel string Message string } func NewPublishInternal(channel, message string) *PublishInternal { return &PublishInternal{ Channel: channel, Message: message, } } func (p *PublishInternal) Command(ctx context.Context) *redis.IntCmd { var args []interface{} args = append(args, PubSub.PublishInternal) args = append(args, p.Channel) args = append(args, p.Message) return redis.NewIntCmd(ctx, args...) } func ParsePublishInternalCommand(cmd redcon.Command) (*PublishInternal, error) { if len(cmd.Args) < 3 { return nil, errWrongNumber(cmd.Args) } return NewPublishInternal( util.BytesToString(cmd.Args[1]), // Channel util.BytesToString(cmd.Args[2]), // Message ), nil } type Subscribe struct { Channels []string } func NewSubscribe(channels ...string) *Subscribe { return &Subscribe{ Channels: channels, } } func (s *Subscribe) Command(ctx context.Context) *redis.SliceCmd { var args []interface{} args = append(args, PubSub.Subscribe) for _, channel := range s.Channels { args = append(args, channel) } return redis.NewSliceCmd(ctx, args...) } func ParseSubscribeCommand(cmd redcon.Command) (*Subscribe, error) { if len(cmd.Args) < 2 { return nil, errWrongNumber(cmd.Args) } var channels []string args := cmd.Args[1:] for len(args) > 0 { arg := util.BytesToString(args[0]) channels = append(channels, arg) args = args[1:] } return NewSubscribe(channels...), nil } type PSubscribe struct { Patterns []string } func NewPSubscribe(patterns ...string) *PSubscribe { return &PSubscribe{ Patterns: patterns, } } func (s *PSubscribe) Command(ctx context.Context) *redis.SliceCmd { var args []interface{} args = append(args, PubSub.Subscribe) for _, channel := range s.Patterns { args = append(args, channel) } return redis.NewSliceCmd(ctx, args...) } func ParsePSubscribeCommand(cmd redcon.Command) (*PSubscribe, error) { if len(cmd.Args) < 2 { return nil, errWrongNumber(cmd.Args) } var patterns []string args := cmd.Args[1:] for len(args) > 0 { arg := util.BytesToString(args[0]) patterns = append(patterns, arg) args = args[1:] } return NewPSubscribe(patterns...), nil } type PubSubChannels struct { Pattern string } func NewPubSubChannels() *PubSubChannels { return &PubSubChannels{} } func (ps *PubSubChannels) SetPattern(pattern string) *PubSubChannels { ps.Pattern = pattern return ps } func (ps *PubSubChannels) Command(ctx context.Context) *redis.SliceCmd { var args []interface{} args = append(args, PubSub.PubSubChannels) if ps.Pattern != "" { args = append(args, ps.Pattern) } return redis.NewSliceCmd(ctx, args...) } func ParsePubSubChannelsCommand(cmd redcon.Command) (*PubSubChannels, error) { if len(cmd.Args) < 2 { return nil, errWrongNumber(cmd.Args) } ps := NewPubSubChannels() if len(cmd.Args) >= 3 { ps.SetPattern(util.BytesToString(cmd.Args[2])) } return ps, nil } type PubSubNumpat struct{} func NewPubSubNumpat() *PubSubNumpat { return &PubSubNumpat{} } func (ps *PubSubNumpat) Command(ctx context.Context) *redis.IntCmd { var args []interface{} args = append(args, PubSub.PubSubNumpat) return redis.NewIntCmd(ctx, args...) } func ParsePubSubNumpatCommand(cmd redcon.Command) (*PubSubNumpat, error) { if len(cmd.Args) < 2 { return nil, errWrongNumber(cmd.Args) } return NewPubSubNumpat(), nil } type PubSubNumsub struct { Channels []string } func NewPubSubNumsub(channels ...string) *PubSubNumsub { return &PubSubNumsub{ Channels: channels, } } func (ps *PubSubNumsub) Command(ctx context.Context) *redis.SliceCmd { var args []interface{} args = append(args, PubSub.PubSubNumsub) for _, channel := range ps.Channels { args = append(args, channel) } return redis.NewSliceCmd(ctx, args...) } func ParsePubSubNumsubCommand(cmd redcon.Command) (*PubSubNumsub, error) { if len(cmd.Args) < 2 { return nil, errWrongNumber(cmd.Args) } var channels []string args := cmd.Args[2:] for len(args) > 0 { arg := util.BytesToString(args[0]) channels = append(channels, arg) args = args[1:] } return NewPubSubNumsub(channels...), nil } ================================================ FILE: internal/protocol/pubsub_test.go ================================================ package protocol import ( "context" "testing" "github.com/stretchr/testify/require" ) func TestProtocol_ParsePublishCommand(t *testing.T) { publishCmd := NewPublish("my-pubsub", "my-message") cmd := stringToCommand(publishCmd.Command(context.Background()).String()) parsed, err := ParsePublishCommand(cmd) require.NoError(t, err) require.Equal(t, "my-pubsub", parsed.Channel) require.Equal(t, "my-message", parsed.Message) } func TestProtocol_ParsePublishInternalCommand(t *testing.T) { publishIntCmd := NewPublishInternal("my-pubsub", "my-message") cmd := stringToCommand(publishIntCmd.Command(context.Background()).String()) parsed, err := ParsePublishInternalCommand(cmd) require.NoError(t, err) require.Equal(t, "my-pubsub", parsed.Channel) require.Equal(t, "my-message", parsed.Message) } func TestProtocol_ParseSubscribeCommand(t *testing.T) { subscribeCmd := NewSubscribe("channel-1", "channel-2", "channel-3") cmd := stringToCommand(subscribeCmd.Command(context.Background()).String()) parsed, err := ParseSubscribeCommand(cmd) require.NoError(t, err) channels := []string{"channel-1", "channel-2", "channel-3"} require.Equal(t, channels, parsed.Channels) } func TestProtocol_ParsePSubscribeCommand(t *testing.T) { psubscribeCmd := NewPSubscribe("ch?nnel-*") cmd := stringToCommand(psubscribeCmd.Command(context.Background()).String()) parsed, err := ParsePSubscribeCommand(cmd) require.NoError(t, err) patterns := []string{"ch?nnel-*"} require.Equal(t, patterns, parsed.Patterns) } func TestProtocol_PubSubChannels(t *testing.T) { pubsubChannelsCmd := NewPubSubChannels() cmd := stringToCommand(pubsubChannelsCmd.Command(context.Background()).String()) parsed, err := ParsePubSubChannelsCommand(cmd) require.NoError(t, err) require.Empty(t, parsed.Pattern) } func TestProtocol_PubSubChannels_Patterns(t *testing.T) { pubsubChannelsCmd := NewPubSubChannels() pubsubChannelsCmd.SetPattern("ch?nnel-*") cmd := stringToCommand(pubsubChannelsCmd.Command(context.Background()).String()) parsed, err := ParsePubSubChannelsCommand(cmd) require.NoError(t, err) require.Equal(t, "ch?nnel-*", parsed.Pattern) } func TestProtocol_PubSubNumpat(t *testing.T) { pubsubNumpatCmd := NewPubSubNumpat() cmd := stringToCommand(pubsubNumpatCmd.Command(context.Background()).String()) _, err := ParsePubSubNumpatCommand(cmd) require.NoError(t, err) } func TestProtocol_PubSubNumsub(t *testing.T) { pubsubNumsubCmd := NewPubSubNumsub("channel-1", "channel-2", "channel-3") cmd := stringToCommand(pubsubNumsubCmd.Command(context.Background()).String()) parsed, err := ParsePubSubNumsubCommand(cmd) require.NoError(t, err) channels := []string{"channel-1", "channel-2", "channel-3"} require.Equal(t, channels, parsed.Channels) } ================================================ FILE: internal/protocol/system.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package protocol import ( "context" "fmt" "strconv" "github.com/olric-data/olric/internal/util" "github.com/redis/go-redis/v9" "github.com/tidwall/redcon" ) type Ping struct { Message string } func NewPing() *Ping { return &Ping{} } func (p *Ping) SetMessage(m string) *Ping { p.Message = m return p } func (p *Ping) Command(ctx context.Context) *redis.StringCmd { var args []interface{} args = append(args, Generic.Ping) if p.Message != "" { args = append(args, p.Message) } return redis.NewStringCmd(ctx, args...) } func ParsePingCommand(cmd redcon.Command) (*Ping, error) { if len(cmd.Args) < 1 { return nil, errWrongNumber(cmd.Args) } p := NewPing() if len(cmd.Args) == 2 { p.SetMessage(util.BytesToString(cmd.Args[1])) } return p, nil } type MoveFragment struct { Payload []byte } func NewMoveFragment(payload []byte) *MoveFragment { return &MoveFragment{ Payload: payload, } } func (m *MoveFragment) Command(ctx context.Context) *redis.StatusCmd { var args []interface{} args = append(args, Internal.MoveFragment) args = append(args, m.Payload) return redis.NewStatusCmd(ctx, args...) } func ParseMoveFragmentCommand(cmd redcon.Command) (*MoveFragment, error) { if len(cmd.Args) < 2 { return nil, errWrongNumber(cmd.Args) } return NewMoveFragment(cmd.Args[1]), nil } type UpdateRouting struct { Payload []byte CoordinatorID uint64 } func NewUpdateRouting(payload []byte, coordinatorID uint64) *UpdateRouting { return &UpdateRouting{ Payload: payload, CoordinatorID: coordinatorID, } } func (u *UpdateRouting) Command(ctx context.Context) *redis.StringCmd { var args []interface{} args = append(args, Internal.UpdateRouting) args = append(args, u.Payload) args = append(args, u.CoordinatorID) return redis.NewStringCmd(ctx, args...) } func ParseUpdateRoutingCommand(cmd redcon.Command) (*UpdateRouting, error) { if len(cmd.Args) < 2 { return nil, errWrongNumber(cmd.Args) } coordinatorID, err := strconv.ParseUint(util.BytesToString(cmd.Args[2]), 10, 64) if err != nil { return nil, err } return NewUpdateRouting(cmd.Args[1], coordinatorID), nil } type LengthOfPart struct { PartID uint64 Replica bool } func NewLengthOfPart(partID uint64) *LengthOfPart { return &LengthOfPart{ PartID: partID, } } func (l *LengthOfPart) SetReplica() *LengthOfPart { l.Replica = true return l } func (l *LengthOfPart) Command(ctx context.Context) *redis.IntCmd { var args []interface{} args = append(args, Internal.LengthOfPart) args = append(args, l.PartID) if l.Replica { args = append(args, "RC") } return redis.NewIntCmd(ctx, args...) } func ParseLengthOfPartCommand(cmd redcon.Command) (*LengthOfPart, error) { if len(cmd.Args) < 2 { return nil, errWrongNumber(cmd.Args) } partID, err := strconv.ParseUint(util.BytesToString(cmd.Args[1]), 10, 64) if err != nil { return nil, err } l := NewLengthOfPart(partID) if len(cmd.Args) == 3 { arg := util.BytesToString(cmd.Args[2]) if arg == "RC" { l.SetReplica() } else { return nil, fmt.Errorf("%w: %s", ErrInvalidArgument, arg) } } return l, nil } type Stats struct { CollectRuntime bool } func NewStats() *Stats { return &Stats{} } func (s *Stats) SetCollectRuntime() *Stats { s.CollectRuntime = true return s } func (s *Stats) Command(ctx context.Context) *redis.StringCmd { var args []interface{} args = append(args, Generic.Stats) if s.CollectRuntime { args = append(args, "CR") } return redis.NewStringCmd(ctx, args...) } func ParseStatsCommand(cmd redcon.Command) (*Stats, error) { if len(cmd.Args) < 1 { return nil, errWrongNumber(cmd.Args) } s := NewStats() if len(cmd.Args) == 2 { arg := util.BytesToString(cmd.Args[1]) if arg == "CR" { s.SetCollectRuntime() } else { return nil, fmt.Errorf("%w: %s", ErrInvalidArgument, arg) } } return s, nil } // Auth represents a structure for authentication containing a password. type Auth struct { Password string } // NewAuth creates and returns a new Auth instance initialized with the given password. func NewAuth(password string) *Auth { return &Auth{ Password: password, } } // Command constructs a Redis AUTH command using the provided authentication password from the Auth instance. func (a *Auth) Command(ctx context.Context) *redis.StatusCmd { var args []interface{} args = append(args, Generic.Auth) args = append(args, a.Password) return redis.NewStatusCmd(ctx, args...) } // ParseAuthCommand parses a redcon.Command to create an Auth instance and validates command arguments. func ParseAuthCommand(cmd redcon.Command) (*Auth, error) { if len(cmd.Args) != 2 { return nil, errWrongNumber(cmd.Args) } return NewAuth( util.BytesToString(cmd.Args[1]), ), nil } ================================================ FILE: internal/protocol/system_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package protocol import ( "context" "testing" "github.com/stretchr/testify/require" ) func TestProtocol_Ping(t *testing.T) { ping := NewPing() cmd := stringToCommand(ping.Command(context.Background()).String()) parsed, err := ParsePingCommand(cmd) require.NoError(t, err) require.Equal(t, "", parsed.Message) } func TestProtocol_Ping_Message(t *testing.T) { ping := NewPing() ping.SetMessage("message") cmd := stringToCommand(ping.Command(context.Background()).String()) parsed, err := ParsePingCommand(cmd) require.NoError(t, err) require.Equal(t, "message", parsed.Message) } func TestProtocol_MoveFragment(t *testing.T) { moveFragmentCmd := NewMoveFragment([]byte("payload")) cmd := stringToCommand(moveFragmentCmd.Command(context.Background()).String()) parsed, err := ParseMoveFragmentCommand(cmd) require.NoError(t, err) require.Equal(t, []byte("payload"), parsed.Payload) } func TestProtocol_UpdateRoutingTable(t *testing.T) { updateRoutingTableCmd := NewUpdateRouting([]byte("payload"), 123) cmd := stringToCommand(updateRoutingTableCmd.Command(context.Background()).String()) parsed, err := ParseUpdateRoutingCommand(cmd) require.NoError(t, err) require.Equal(t, []byte("payload"), parsed.Payload) require.Equal(t, uint64(123), parsed.CoordinatorID) } func TestProtocol_LengthOfPart(t *testing.T) { updateRoutingTableCmd := NewLengthOfPart(123) cmd := stringToCommand(updateRoutingTableCmd.Command(context.Background()).String()) parsed, err := ParseLengthOfPartCommand(cmd) require.NoError(t, err) require.Equal(t, uint64(123), parsed.PartID) require.False(t, parsed.Replica) } func TestProtocol_LengthOfPart_RC(t *testing.T) { updateRoutingTableCmd := NewLengthOfPart(123) updateRoutingTableCmd.SetReplica() cmd := stringToCommand(updateRoutingTableCmd.Command(context.Background()).String()) parsed, err := ParseLengthOfPartCommand(cmd) require.NoError(t, err) require.Equal(t, uint64(123), parsed.PartID) require.True(t, parsed.Replica) } func TestProtocol_Stats(t *testing.T) { statsCmd := NewStats() cmd := stringToCommand(statsCmd.Command(context.Background()).String()) parsed, err := ParseStatsCommand(cmd) require.NoError(t, err) require.False(t, parsed.CollectRuntime) } func TestProtocol_Stats_CR(t *testing.T) { statsCmd := NewStats() statsCmd.SetCollectRuntime() cmd := stringToCommand(statsCmd.Command(context.Background()).String()) parsed, err := ParseStatsCommand(cmd) require.NoError(t, err) require.True(t, parsed.CollectRuntime) } func TestProtocol_Auth(t *testing.T) { auth := NewAuth("secret") cmd := stringToCommand(auth.Command(context.Background()).String()) parsed, err := ParseAuthCommand(cmd) require.NoError(t, err) require.Equal(t, "secret", parsed.Password) } func TestProtocol_Auth_errWrongNumber(t *testing.T) { cmd := stringToCommand("auth:") _, err := ParseAuthCommand(cmd) require.Error(t, err) require.Equal(t, "wrong number of arguments for 'auth' command", err.Error()) } ================================================ FILE: internal/pubsub/handlers.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package pubsub import ( "github.com/olric-data/olric/internal/protocol" "github.com/tidwall/redcon" ) func (s *Service) subscribeCommandHandler(conn redcon.Conn, cmd redcon.Command) { subscribeCmd, err := protocol.ParseSubscribeCommand(cmd) if err != nil { protocol.WriteError(conn, err) return } for _, channel := range subscribeCmd.Channels { s.pubsub.Subscribe(conn, channel) CurrentSubscribers.Increase(1) SubscribersTotal.Increase(1) } } func (s *Service) publishCommandHandler(conn redcon.Conn, cmd redcon.Command) { publishCmd, err := protocol.ParsePublishCommand(cmd) if err != nil { protocol.WriteError(conn, err) return } var total int members := s.rt.Discovery().GetMembers() for _, member := range members { if member.CompareByID(s.rt.This()) { count := s.pubsub.Publish(publishCmd.Channel, publishCmd.Message) total += count PublishedTotal.Increase(int64(count)) continue } pi := protocol.NewPublishInternal(publishCmd.Channel, publishCmd.Message).Command(s.ctx) rc := s.client.Get(member.String()) err = rc.Process(s.ctx, pi) if err != nil { protocol.WriteError(conn, err) return } pcount, err := pi.Result() if err != nil { protocol.WriteError(conn, err) return } total += int(pcount) PublishedTotal.Increase(pcount) } conn.WriteInt(total) } func (s *Service) publishInternalCommandHandler(conn redcon.Conn, cmd redcon.Command) { publishInternalCmd, err := protocol.ParsePublishInternalCommand(cmd) if err != nil { protocol.WriteError(conn, err) return } count := s.pubsub.Publish(publishInternalCmd.Channel, publishInternalCmd.Message) conn.WriteInt(count) } func (s *Service) psubscribeCommandHandler(conn redcon.Conn, cmd redcon.Command) { psubscribeCmd, err := protocol.ParsePSubscribeCommand(cmd) if err != nil { protocol.WriteError(conn, err) return } for _, pattern := range psubscribeCmd.Patterns { s.pubsub.Psubscribe(conn, pattern) PSubscribersTotal.Increase(1) CurrentPSubscribers.Increase(1) } } func (s *Service) pubsubChannelsCommandHandler(conn redcon.Conn, cmd redcon.Command) { pubsubChannelsCmd, err := protocol.ParsePubSubChannelsCommand(cmd) if err != nil { protocol.WriteError(conn, err) return } var channels []string if pubsubChannelsCmd.Pattern != "" { channels = s.pubsub.ChannelsWithPatterns(pubsubChannelsCmd.Pattern) } else { channels = s.pubsub.Channels() } conn.WriteArray(len(channels)) for _, channel := range channels { conn.WriteBulkString(channel) } } func (s *Service) pubsubNumpatCommandHandler(conn redcon.Conn, cmd redcon.Command) { _, err := protocol.ParsePubSubNumpatCommand(cmd) if err != nil { protocol.WriteError(conn, err) return } conn.WriteInt(s.pubsub.Numpat()) } func (s *Service) pubsubNumsubCommandHandler(conn redcon.Conn, cmd redcon.Command) { pubsubNumsubCmd, err := protocol.ParsePubSubNumsubCommand(cmd) if err != nil { protocol.WriteError(conn, err) return } if len(pubsubNumsubCmd.Channels) == 0 { conn.WriteArray(0) return } conn.WriteArray(len(pubsubNumsubCmd.Channels) * 2) for _, channel := range pubsubNumsubCmd.Channels { conn.WriteBulkString(channel) conn.WriteInt(s.pubsub.Numsub(channel)) } } ================================================ FILE: internal/pubsub/handlers_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package pubsub import ( "context" "fmt" "testing" "time" "github.com/olric-data/olric/internal/testcluster" "github.com/redis/go-redis/v9" "github.com/stretchr/testify/require" ) func TestPubSub_Handler_Subscribe(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() rc := s.client.Get(s.rt.This().String()) ctx := context.Background() ps := rc.Subscribe(ctx, "my-channel") // Wait for confirmation that subscription is created before publishing anything. msgi, err := ps.ReceiveTimeout(ctx, time.Second) require.NoError(t, err) subs := msgi.(*redis.Subscription) require.Equal(t, "subscribe", subs.Kind) require.Equal(t, "my-channel", subs.Channel) require.Equal(t, 1, subs.Count) // Go channel which receives messages. ch := ps.Channel() expected := make(map[string]struct{}) for i := 0; i < 10; i++ { msg := fmt.Sprintf("my-message-%d", i) err = rc.Publish(ctx, "my-channel", msg).Err() require.NoError(t, err) expected[msg] = struct{}{} } consumed := make(map[string]struct{}) L: for { select { case msg := <-ch: require.Equal(t, "my-channel", msg.Channel) consumed[msg.Payload] = struct{}{} if len(consumed) == 10 { // It would be OK break L } case <-time.After(5 * time.Second): // Enough. Break it and check the consumed items. break L } } require.Equal(t, expected, consumed) } func TestPubSub_Handler_Unsubscribe(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() rc := s.client.Get(s.rt.This().String()) ctx := context.Background() ps := rc.Subscribe(ctx, "my-channel") // Wait for confirmation that subscription is created before publishing anything. _, err := ps.ReceiveTimeout(ctx, time.Second) require.NoError(t, err) // Go channel which receives messages. ch := ps.Channel() err = ps.Unsubscribe(ctx, "my-channel") require.NoError(t, err) // Wait for some time. Because the Redis client doesn't wait for the response after // writing 'unsubscribe' command. <-time.After(250 * time.Millisecond) err = rc.Publish(ctx, "my-channel", "hello, world!").Err() require.NoError(t, err) L: for { select { case <-ch: require.Fail(t, "Received a message from an unsubscribed channel") case <-time.After(250 * time.Millisecond): // Enough. Break it and check the consumed items. break L } } } func TestPubSub_Handler_PSubscribe(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() rc := s.client.Get(s.rt.This().String()) ctx := context.Background() ps := rc.PSubscribe(ctx, "h?llo") // Wait for confirmation that subscription is created before publishing anything. msgi, err := ps.ReceiveTimeout(ctx, time.Second) require.NoError(t, err) subs := msgi.(*redis.Subscription) require.Equal(t, "psubscribe", subs.Kind) require.Equal(t, "h?llo", subs.Channel) require.Equal(t, 1, subs.Count) // Go channel which receives messages. ch := ps.Channel() expected := make(map[string]struct{}) for _, channel := range []string{"hello", "hallo", "hxllo"} { for i := 0; i < 10; i++ { msg := fmt.Sprintf("my-message-%s-%d", channel, i) err = rc.Publish(ctx, channel, msg).Err() require.NoError(t, err) expected[msg] = struct{}{} } } consumed := make(map[string]struct{}) L: for { select { case msg := <-ch: consumed[msg.Payload] = struct{}{} if len(consumed) == 30 { // It would be OK break L } case <-time.After(5 * time.Second): // Enough. Break it and check the consumed items. break L } } require.Equal(t, expected, consumed) } func TestPubSub_Handler_PUnsubscribe(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() rc := s.client.Get(s.rt.This().String()) ctx := context.Background() ps := rc.PSubscribe(ctx, "h?llo") // Wait for confirmation that subscription is created before publishing anything. _, err := ps.ReceiveTimeout(ctx, time.Second) require.NoError(t, err) // Go channel which receives messages. ch := ps.Channel() err = ps.PUnsubscribe(ctx, "h?llo") require.NoError(t, err) // Wait for some time. Because the Redis client doesn't wait for the response after // writing 'unsubscribe' command. <-time.After(250 * time.Millisecond) for _, channel := range []string{"hello", "hallo", "hxllo"} { err = rc.Publish(ctx, channel, "hello, world!").Err() require.NoError(t, err) } L: for { select { case <-ch: require.Fail(t, "Received a message from an unsubscribed channel") case <-time.After(250 * time.Millisecond): // Enough. Break it and check the consumed items. break L } } } func TestPubSub_Handler_Ping(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() rc := s.client.Get(s.rt.This().String()) ctx := context.Background() ps := rc.Subscribe(ctx, "my-channel") // Wait for confirmation that subscription is created before publishing anything. _, err := ps.ReceiveTimeout(ctx, time.Second) require.NoError(t, err) err = ps.Ping(ctx, "hello, world!") require.NoError(t, err) msg, err := ps.ReceiveTimeout(ctx, time.Second) require.NoError(t, err) require.Equal(t, "Pong", msg.(*redis.Pong).String()) } func TestPubSub_Handler_Close(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() rc := s.client.Get(s.rt.This().String()) ctx := context.Background() ps := rc.Subscribe(ctx, "my-channel") // Wait for confirmation that subscription is created before publishing anything. _, err := ps.ReceiveTimeout(ctx, time.Second) require.NoError(t, err) err = ps.Close() require.NoError(t, err) err = ps.Ping(ctx) require.Error(t, err, "redis: client is closed") //TODO: Control active subscriber count } func TestPubSub_Handler_PubSubChannels_Without_Patterns(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() rc := s.client.Get(s.rt.This().String()) ctx := context.Background() channels := make(map[string]struct{}) for i := 0; i < 10; i++ { channel := fmt.Sprintf("my-channel-%d", i) ps := rc.Subscribe(ctx, channel) // Wait for confirmation that subscription is created before publishing anything. _, err := ps.ReceiveTimeout(ctx, time.Second) require.NoError(t, err) channels[channel] = struct{}{} } res := rc.PubSubChannels(ctx, "") result, err := res.Result() require.NoError(t, err) require.Len(t, result, len(channels)) for _, channel := range result { require.Contains(t, channels, channel) } } func TestPubSub_Handler_PubSubChannels_With_Patterns(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() rc := s.client.Get(s.rt.This().String()) ctx := context.Background() channels := make(map[string]struct{}) for _, channel := range []string{"hello-1", "hello-2", "hello-3", "foobar"} { ps := rc.Subscribe(ctx, channel) // Wait for confirmation that subscription is created before publishing anything. _, err := ps.ReceiveTimeout(ctx, time.Second) require.NoError(t, err) channels[channel] = struct{}{} } res := rc.PubSubChannels(ctx, "h*") result, err := res.Result() require.NoError(t, err) require.Len(t, result, len(channels)-1) require.NotContains(t, result, "foobar") for _, channel := range result { require.Contains(t, channels, channel) } } func TestPubSub_Handler_PubSubNumpat(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() rc := s.client.Get(s.rt.This().String()) ctx := context.Background() for _, channel := range []string{"h*llo", "f*bar"} { ps := rc.PSubscribe(ctx, channel) // Wait for confirmation that subscription is created before publishing anything. _, err := ps.ReceiveTimeout(ctx, time.Second) require.NoError(t, err) } res := rc.PubSubNumPat(ctx) nr, err := res.Result() require.NoError(t, err) require.Equal(t, int64(2), nr) } func TestPubSub_Handler_PubSubNumsub(t *testing.T) { cluster := testcluster.New(NewService) s := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() rc := s.client.Get(s.rt.This().String()) ctx := context.Background() for _, channel := range []string{"hello", "hello", "foobar", "barfoo"} { ps := rc.Subscribe(ctx, channel) // Wait for confirmation that subscription is created before publishing anything. _, err := ps.ReceiveTimeout(ctx, time.Second) require.NoError(t, err) } res := rc.PubSubNumSub(ctx, "hello", "foobar", "barfoo") nr, err := res.Result() require.NoError(t, err) require.Equal(t, int64(2), nr["hello"]) require.Equal(t, int64(1), nr["foobar"]) require.Equal(t, int64(1), nr["barfoo"]) } func TestPubSub_Cluster(t *testing.T) { cluster := testcluster.New(NewService) s1 := cluster.AddMember(nil).(*Service) s2 := cluster.AddMember(nil).(*Service) defer cluster.Shutdown() rc1 := s1.client.Get(s1.rt.This().String()) ctx := context.Background() ps := rc1.Subscribe(ctx, "my-channel") // Wait for confirmation that subscription is created before publishing anything. msgi, err := ps.ReceiveTimeout(ctx, time.Second) require.NoError(t, err) subs := msgi.(*redis.Subscription) require.Equal(t, "subscribe", subs.Kind) require.Equal(t, "my-channel", subs.Channel) require.Equal(t, 1, subs.Count) // Go channel which receives messages. ch := ps.Channel() rc2 := s2.client.Get(s2.rt.This().String()) expected := make(map[string]struct{}) for i := 0; i < 10; i++ { msg := fmt.Sprintf("my-message-%d", i) err = rc2.Publish(ctx, "my-channel", msg).Err() require.NoError(t, err) expected[msg] = struct{}{} } consumed := make(map[string]struct{}) L: for { select { case msg := <-ch: require.Equal(t, "my-channel", msg.Channel) consumed[msg.Payload] = struct{}{} if len(consumed) == 10 { // It would be OK break L } case <-time.After(5 * time.Second): // Enough. Break it and check the consumed items. break L } } require.Equal(t, expected, consumed) } ================================================ FILE: internal/pubsub/pubsub.go ================================================ // The MIT License (MIT) // // Copyright (c) 2016 Josh Baker // // Permission is hereby granted, free of charge, to any person obtaining a copy of // this software and associated documentation files (the "Software"), to deal in // the Software without restriction, including without limitation the rights to // use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of // the Software, and to permit persons to whom the Software is furnished to do so, // subject to the following conditions: // // The above copyright notice and this permission notice shall be included in all // copies or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS // FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR // COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER // IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN // CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. package pubsub import ( "fmt" "strings" "sync" "github.com/tidwall/btree" "github.com/tidwall/match" "github.com/tidwall/redcon" ) // PubSub is a Redis compatible pub/sub server type PubSub struct { mu sync.RWMutex nextid uint64 initd bool chans *btree.BTree conns map[redcon.Conn]*pubSubConn // callbacks unsubscribeCallback func() punsubscribeCallback func() } // Subscribe a connection to PubSub func (ps *PubSub) Subscribe(conn redcon.Conn, channel string) { ps.subscribe(conn, false, channel) } // Psubscribe a connection to PubSub func (ps *PubSub) Psubscribe(conn redcon.Conn, channel string) { ps.subscribe(conn, true, channel) } // Publish a message to subscribers func (ps *PubSub) Publish(channel, message string) int { ps.mu.RLock() defer ps.mu.RUnlock() if !ps.initd { return 0 } var sent int // write messages to all clients that are subscribed on the channel pivot := &pubSubEntry{pattern: false, channel: channel} ps.chans.Ascend(pivot, func(item interface{}) bool { entry := item.(*pubSubEntry) if entry.channel != pivot.channel || entry.pattern != pivot.pattern { return false } entry.sconn.writeMessage(entry.pattern, "", channel, message) sent++ return true }) // match on and write all psubscribe clients pivot = &pubSubEntry{pattern: true} ps.chans.Ascend(pivot, func(item interface{}) bool { entry := item.(*pubSubEntry) if match.Match(channel, entry.channel) { entry.sconn.writeMessage(entry.pattern, entry.channel, channel, message) } sent++ return true }) return sent } type pubSubConn struct { id uint64 mu sync.Mutex conn redcon.Conn dconn redcon.DetachedConn entries map[*pubSubEntry]bool } type pubSubEntry struct { pattern bool sconn *pubSubConn channel string } func (sconn *pubSubConn) writeMessage(pat bool, pchan, channel, msg string) { sconn.mu.Lock() defer sconn.mu.Unlock() if pat { sconn.dconn.WriteArray(4) sconn.dconn.WriteBulkString("pmessage") sconn.dconn.WriteBulkString(pchan) sconn.dconn.WriteBulkString(channel) sconn.dconn.WriteBulkString(msg) } else { sconn.dconn.WriteArray(3) sconn.dconn.WriteBulkString("message") sconn.dconn.WriteBulkString(channel) sconn.dconn.WriteBulkString(msg) } sconn.dconn.Flush() } // bgrunner runs in the background and reads incoming commands from the // detached client. func (sconn *pubSubConn) bgrunner(ps *PubSub) { defer func() { // client connection has ended, disconnect from the PubSub instances // and close the network connection. ps.mu.Lock() defer ps.mu.Unlock() for entry := range sconn.entries { ps.chans.Delete(entry) } delete(ps.conns, sconn.conn) sconn.mu.Lock() defer sconn.mu.Unlock() sconn.dconn.Close() }() for { cmd, err := sconn.dconn.ReadCommand() if err != nil { return } if len(cmd.Args) == 0 { continue } switch strings.ToLower(string(cmd.Args[0])) { case "psubscribe", "subscribe": if len(cmd.Args) < 2 { func() { sconn.mu.Lock() defer sconn.mu.Unlock() sconn.dconn.WriteError(fmt.Sprintf("ERR wrong number of "+ "arguments for '%s'", cmd.Args[0])) sconn.dconn.Flush() }() continue } command := strings.ToLower(string(cmd.Args[0])) for i := 1; i < len(cmd.Args); i++ { if command == "psubscribe" { ps.Psubscribe(sconn.conn, string(cmd.Args[i])) } else { ps.Subscribe(sconn.conn, string(cmd.Args[i])) } } case "unsubscribe", "punsubscribe": pattern := strings.ToLower(string(cmd.Args[0])) == "punsubscribe" if len(cmd.Args) == 1 { ps.unsubscribe(sconn.conn, pattern, true, "") } else { for i := 1; i < len(cmd.Args); i++ { channel := string(cmd.Args[i]) ps.unsubscribe(sconn.conn, pattern, false, channel) } } case "quit": func() { sconn.mu.Lock() defer sconn.mu.Unlock() sconn.dconn.WriteString("OK") sconn.dconn.Flush() sconn.dconn.Close() }() return case "ping": var msg string switch len(cmd.Args) { case 1: case 2: msg = string(cmd.Args[1]) default: func() { sconn.mu.Lock() defer sconn.mu.Unlock() sconn.dconn.WriteError(fmt.Sprintf("ERR wrong number of "+ "arguments for '%s'", cmd.Args[0])) sconn.dconn.Flush() }() continue } func() { sconn.mu.Lock() defer sconn.mu.Unlock() sconn.dconn.WriteArray(2) sconn.dconn.WriteBulkString("pong") sconn.dconn.WriteBulkString(msg) sconn.dconn.Flush() }() default: func() { sconn.mu.Lock() defer sconn.mu.Unlock() sconn.dconn.WriteError(fmt.Sprintf("ERR Can't execute '%s': "+ "only (P)SUBSCRIBE / (P)UNSUBSCRIBE / PING / QUIT are "+ "allowed in this context", cmd.Args[0])) sconn.dconn.Flush() }() } } } // byEntry is a "less" function that sorts the entries in a btree. The tree // is sorted be (pattern, channel, conn.id). All pattern=true entries are at // the end (right) of the tree. func byEntry(a, b interface{}) bool { aa := a.(*pubSubEntry) bb := b.(*pubSubEntry) if !aa.pattern && bb.pattern { return true } if aa.pattern && !bb.pattern { return false } if aa.channel < bb.channel { return true } if aa.channel > bb.channel { return false } var aid uint64 var bid uint64 if aa.sconn != nil { aid = aa.sconn.id } if bb.sconn != nil { bid = bb.sconn.id } return aid < bid } func (ps *PubSub) subscribe(conn redcon.Conn, pattern bool, channel string) { ps.mu.Lock() defer ps.mu.Unlock() // initialize the PubSub instance if !ps.initd { ps.conns = make(map[redcon.Conn]*pubSubConn) ps.chans = btree.New(byEntry) ps.initd = true } // fetch the pubSubConn sconn, ok := ps.conns[conn] if !ok { // initialize a new pubSubConn, which runs on a detached connection, // and attach it to the PubSub channels/conn btree ps.nextid++ dconn := conn.Detach() sconn = &pubSubConn{ id: ps.nextid, conn: conn, dconn: dconn, entries: make(map[*pubSubEntry]bool), } ps.conns[conn] = sconn } sconn.mu.Lock() defer sconn.mu.Unlock() // add an entry to the pubsub btree entry := &pubSubEntry{ pattern: pattern, channel: channel, sconn: sconn, } ps.chans.Set(entry) sconn.entries[entry] = true // send a message to the client sconn.dconn.WriteArray(3) if pattern { sconn.dconn.WriteBulkString("psubscribe") } else { sconn.dconn.WriteBulkString("subscribe") } sconn.dconn.WriteBulkString(channel) var count int for ient := range sconn.entries { if ient.pattern == pattern { count++ } } sconn.dconn.WriteInt(count) sconn.dconn.Flush() // start the background client operation if !ok { go sconn.bgrunner(ps) } } func (ps *PubSub) unsubscribe(conn redcon.Conn, pattern, all bool, channel string) { ps.mu.Lock() defer ps.mu.Unlock() // fetch the pubSubConn. This must exist sconn := ps.conns[conn] sconn.mu.Lock() defer sconn.mu.Unlock() removeEntry := func(entry *pubSubEntry) { if entry != nil { ps.chans.Delete(entry) delete(sconn.entries, entry) } sconn.dconn.WriteArray(3) if pattern { if ps.punsubscribeCallback != nil { ps.punsubscribeCallback() } sconn.dconn.WriteBulkString("punsubscribe") } else { if ps.unsubscribeCallback != nil { ps.unsubscribeCallback() } sconn.dconn.WriteBulkString("unsubscribe") } if entry != nil { sconn.dconn.WriteBulkString(entry.channel) } else { sconn.dconn.WriteNull() } var count int for ient := range sconn.entries { if ient.pattern == pattern { count++ } } sconn.dconn.WriteInt(count) } if all { // unsubscribe from all (p)subscribe entries var entries []*pubSubEntry for ient := range sconn.entries { if ient.pattern == pattern { entries = append(entries, ient) } } if len(entries) == 0 { removeEntry(nil) } else { for _, entry := range entries { removeEntry(entry) } } } else { // unsubscribe single channel from (p)subscribe. var entry *pubSubEntry for ient := range sconn.entries { if ient.pattern == pattern && ient.channel == channel { entry = ient break } } removeEntry(entry) } sconn.dconn.Flush() } func (ps *PubSub) Channels() []string { ps.mu.RLock() defer ps.mu.RUnlock() if !ps.initd { return nil } var channels []string for _, sconn := range ps.conns { sconn.mu.Lock() for ient := range sconn.entries { if !ient.pattern { channels = append(channels, ient.channel) } } sconn.mu.Unlock() } return channels } func (ps *PubSub) ChannelsWithPatterns(pattern string) []string { ps.mu.RLock() defer ps.mu.RUnlock() if !ps.initd { return nil } var channels []string for _, sconn := range ps.conns { sconn.mu.Lock() for ient := range sconn.entries { if match.Match(ient.channel, pattern) { channels = append(channels, ient.channel) } } sconn.mu.Unlock() } return channels } func (ps *PubSub) Numpat() int { ps.mu.RLock() defer ps.mu.RUnlock() if !ps.initd { return 0 } set := make(map[string]struct{}) for _, sconn := range ps.conns { sconn.mu.Lock() for ient := range sconn.entries { if ient.pattern { set[ient.channel] = struct{}{} } } sconn.mu.Unlock() } return len(set) } func (ps *PubSub) Numsub(channel string) int { ps.mu.RLock() defer ps.mu.RUnlock() if !ps.initd { return 0 } var result int for _, sconn := range ps.conns { sconn.mu.Lock() for ient := range sconn.entries { if ient.channel == channel { result++ } } sconn.mu.Unlock() } return result } ================================================ FILE: internal/pubsub/pubsub_test.go ================================================ // The MIT License (MIT) // // Copyright (c) 2016 Josh Baker // // Permission is hereby granted, free of charge, to any person obtaining a copy of // this software and associated documentation files (the "Software"), to deal in // the Software without restriction, including without limitation the rights to // use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of // the Software, and to permit persons to whom the Software is furnished to do so, // subject to the following conditions: // // The above copyright notice and this permission notice shall be included in all // copies or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS // FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR // COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER // IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN // CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. package pubsub import ( "bufio" "fmt" "net" "strconv" "strings" "sync" "testing" "time" "github.com/stretchr/testify/require" "github.com/tidwall/redcon" ) func testPubSubServer(addr string, done chan bool) { var ps PubSub go func() { tch := time.NewTicker(time.Millisecond * 5) defer tch.Stop() channels := []string{"achan1", "bchan2", "cchan3", "dchan4"} for i := 0; ; i++ { select { case <-tch.C: case <-done: for { var empty bool ps.mu.Lock() if len(ps.conns) == 0 { if ps.chans.Len() != 0 { panic("chans not empty") } empty = true } ps.mu.Unlock() if empty { break } time.Sleep(time.Millisecond * 10) } done <- true return } channel := channels[i%len(channels)] message := fmt.Sprintf("message %d", i) ps.Publish(channel, message) } }() panic(redcon.ListenAndServe(addr, func(conn redcon.Conn, cmd redcon.Command) { switch strings.ToLower(string(cmd.Args[0])) { default: conn.WriteError("ERR unknown command '" + string(cmd.Args[0]) + "'") case "publish": if len(cmd.Args) != 3 { conn.WriteError("ERR wrong number of arguments for '" + string(cmd.Args[0]) + "' command") return } count := ps.Publish(string(cmd.Args[1]), string(cmd.Args[2])) conn.WriteInt(count) case "subscribe", "psubscribe": if len(cmd.Args) < 2 { conn.WriteError("ERR wrong number of arguments for '" + string(cmd.Args[0]) + "' command") return } command := strings.ToLower(string(cmd.Args[0])) for i := 1; i < len(cmd.Args); i++ { if command == "psubscribe" { ps.Psubscribe(conn, string(cmd.Args[i])) } else { ps.Subscribe(conn, string(cmd.Args[i])) } } } }, nil, nil)) } func TestPubSub(t *testing.T) { addr := ":12346" done := make(chan bool) go testPubSubServer(addr, done) final := make(chan bool) go func() { select { case <-time.Tick(time.Second * 30): panic("timeout") case <-final: return } }() // create 10 connections var wg sync.WaitGroup wg.Add(10) for i := 0; i < 10; i++ { go func(i int) { defer wg.Done() var conn net.Conn for i := 0; i < 5; i++ { var err error conn, err = net.Dial("tcp", addr) if err != nil { time.Sleep(time.Second / 10) continue } } if conn == nil { require.Fail(t, "could not connect to server") } defer conn.Close() regs := make(map[string]int) var maxp int var maxs int _, err := fmt.Fprintf(conn, "subscribe achan1\r\n") require.NoError(t, err) _, err = fmt.Fprintf(conn, "subscribe bchan2 cchan3\r\n") require.NoError(t, err) _, err = fmt.Fprintf(conn, "psubscribe a*1\r\n") require.NoError(t, err) _, err = fmt.Fprintf(conn, "psubscribe b*2 c*3\r\n") require.NoError(t, err) // collect 50 messages from each channel rd := bufio.NewReader(conn) var buf []byte for { line, err := rd.ReadBytes('\n') if err != nil { require.NoError(t, err) } buf = append(buf, line...) n, resp := redcon.ReadNextRESP(buf) if n == 0 { continue } buf = nil if resp.Type != redcon.Array { require.Fail(t, "expected array") } var vals []redcon.RESP resp.ForEach(func(item redcon.RESP) bool { vals = append(vals, item) return true }) name := string(vals[0].Data) switch name { case "subscribe": require.Len(t, vals, 3) ch := string(vals[1].Data) regs[ch] = 0 maxs, _ = strconv.Atoi(string(vals[2].Data)) case "psubscribe": require.Len(t, vals, 3) ch := string(vals[1].Data) regs[ch] = 0 maxp, _ = strconv.Atoi(string(vals[2].Data)) case "message": require.Len(t, vals, 3) ch := string(vals[1].Data) regs[ch] = regs[ch] + 1 case "pmessage": require.Len(t, vals, 4) ch := string(vals[1].Data) regs[ch] = regs[ch] + 1 } if len(regs) == 6 && maxp == 3 && maxs == 3 { ready := true for _, count := range regs { if count < 50 { ready = false break } } if ready { // all messages have been received return } } } }(i) } wg.Wait() // notify sender done <- true // wait for sender <-done // stop the timeout final <- true } ================================================ FILE: internal/pubsub/service.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package pubsub import ( "context" "sync" "github.com/olric-data/olric/internal/cluster/routingtable" "github.com/olric-data/olric/internal/environment" "github.com/olric-data/olric/internal/protocol" "github.com/olric-data/olric/internal/server" "github.com/olric-data/olric/internal/service" "github.com/olric-data/olric/internal/stats" "github.com/olric-data/olric/pkg/flog" ) var ( // PublishedTotal is the total number of published messages during the life of this instance. PublishedTotal = stats.NewInt64Counter() // CurrentSubscribers is the current number of listeners of Pub/Sub. CurrentSubscribers = stats.NewInt64Gauge() // SubscribersTotal is the total number of registered listeners during the life of this instance. SubscribersTotal = stats.NewInt64Counter() CurrentPSubscribers = stats.NewInt64Gauge() PSubscribersTotal = stats.NewInt64Counter() ) type Service struct { sync.RWMutex log *flog.Logger pubsub *PubSub rt *routingtable.RoutingTable server *server.Server client *server.Client wg sync.WaitGroup ctx context.Context cancel context.CancelFunc } func (s *Service) RegisterHandlers() { s.server.ServeMux().HandleFunc(protocol.PubSub.Subscribe, s.subscribeCommandHandler) s.server.ServeMux().HandleFunc(protocol.PubSub.PSubscribe, s.psubscribeCommandHandler) s.server.ServeMux().HandleFunc(protocol.PubSub.Publish, s.publishCommandHandler) s.server.ServeMux().HandleFunc(protocol.PubSub.PublishInternal, s.publishInternalCommandHandler) s.server.ServeMux().HandleFunc(protocol.PubSub.PubSubChannels, s.pubsubChannelsCommandHandler) s.server.ServeMux().HandleFunc(protocol.PubSub.PubSubNumpat, s.pubsubNumpatCommandHandler) s.server.ServeMux().HandleFunc(protocol.PubSub.PubSubNumsub, s.pubsubNumsubCommandHandler) } func NewService(e *environment.Environment) (service.Service, error) { ctx, cancel := context.WithCancel(context.Background()) ps := &PubSub{ unsubscribeCallback: func() { CurrentSubscribers.Decrease(1) }, punsubscribeCallback: func() { CurrentPSubscribers.Decrease(1) }, } s := &Service{ log: e.Get("logger").(*flog.Logger), rt: e.Get("routingtable").(*routingtable.RoutingTable), server: e.Get("server").(*server.Server), client: e.Get("client").(*server.Client), pubsub: ps, ctx: ctx, cancel: cancel, } s.RegisterHandlers() return s, nil } func (s *Service) Start() error { // dummy implementation return nil } func (s *Service) Shutdown(ctx context.Context) error { s.cancel() done := make(chan struct{}) go func() { s.wg.Wait() close(done) }() select { case <-ctx.Done(): err := ctx.Err() if err != nil { return err } case <-done: } return nil } var _ service.Service = (*Service)(nil) ================================================ FILE: internal/ramblock/compaction.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package ramblock import ( "errors" "fmt" "time" "github.com/olric-data/olric/internal/ramblock/table" "github.com/olric-data/olric/pkg/storage" ) func (rb *RamBlock) evictTable(t *table.Table) error { var total int var evictErr error t.Range(func(hkey uint64, e storage.Entry) bool { entry, _ := t.GetRaw(hkey) err := rb.PutRaw(hkey, entry) if errors.Is(err, table.ErrNotEnoughSpace) { err := rb.makeTable() if err != nil { evictErr = err return false } // try again return false } if err != nil { // log this error and continue evictErr = fmt.Errorf("put command failed: HKey: %d: %w", hkey, err) return false } err = t.Delete(hkey) if errors.Is(err, table.ErrHKeyNotFound) { err = nil } if err != nil { evictErr = err return false } total++ return total <= 1000 }) stats := t.Stats() if stats.Inuse == 0 { delete(rb.tablesByCoefficient, t.Coefficient()) t.Reset() } return evictErr } func (rb *RamBlock) isTableExpired(recycledAt int64) bool { timeout, err := rb.config.Get("maxIdleTableTimeout") if err != nil { // That would be impossible panic(err) } limit := (timeout.(time.Duration).Nanoseconds() + recycledAt) / 1000000 return (time.Now().UnixNano() / 1000000) >= limit } func (rb *RamBlock) isCompactionOK(t *table.Table) bool { s := t.Stats() return float64(s.Garbage) >= float64(s.Allocated)*maxGarbageRatio } func (rb *RamBlock) Compaction() (bool, error) { for _, t := range rb.tables { if rb.isCompactionOK(t) { err := rb.evictTable(t) if err != nil { return false, err } // Continue scanning return false, nil } } for i := 0; i < len(rb.tables); i++ { t := rb.tables[i] s := t.Stats() if t.State() == table.RecycledState { if rb.isTableExpired(s.RecycledAt) { if len(rb.tables) == 1 { break } delete(rb.tablesByCoefficient, t.Coefficient()) rb.tables = append(rb.tables[:i], rb.tables[i+1:]...) i-- } } } return true, nil } ================================================ FILE: internal/ramblock/compaction_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package ramblock import ( "fmt" "testing" "time" "github.com/cespare/xxhash/v2" "github.com/olric-data/olric/internal/ramblock/entry" "github.com/olric-data/olric/internal/ramblock/table" "github.com/stretchr/testify/require" ) func TestRamBlock_Compaction(t *testing.T) { s := testRamBlock(t, nil) timestamp := time.Now().UnixNano() // The current free space is 1 MB. Trigger a compaction operation. for i := 0; i < 1500; i++ { e := entry.New() e.SetKey(bkey(i)) e.SetTTL(int64(i)) e.SetValue([]byte(fmt.Sprintf("%01000d", i))) e.SetTTL(timestamp) hkey := xxhash.Sum64([]byte(e.Key())) err := s.Put(hkey, e) require.NoError(t, err) } for i := 0; i < 750; i++ { hkey := xxhash.Sum64([]byte(bkey(i))) err := s.Delete(hkey) require.NoError(t, err) } for { done, err := s.Compaction() require.NoError(t, err) if done { break } } var compacted bool for _, tb := range s.(*RamBlock).tables { stats := tb.Stats() if stats.Inuse == 0 { require.Equal(t, table.RecycledState, tb.State()) compacted = true } else { require.Equal(t, 750, stats.Length) require.Equal(t, table.ReadWriteState, tb.State()) } } require.Truef(t, compacted, "Compaction could not work properly") } func TestRamBlock_Compaction_MaxIdleTableDuration(t *testing.T) { c := DefaultConfig() c.Add("maxIdleTableTimeout", time.Millisecond) s := testRamBlock(t, c) timestamp := time.Now().UnixNano() // The current free space is 1 MB. Trigger a compaction operation. for i := 0; i < 1500; i++ { e := entry.New() e.SetKey(bkey(i)) e.SetTTL(int64(i)) e.SetValue([]byte(fmt.Sprintf("%01000d", i))) e.SetTTL(timestamp) hkey := xxhash.Sum64([]byte(e.Key())) err := s.Put(hkey, e) require.NoError(t, err) } require.Equal(t, 2, len(s.(*RamBlock).tables)) for i := 0; i < 800; i++ { hkey := xxhash.Sum64([]byte(bkey(i))) err := s.Delete(hkey) require.NoError(t, err) } // It's still two because we have not triggered the compaction yet. require.Equal(t, 2, len(s.(*RamBlock).tables)) for { done, err := s.Compaction() require.NoError(t, err) if done { break } } <-time.After(100 * time.Millisecond) // Be sure deletion of the idle table. for { done, err := s.Compaction() require.NoError(t, err) if done { break } } require.Equal(t, 1, len(s.(*RamBlock).tables)) } ================================================ FILE: internal/ramblock/entry/entry.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package entry import ( "encoding/binary" "github.com/olric-data/olric/pkg/storage" ) // In-memory layout for an entry: // // KEY-LENGTH(uint8) | KEY(bytes) | TTL(uint64) | | Timestamp(uint64) | VALUE-LENGTH(uint32) | VALUE(bytes) // Entry represents a value with its metadata. type Entry struct { key string ttl int64 timestamp int64 lastAccess int64 value []byte } var _ storage.Entry = (*Entry)(nil) func New() *Entry { return &Entry{} } func (e *Entry) SetKey(key string) { e.key = key } func (e *Entry) Key() string { return e.key } func (e *Entry) SetValue(value []byte) { e.value = value } func (e *Entry) Value() []byte { return e.value } func (e *Entry) SetTTL(ttl int64) { e.ttl = ttl } func (e *Entry) TTL() int64 { return e.ttl } func (e *Entry) SetTimestamp(timestamp int64) { e.timestamp = timestamp } func (e *Entry) Timestamp() int64 { return e.timestamp } func (e *Entry) SetLastAccess(lastAccess int64) { e.lastAccess = lastAccess } func (e *Entry) LastAccess() int64 { return e.lastAccess } func (e *Entry) Encode() []byte { var offset int klen := uint8(len(e.Key())) vlen := len(e.Value()) length := 29 + len(e.Key()) + vlen buf := make([]byte, length) // Set key length. It's 1 byte. copy(buf[offset:], []byte{klen}) offset++ // Set the key. copy(buf[offset:], e.Key()) offset += len(e.Key()) // Set the TTL. It's 8 bytes. binary.BigEndian.PutUint64(buf[offset:], uint64(e.TTL())) offset += 8 // Set the Timestamp. It's 8 bytes. binary.BigEndian.PutUint64(buf[offset:], uint64(e.Timestamp())) offset += 8 // Set the LastAccess. It's 8 bytes. binary.BigEndian.PutUint64(buf[offset:], uint64(e.LastAccess())) offset += 8 // Set the value length. It's 4 bytes. binary.BigEndian.PutUint32(buf[offset:], uint32(len(e.Value()))) offset += 4 // Set the value. copy(buf[offset:], e.Value()) return buf } func (e *Entry) Decode(buf []byte) { var offset int keyLength := int(buf[offset]) offset++ e.key = string(buf[offset : offset+keyLength]) offset += keyLength e.ttl = int64(binary.BigEndian.Uint64(buf[offset : offset+8])) offset += 8 e.timestamp = int64(binary.BigEndian.Uint64(buf[offset : offset+8])) offset += 8 e.lastAccess = int64(binary.BigEndian.Uint64(buf[offset : offset+8])) offset += 8 vlen := binary.BigEndian.Uint32(buf[offset : offset+4]) offset += 4 e.value = buf[offset : offset+int(vlen)] } ================================================ FILE: internal/ramblock/entry/entry_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package entry import ( "strings" "testing" "time" "github.com/stretchr/testify/require" ) func TestEntryEncodeDecode(t *testing.T) { e := New() e.SetKey("mykey") e.SetTTL(200) e.SetTimestamp(time.Now().UnixNano()) e.SetLastAccess(time.Now().UnixNano()) e.SetValue([]byte("mydata")) t.Run("Encode", func(t *testing.T) { buf := e.Encode() require.NotNilf(t, buf, "Expected some data. Got nil") t.Run("Decode", func(t *testing.T) { item := New() item.Decode(buf) require.Equalf(t, e, item, "Decoded Entry is different") }) }) } func TestEntry_Encode_EmptyKey(t *testing.T) { e := New() e.SetKey("") e.SetTTL(100) e.SetTimestamp(time.Now().UnixNano()) e.SetLastAccess(time.Now().UnixNano()) e.SetValue([]byte("somevalue")) buf := e.Encode() require.NotNil(t, buf) decoded := New() decoded.Decode(buf) require.Equal(t, "", decoded.Key()) require.Equal(t, []byte("somevalue"), decoded.Value()) require.Equal(t, e.TTL(), decoded.TTL()) require.Equal(t, e.Timestamp(), decoded.Timestamp()) require.Equal(t, e.LastAccess(), decoded.LastAccess()) } func TestEntry_Encode_EmptyValue(t *testing.T) { e := New() e.SetKey("mykey") e.SetTTL(50) e.SetTimestamp(time.Now().UnixNano()) e.SetLastAccess(time.Now().UnixNano()) // value is nil (zero value) buf := e.Encode() require.NotNil(t, buf) decoded := New() decoded.Decode(buf) require.Equal(t, "mykey", decoded.Key()) require.Empty(t, decoded.Value()) require.Equal(t, e.TTL(), decoded.TTL()) require.Equal(t, e.Timestamp(), decoded.Timestamp()) require.Equal(t, e.LastAccess(), decoded.LastAccess()) } func TestEntry_Encode_MaxLengthKey(t *testing.T) { // 255 bytes is the maximum key length that fits in a uint8. maxKey := strings.Repeat("k", 255) e := New() e.SetKey(maxKey) e.SetTTL(999) e.SetTimestamp(time.Now().UnixNano()) e.SetLastAccess(time.Now().UnixNano()) e.SetValue([]byte("val")) buf := e.Encode() require.NotNil(t, buf) decoded := New() decoded.Decode(buf) require.Equal(t, maxKey, decoded.Key()) require.Equal(t, []byte("val"), decoded.Value()) require.Equal(t, e.TTL(), decoded.TTL()) } func TestEntry_Encode_KeyLengthOverflow(t *testing.T) { // 256-byte key overflows uint8. The cast `uint8(256)` becomes 0, // so Encode writes the key length as 0 but copies the full 256 bytes // into the buffer. Decode then reads corrupted metadata offsets, // which causes a panic due to slice bounds out of range. overflowKey := strings.Repeat("x", 256) e := New() e.SetKey(overflowKey) e.SetTTL(1) e.SetTimestamp(time.Now().UnixNano()) e.SetLastAccess(time.Now().UnixNano()) e.SetValue([]byte("v")) buf := e.Encode() require.NotNil(t, buf) // Decode panics because the uint8-truncated key length (0) shifts all // field offsets, causing the decoded value length to be a garbage large // number that exceeds the buffer capacity. require.Panics(t, func() { decoded := New() decoded.Decode(buf) }, "Expected panic due to uint8 overflow of key length causing corrupted offsets") } ================================================ FILE: internal/ramblock/ramblock.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. /* Package ramblock implements a GC-friendly in-memory storage engine by using built-in maps and byte slices. It also supports compaction. */ package ramblock import ( "errors" "fmt" "io" "log" "reflect" "sort" "time" "github.com/olric-data/olric/internal/ramblock/entry" "github.com/olric-data/olric/internal/ramblock/table" "github.com/olric-data/olric/pkg/storage" ) const ( maxGarbageRatio = 0.40 // 1MB defaultTableSize = uint64(1 << 20) defaultMaxIdleTableTimeout = 15 * time.Minute ) // RamBlock implements an in-memory storage engine. type RamBlock struct { coefficient uint64 tableSize uint64 tablesByCoefficient map[uint64]*table.Table tables []*table.Table config *storage.Config } func DefaultConfig() *storage.Config { options := storage.NewConfig(nil) options.Add("tableSize", defaultTableSize) options.Add("maxIdleTableTimeout", defaultMaxIdleTableTimeout) return options } func New(c *storage.Config) (*RamBlock, error) { if c == nil { c = DefaultConfig() } raw, err := c.Get("tableSize") if err != nil { return nil, err } size, err := prepareTableSize(raw) if err != nil { return nil, err } return &RamBlock{ tableSize: size, tablesByCoefficient: make(map[uint64]*table.Table), config: c, }, nil } func (rb *RamBlock) SetConfig(c *storage.Config) { rb.config = c } func (rb *RamBlock) makeTable() error { if len(rb.tables) != 0 { head := rb.tables[len(rb.tables)-1] head.SetState(table.ReadOnlyState) for i, t := range rb.tables { if t.State() == table.RecycledState { rb.tables = append(rb.tables[:i], rb.tables[i+1:]...) rb.tables = append(rb.tables, t) t.SetCoefficient(rb.coefficient) rb.tablesByCoefficient[rb.coefficient] = t rb.coefficient++ t.SetState(table.ReadWriteState) return nil } } } newTable := table.New(rb.tableSize) rb.tables = append(rb.tables, newTable) newTable.SetCoefficient(rb.coefficient) rb.tablesByCoefficient[rb.coefficient] = newTable rb.coefficient++ return nil } func (rb *RamBlock) SetLogger(_ *log.Logger) {} func (rb *RamBlock) Start() error { if rb.config == nil { return errors.New("config cannot be nil") } return nil } func requiredSizeForAnEntry(e storage.Entry) uint64 { return uint64(len(e.Key()) + len(e.Value()) + table.MetadataLength) } func prepareTableSize(raw interface{}) (size uint64, err error) { switch raw.(type) { case uint: size = uint64(raw.(uint)) case uint8: size = uint64(raw.(uint8)) case uint16: size = uint64(raw.(uint16)) case uint32: size = uint64(raw.(uint32)) case uint64: size = raw.(uint64) case int: v := raw.(int) if v < 0 { err = fmt.Errorf("tableSize cannot be negative: %d", v) return } size = uint64(v) case int8: v := raw.(int8) if v < 0 { err = fmt.Errorf("tableSize cannot be negative: %d", v) return } size = uint64(v) case int16: v := raw.(int16) if v < 0 { err = fmt.Errorf("tableSize cannot be negative: %d", v) return } size = uint64(v) case int32: v := raw.(int32) if v < 0 { err = fmt.Errorf("tableSize cannot be negative: %d", v) return } size = uint64(v) case int64: v := raw.(int64) if v < 0 { err = fmt.Errorf("tableSize cannot be negative: %d", v) return } size = uint64(v) default: err = fmt.Errorf("invalid type for tableSize: %s", reflect.TypeOf(raw)) return } return } // Fork creates a new RamBlock instance. func (rb *RamBlock) Fork(c *storage.Config) (storage.Engine, error) { if c == nil { c = rb.config.Copy() } child, err := New(c) if err != nil { return nil, err } t := table.New(rb.tableSize) child.tables = append(child.tables, t) t.SetCoefficient(child.coefficient) child.tablesByCoefficient[child.coefficient] = t child.coefficient++ return child, nil } func (rb *RamBlock) Name() string { return "ramblock" } func (rb *RamBlock) NewEntry() storage.Entry { return entry.New() } // putWithRetry ensures at least one table exists and retries the given write // function on a new table when the current one runs out of space. func (rb *RamBlock) putWithRetry(writeFn func(t *table.Table) error) error { if len(rb.tables) == 0 { if err := rb.makeTable(); err != nil { return err } } for { // Get the last value, storage only calls Put on the last created table. t := rb.tables[len(rb.tables)-1] err := writeFn(t) if errors.Is(err, table.ErrNotEnoughSpace) { if err := rb.makeTable(); err != nil { return err } // try again continue } if err != nil { return err } // everything is ok return nil } } // PutRaw sets the raw value for the given key. func (rb *RamBlock) PutRaw(hkey uint64, value []byte) error { if uint64(len(value)) > rb.tableSize { return storage.ErrEntryTooLarge } return rb.putWithRetry(func(t *table.Table) error { return t.PutRaw(hkey, value) }) } // Put sets the value for the given key. It overwrites any previous value for that key func (rb *RamBlock) Put(hkey uint64, value storage.Entry) error { if requiredSizeForAnEntry(value) > rb.tableSize { return storage.ErrEntryTooLarge } return rb.putWithRetry(func(t *table.Table) error { return t.Put(hkey, value) }) } // GetRaw extracts encoded value for the given hkey. This is useful for merging tables. func (rb *RamBlock) GetRaw(hkey uint64) ([]byte, error) { // Scan available tables by starting the last added table. for i := len(rb.tables) - 1; i >= 0; i-- { t := rb.tables[i] raw, err := t.GetRaw(hkey) if errors.Is(err, table.ErrHKeyNotFound) { // Try out the other tables. continue } if err != nil { return nil, err } // Found the key, return the stored value with its metadata. return raw, nil } // Nothing here. return nil, storage.ErrKeyNotFound } // Get gets the value for the given key. It returns storage.ErrKeyNotFound if the DB // does not contain the key. The returned Entry is its own copy, // it is safe to modify the contents of the returned slice. func (rb *RamBlock) Get(hkey uint64) (storage.Entry, error) { // Scan available tables by starting the last added table. for i := len(rb.tables) - 1; i >= 0; i-- { t := rb.tables[i] res, err := t.Get(hkey) if errors.Is(err, table.ErrHKeyNotFound) { // Try out the other tables. continue } if err != nil { return nil, err } // Found the key, return the stored value with its metadata. return res, nil } // Nothing here. return nil, storage.ErrKeyNotFound } // GetTTL gets the timeout for the given key. It returns storage.ErrKeyNotFound if the DB // does not contain the key. func (rb *RamBlock) GetTTL(hkey uint64) (int64, error) { // Scan available tables by starting the last added table. for i := len(rb.tables) - 1; i >= 0; i-- { t := rb.tables[i] ttl, err := t.GetTTL(hkey) if errors.Is(err, table.ErrHKeyNotFound) { // Try out the other tables. continue } if err != nil { return 0, err } // Found the key, return its ttl return ttl, nil } // Nothing here. return 0, storage.ErrKeyNotFound } func (rb *RamBlock) GetLastAccess(hkey uint64) (int64, error) { // Scan available tables by starting the last added table. for i := len(rb.tables) - 1; i >= 0; i-- { t := rb.tables[i] lastAccess, err := t.GetLastAccess(hkey) if errors.Is(err, table.ErrHKeyNotFound) { // Try out the other tables. continue } if err != nil { return 0, err } // Found the key, return its ttl return lastAccess, nil } // Nothing here. return 0, storage.ErrKeyNotFound } // GetKey gets the key for the given hkey. It returns storage.ErrKeyNotFound if the DB // does not contain the key. func (rb *RamBlock) GetKey(hkey uint64) (string, error) { // Scan available tables by starting the last added table. for i := len(rb.tables) - 1; i >= 0; i-- { t := rb.tables[i] key, err := t.GetKey(hkey) if errors.Is(err, table.ErrHKeyNotFound) { // Try out the other tables. continue } if err != nil { return "", err } // Found the key, return its ttl return key, nil } // Nothing here. return "", storage.ErrKeyNotFound } // Delete deletes the value for the given key. Delete will not returns error if key doesn't exist. func (rb *RamBlock) Delete(hkey uint64) error { // Scan available tables by starting the last added table. for i := len(rb.tables) - 1; i >= 0; i-- { t := rb.tables[i] err := t.Delete(hkey) if errors.Is(err, table.ErrHKeyNotFound) { // Try out the other tables. continue } if err != nil { return err } break } return nil } // UpdateTTL updates the expiry for the given key. func (rb *RamBlock) UpdateTTL(hkey uint64, data storage.Entry) error { // Scan available tables by starting the last added table. for i := len(rb.tables) - 1; i >= 0; i-- { t := rb.tables[i] err := t.UpdateTTL(hkey, data) if errors.Is(err, table.ErrHKeyNotFound) { // Try out the other tables. continue } if err != nil { return err } // Found the key, return the stored value with its metadata. return nil } // Nothing here. return storage.ErrKeyNotFound } // Stats is a function which provides memory allocation and garbage ratio of a storage instance. func (rb *RamBlock) Stats() storage.Stats { stats := storage.Stats{ NumTables: len(rb.tables), } for _, t := range rb.tables { s := t.Stats() stats.Allocated += int(s.Allocated) stats.Inuse += int(s.Inuse) stats.Garbage += int(s.Garbage) stats.Length += s.Length } return stats } // Check checks the key existence. func (rb *RamBlock) Check(hkey uint64) bool { // Scan available tables by starting the last added table. for i := len(rb.tables) - 1; i >= 0; i-- { t := rb.tables[i] ok := t.Check(hkey) if ok { return true } } // Nothing there. return false } // Range calls f sequentially for each key and value present in the map. // If f returns false, range stops the iteration. Range may be O(N) with // the number of elements in the map even if f returns false after a constant // number of calls. func (rb *RamBlock) Range(f func(hkey uint64, e storage.Entry) bool) { // Scan available tables by starting the last added table. for i := len(rb.tables) - 1; i >= 0; i-- { t := rb.tables[i] t.Range(func(hkey uint64, e storage.Entry) bool { return f(hkey, e) }) } } // RangeHKey calls f sequentially for each key present in the map. // If f returns false, range stops the iteration. Range may be O(N) with // the number of elements in the map even if f returns false after a constant // number of calls. func (rb *RamBlock) RangeHKey(f func(hkey uint64) bool) { // Scan available tables by starting the last added table. for i := len(rb.tables) - 1; i >= 0; i-- { t := rb.tables[i] t.RangeHKey(func(hkey uint64) bool { return f(hkey) }) } } func (rb *RamBlock) findCoefficient(coefficient uint64) (uint64, error) { var sortedCoefficients []uint64 for newCf := range rb.tablesByCoefficient { sortedCoefficients = append(sortedCoefficients, newCf) } sort.Slice(sortedCoefficients, func(i, j int) bool { return sortedCoefficients[i] < sortedCoefficients[j] }) for _, cf := range sortedCoefficients { if cf > coefficient { return cf, nil } } return 0, io.EOF } func (rb *RamBlock) scanCommon(cursor uint64, expr string, count int, f func(e storage.Entry) bool) (uint64, error) { if len(rb.tables) == 0 { return 0, nil } var err error cf := cursor / rb.tableSize t, ok := rb.tablesByCoefficient[cf] if !ok { cf, err = rb.findCoefficient(cf) if err != nil { // Invalid cursor return 0, nil } t = rb.tablesByCoefficient[cf] cursor = cf * rb.tableSize } var tableCursor = cursor if cf > 0 { tableCursor = cursor - (rb.tableSize * cf) } if expr == "" { tableCursor, err = t.Scan(tableCursor, count, f) } else { tableCursor, err = t.ScanRegexMatch(tableCursor, expr, count, f) } if err != nil { return 0, err } if tableCursor == 0 { _, ok := rb.tablesByCoefficient[cf+1] if !ok { cf, err = rb.findCoefficient(cf) if err != nil { // Invalid cursor return 0, nil } // findCoefficient already returns the next valid coefficient return rb.tableSize * cf, nil } // The next table return rb.tableSize * (cf + 1), nil } return tableCursor + (rb.tableSize * cf), nil } func (rb *RamBlock) Scan(cursor uint64, count int, f func(e storage.Entry) bool) (uint64, error) { return rb.scanCommon(cursor, "", count, f) } func (rb *RamBlock) ScanRegexMatch(cursor uint64, expr string, count int, f func(e storage.Entry) bool) (uint64, error) { return rb.scanCommon(cursor, expr, count, f) } func (rb *RamBlock) Close() error { return nil } func (rb *RamBlock) Destroy() error { return nil } var _ storage.Engine = (*RamBlock)(nil) ================================================ FILE: internal/ramblock/ramblock_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package ramblock import ( "bytes" "errors" "fmt" "io" "strconv" "testing" "time" "github.com/cespare/xxhash/v2" "github.com/olric-data/olric/internal/ramblock/entry" "github.com/olric-data/olric/internal/ramblock/table" "github.com/olric-data/olric/pkg/storage" "github.com/stretchr/testify/require" ) func bkey(i int) string { return fmt.Sprintf("%09d", i) } func bval(i int) []byte { return []byte(fmt.Sprintf("%025d", i)) } func testRamBlock(t *testing.T, c *storage.Config) storage.Engine { kv, err := New(c) require.NoError(t, err) child, err := kv.Fork(nil) require.NoError(t, err) err = child.Start() require.NoError(t, err) return child } func TestRamBlock_Put(t *testing.T) { s := testRamBlock(t, nil) for i := 0; i < 100; i++ { e := entry.New() e.SetKey(bkey(i)) e.SetValue(bval(i)) e.SetTTL(int64(i)) e.SetTimestamp(time.Now().UnixNano()) hkey := xxhash.Sum64([]byte(e.Key())) err := s.Put(hkey, e) require.NoError(t, err) } } func TestRamBlock_Get(t *testing.T) { s := testRamBlock(t, nil) timestamp := time.Now().UnixNano() for i := 0; i < 100; i++ { e := entry.New() e.SetKey(bkey(i)) e.SetTTL(int64(i)) e.SetValue(bval(i)) e.SetTimestamp(timestamp) hkey := xxhash.Sum64([]byte(e.Key())) err := s.Put(hkey, e) require.NoError(t, err) } for i := 0; i < 100; i++ { hkey := xxhash.Sum64([]byte(bkey(i))) e, err := s.Get(hkey) require.NoError(t, err) require.Equal(t, bkey(i), e.Key()) require.Equal(t, int64(i), e.TTL()) require.Equal(t, bval(i), e.Value()) require.Equal(t, timestamp, e.Timestamp()) } } func TestRamBlock_Delete(t *testing.T) { s := testRamBlock(t, nil) for i := 0; i < 100; i++ { e := entry.New() e.SetKey(bkey(i)) e.SetTTL(int64(i)) e.SetValue(bval(i)) e.SetTimestamp(time.Now().UnixNano()) hkey := xxhash.Sum64([]byte(e.Key())) err := s.Put(hkey, e) require.NoError(t, err) } garbage := make(map[int]uint64) for i, tb := range s.(*RamBlock).tables { s := tb.Stats() garbage[i] = s.Inuse } for i := 0; i < 100; i++ { hkey := xxhash.Sum64([]byte(bkey(i))) err := s.Delete(hkey) require.NoError(t, err) _, err = s.Get(hkey) require.ErrorIs(t, err, storage.ErrKeyNotFound) } for i, tb := range s.(*RamBlock).tables { s := tb.Stats() require.Equal(t, uint64(0), s.Inuse) require.Equal(t, 0, s.Length) require.Equal(t, garbage[i], s.Garbage) } } func TestRamBlock_ExportImport(t *testing.T) { timestamp := time.Now().UnixNano() s := testRamBlock(t, nil) for i := 0; i < 1000; i++ { e := entry.New() e.SetKey(bkey(i)) e.SetTTL(int64(i)) e.SetValue(bval(i)) e.SetTimestamp(timestamp) hkey := xxhash.Sum64([]byte(e.Key())) err := s.Put(hkey, e) require.NoError(t, err) } fresh := testRamBlock(t, nil) ti := s.TransferIterator() for ti.Next() { data, index, err := ti.Export() require.NoError(t, err) err = fresh.Import(data, func(u uint64, e storage.Entry) error { return fresh.Put(u, e) }) require.NoError(t, err) err = ti.Drop(index) require.NoError(t, err) } _, _, err := ti.Export() require.ErrorIs(t, err, io.EOF) for i := 0; i < 1000; i++ { hkey := xxhash.Sum64([]byte(bkey(i))) e, err := fresh.Get(hkey) require.NoError(t, err) require.Equal(t, bkey(i), e.Key()) require.Equal(t, int64(i), e.TTL()) require.Equal(t, bval(i), e.Value()) require.Equal(t, timestamp, e.Timestamp()) } } func TestRamBlock_Stats_Length(t *testing.T) { s := testRamBlock(t, nil) for i := 0; i < 100; i++ { e := entry.New() e.SetKey(bkey(i)) e.SetTTL(int64(i)) e.SetValue(bval(i)) hkey := xxhash.Sum64([]byte(e.Key())) err := s.Put(hkey, e) require.NoError(t, err) } require.Equal(t, 100, s.Stats().Length) } func TestRamBlock_Range(t *testing.T) { s := testRamBlock(t, nil) hkeys := make(map[uint64]struct{}) for i := 0; i < 100; i++ { e := entry.New() e.SetKey(bkey(i)) e.SetTTL(int64(i)) e.SetValue(bval(i)) e.SetTimestamp(time.Now().UnixNano()) hkey := xxhash.Sum64([]byte(e.Key())) err := s.Put(hkey, e) require.NoError(t, err) hkeys[hkey] = struct{}{} } s.Range(func(hkey uint64, entry storage.Entry) bool { _, ok := hkeys[hkey] require.Truef(t, ok, "Invalid hkey: %d", hkey) return true }) } func TestRamBlock_Check(t *testing.T) { s := testRamBlock(t, nil) hkeys := make(map[uint64]struct{}) for i := 0; i < 100; i++ { e := entry.New() e.SetKey(bkey(i)) e.SetTTL(int64(i)) e.SetValue(bval(i)) e.SetTimestamp(time.Now().UnixNano()) hkey := xxhash.Sum64([]byte(e.Key())) err := s.Put(hkey, e) require.NoError(t, err) hkeys[hkey] = struct{}{} } for hkey := range hkeys { require.Truef(t, s.Check(hkey), "hkey could not be found: %d", hkey) } } func TestRamBlock_UpdateTTL(t *testing.T) { s := testRamBlock(t, nil) for i := 0; i < 100; i++ { e := entry.New() e.SetKey(bkey(i)) e.SetValue(bval(i)) e.SetTimestamp(time.Now().UnixNano()) hkey := xxhash.Sum64([]byte(e.Key())) err := s.Put(hkey, e) require.NoError(t, err) } for i := 0; i < 100; i++ { e := entry.New() e.SetKey(bkey(i)) e.SetTTL(10) e.SetTimestamp(time.Now().UnixNano()) hkey := xxhash.Sum64([]byte(e.Key())) err := s.UpdateTTL(hkey, e) require.NoError(t, err) } for i := 0; i < 100; i++ { hkey := xxhash.Sum64([]byte(bkey(i))) e, err := s.Get(hkey) require.NoError(t, err) if e.Key() != bkey(i) { t.Fatalf("Expected key: %s. Got %s", bkey(i), e.Key()) } if e.TTL() != 10 { t.Fatalf("Expected ttl: %d. Got %v", i, e.TTL()) } } } func TestRamBlock_GetKey(t *testing.T) { s := testRamBlock(t, nil) e := entry.New() e.SetKey(bkey(1)) e.SetTTL(int64(1)) e.SetValue(bval(1)) hkey := xxhash.Sum64([]byte(e.Key())) err := s.Put(hkey, e) require.NoError(t, err) key, err := s.GetKey(hkey) require.NoError(t, err) if key != bkey(1) { t.Fatalf("Expected %s. Got %v", bkey(1), key) } } func TestRamBlock_PutRawGetRaw(t *testing.T) { s := testRamBlock(t, nil) value := []byte("value") hkey := xxhash.Sum64([]byte("key")) err := s.PutRaw(hkey, value) require.NoError(t, err) rawval, err := s.GetRaw(hkey) require.NoError(t, err) if bytes.Equal(value, rawval) { t.Fatalf("Expected %s. Got %v", value, rawval) } } func TestRamBlock_GetTTL(t *testing.T) { s := testRamBlock(t, nil) e := entry.New() e.SetKey(bkey(1)) e.SetTTL(int64(1)) e.SetValue(bval(1)) hkey := xxhash.Sum64([]byte(e.Key())) err := s.Put(hkey, e) require.NoError(t, err) ttl, err := s.GetTTL(hkey) require.NoError(t, err) if ttl != e.TTL() { t.Fatalf("Expected TTL %d. Got %d", ttl, e.TTL()) } } func TestRamBlock_GetLastAccess(t *testing.T) { s := testRamBlock(t, nil) e := entry.New() e.SetKey(bkey(1)) e.SetTTL(int64(1)) e.SetValue(bval(1)) hkey := xxhash.Sum64([]byte(e.Key())) err := s.Put(hkey, e) require.NoError(t, err) lastAccess, err := s.GetLastAccess(hkey) require.NoError(t, err) require.NotEqual(t, 0, lastAccess) } func TestRamBlock_Fork(t *testing.T) { s := testRamBlock(t, nil) timestamp := time.Now().UnixNano() for i := 0; i < 10; i++ { e := entry.New() e.SetKey(bkey(i)) e.SetTTL(int64(i)) e.SetValue(bval(i)) e.SetTimestamp(timestamp) hkey := xxhash.Sum64([]byte(e.Key())) err := s.Put(hkey, e) require.NoError(t, err) } child, err := s.Fork(nil) require.NoError(t, err) for i := 0; i < 100; i++ { hkey := xxhash.Sum64([]byte(bkey(i))) _, err = child.Get(hkey) if !errors.Is(err, storage.ErrKeyNotFound) { t.Fatalf("Expected storage.ErrKeyNotFound. Got %v", err) } } stats := child.Stats() if uint64(stats.Allocated) != defaultTableSize { t.Fatalf("Expected Stats.Allocated: %d. Got: %d", defaultTableSize, stats.Allocated) } if stats.Inuse != 0 { t.Fatalf("Expected Stats.Inuse: 0. Got: %d", stats.Inuse) } if stats.Garbage != 0 { t.Fatalf("Expected Stats.Garbage: 0. Got: %d", stats.Garbage) } if stats.Length != 0 { t.Fatalf("Expected Stats.Length: 0. Got: %d", stats.Length) } if stats.NumTables != 1 { t.Fatalf("Expected Stats.NumTables: 1. Got: %d", stats.NumTables) } } func TestRamBlock_StateChange(t *testing.T) { s := testRamBlock(t, nil) timestamp := time.Now().UnixNano() // Current free space is 1 MB. Trigger a compaction operation. for i := 0; i < 100000; i++ { e := entry.New() e.SetKey(bkey(i)) e.SetTTL(int64(i)) e.SetValue([]byte(fmt.Sprintf("%01000d", i))) e.SetTTL(timestamp) hkey := xxhash.Sum64([]byte(e.Key())) err := s.Put(hkey, e) require.NoError(t, err) } for i, tb := range s.(*RamBlock).tables { if tb.State() == table.ReadWriteState { require.Equalf(t, len(s.(*RamBlock).tables)-1, i, "Writable table has to be the latest table") } else if tb.State() == table.ReadOnlyState { require.True(t, i < len(s.(*RamBlock).tables)-1) } } } func TestRamBlock_NewEntry(t *testing.T) { s := testRamBlock(t, nil) i := s.NewEntry() _, ok := i.(*entry.Entry) require.True(t, ok) } func TestRamBlock_Name(t *testing.T) { s := testRamBlock(t, nil) require.Equal(t, "ramblock", s.Name()) } func TestRamBlock_CloseDestroy(t *testing.T) { s := testRamBlock(t, nil) require.NoError(t, s.Close()) require.NoError(t, s.Destroy()) } func TestStorage_Scan(t *testing.T) { s := testRamBlock(t, nil) for i := 0; i < 1000000; i++ { e := entry.New() e.SetKey(bkey(i)) e.SetTTL(int64(i)) e.SetValue(bval(i)) e.SetTimestamp(time.Now().UnixNano()) hkey := xxhash.Sum64([]byte(e.Key())) err := s.Put(hkey, e) require.NoError(t, err) } var ( count int cursor uint64 err error ) k := s.(*RamBlock) for { cursor, err = k.Scan(cursor, 10, func(e storage.Entry) bool { count++ return true }) require.NoError(t, err) if cursor == 0 { break } } require.Equal(t, 1000000, count) } func TestStorage_ScanRegexMatch(t *testing.T) { s := testRamBlock(t, nil) var key string for i := 0; i < 1000000; i++ { if i%2 == 0 { key = "even:" + strconv.Itoa(i) } else { key = "odd:" + strconv.Itoa(i) } e := entry.New() e.SetKey(key) e.SetTTL(int64(i)) e.SetValue(bval(i)) e.SetTimestamp(time.Now().UnixNano()) hkey := xxhash.Sum64([]byte(e.Key())) err := s.Put(hkey, e) require.NoError(t, err) } var ( count int cursor uint64 err error ) k := s.(*RamBlock) for { cursor, err = k.ScanRegexMatch(cursor, "even:", 10, func(entry storage.Entry) bool { count++ return true }) require.NoError(t, err) if cursor == 0 { break } } require.Equal(t, 500000, count) } func TestStorage_ScanRegexMatch_OnlyOneEntry(t *testing.T) { s := testRamBlock(t, nil) for i := 0; i < 100; i++ { e := entry.New() e.SetKey(bkey(i)) e.SetTTL(int64(i)) e.SetValue(bval(i)) e.SetTimestamp(time.Now().UnixNano()) hkey := xxhash.Sum64([]byte(e.Key())) err := s.Put(hkey, e) require.NoError(t, err) } e := entry.New() e.SetKey("even:200") e.SetTTL(123123) e.SetValue([]byte("my-value")) e.SetTimestamp(time.Now().UnixNano()) hkey := xxhash.Sum64([]byte(e.Key())) err := s.Put(hkey, e) require.NoError(t, err) var ( num int count int cursor uint64 ) k := s.(*RamBlock) for { num += 1 cursor, err = k.ScanRegexMatch(cursor, "even:", 10, func(entry storage.Entry) bool { count++ require.Equal(t, "even:200", e.Key()) require.Equal(t, "my-value", string(e.Value())) return true }) require.NoError(t, err) if cursor == 0 { break } } require.Equal(t, 1, num) require.Equal(t, 1, count) } func TestStorage_Scan_NonContiguousCoefficients(t *testing.T) { // Use a small tableSize so that multiple tables are created quickly. c := DefaultConfig() c.Add("tableSize", 1024) s := testRamBlock(t, c) k := s.(*RamBlock) // Insert enough entries to create several tables (at least 4). for i := 0; i < 200; i++ { e := entry.New() e.SetKey(bkey(i)) e.SetValue(bval(i)) e.SetTTL(int64(i)) e.SetTimestamp(time.Now().UnixNano()) hkey := xxhash.Sum64([]byte(e.Key())) err := s.Put(hkey, e) require.NoError(t, err) } require.Greater(t, len(k.tables), 3, "need at least 4 tables for this test") // Count entries per table before deletion. totalBefore := 0 for _, tbl := range k.tables { totalBefore += tbl.Stats().Length } // Pick a middle table to delete (simulate compaction gap). // Find a table that is not the first or last and has entries. var deletedTable *table.Table for _, tbl := range k.tables[1 : len(k.tables)-1] { if tbl.Stats().Length > 0 { deletedTable = tbl break } } require.NotNil(t, deletedTable, "could not find a middle table to delete") deletedCf := deletedTable.Coefficient() deletedCount := deletedTable.Stats().Length // Remove the table from tablesByCoefficient to create a gap (like compaction does). delete(k.tablesByCoefficient, deletedCf) // Remove from tables slice as well. for i, tbl := range k.tables { if tbl == deletedTable { k.tables = append(k.tables[:i], k.tables[i+1:]...) break } } expectedCount := totalBefore - deletedCount // Scan all remaining entries. var ( scannedCount int cursor uint64 err error ) for { cursor, err = k.Scan(cursor, 10, func(e storage.Entry) bool { scannedCount++ return true }) require.NoError(t, err) if cursor == 0 { break } } require.Equal(t, expectedCount, scannedCount, "scan should find all entries in remaining tables after coefficient gap") } func TestRamBlock_Put_ErrEntryTooLarge(t *testing.T) { c := DefaultConfig() c.Add("tableSize", 1024) s := testRamBlock(t, c) value := make([]byte, 2048) e := entry.New() e.SetKey("key") e.SetValue(value) e.SetTTL(10) e.SetTimestamp(time.Now().UnixNano()) hkey := xxhash.Sum64([]byte(e.Key())) err := s.Put(hkey, e) require.ErrorIs(t, err, storage.ErrEntryTooLarge) } func TestPrepareTableSize_NegativeValues(t *testing.T) { negativeTests := []struct { name string raw interface{} }{ {"int", int(-1)}, {"int8", int8(-1)}, {"int16", int16(-1)}, {"int32", int32(-1)}, {"int64", int64(-1)}, } for _, tt := range negativeTests { t.Run(tt.name, func(t *testing.T) { _, err := prepareTableSize(tt.raw) require.Error(t, err) require.Contains(t, err.Error(), "tableSize cannot be negative") }) } } func TestPrepareTableSize_ValidValues(t *testing.T) { tests := []struct { name string raw interface{} expected uint64 }{ {"uint64", uint64(1024), 1024}, {"uint32", uint32(1024), 1024}, {"uint", uint(1024), 1024}, {"int", int(1024), 1024}, {"int64", int64(1024), 1024}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { size, err := prepareTableSize(tt.raw) require.NoError(t, err) require.Equal(t, tt.expected, size) }) } } func TestPrepareTableSize_InvalidType(t *testing.T) { _, err := prepareTableSize("invalid") require.Error(t, err) require.Contains(t, err.Error(), "invalid type for tableSize") } func TestRamBlock_New_NegativeTableSize(t *testing.T) { c := storage.NewConfig(nil) c.Add("tableSize", int(-1)) c.Add("maxIdleTableTimeout", defaultMaxIdleTableTimeout) _, err := New(c) require.Error(t, err) require.Contains(t, err.Error(), "tableSize cannot be negative") } func TestRamBlock_Start_NilConfig(t *testing.T) { kv, err := New(nil) require.NoError(t, err) kv.SetConfig(nil) err = kv.Start() require.Error(t, err) require.Equal(t, "config cannot be nil", err.Error()) } func TestRamBlock_PutRaw_ErrEntryTooLarge(t *testing.T) { c := DefaultConfig() c.Add("tableSize", 1024) s := testRamBlock(t, c) value := make([]byte, 2048) hkey := xxhash.Sum64([]byte("key")) err := s.PutRaw(hkey, value) require.ErrorIs(t, err, storage.ErrEntryTooLarge) } func TestRamBlock_GetRaw_KeyNotFound(t *testing.T) { s := testRamBlock(t, nil) hkey := xxhash.Sum64([]byte("nonexistent")) raw, err := s.GetRaw(hkey) require.ErrorIs(t, err, storage.ErrKeyNotFound) require.Nil(t, raw) } func TestRamBlock_GetTTL_KeyNotFound(t *testing.T) { s := testRamBlock(t, nil) hkey := xxhash.Sum64([]byte("nonexistent")) ttl, err := s.GetTTL(hkey) require.ErrorIs(t, err, storage.ErrKeyNotFound) require.Equal(t, int64(0), ttl) } func TestRamBlock_GetLastAccess_KeyNotFound(t *testing.T) { s := testRamBlock(t, nil) hkey := xxhash.Sum64([]byte("nonexistent")) lastAccess, err := s.GetLastAccess(hkey) require.ErrorIs(t, err, storage.ErrKeyNotFound) require.Equal(t, int64(0), lastAccess) } func TestRamBlock_GetKey_KeyNotFound(t *testing.T) { s := testRamBlock(t, nil) hkey := xxhash.Sum64([]byte("nonexistent")) key, err := s.GetKey(hkey) require.ErrorIs(t, err, storage.ErrKeyNotFound) require.Equal(t, "", key) } func TestRamBlock_Delete_NonExistentKey(t *testing.T) { s := testRamBlock(t, nil) hkey := xxhash.Sum64([]byte("nonexistent")) err := s.Delete(hkey) require.NoError(t, err) } func TestRamBlock_UpdateTTL_KeyNotFound(t *testing.T) { s := testRamBlock(t, nil) e := entry.New() e.SetTTL(100) e.SetTimestamp(time.Now().UnixNano()) hkey := xxhash.Sum64([]byte("nonexistent")) err := s.UpdateTTL(hkey, e) require.ErrorIs(t, err, storage.ErrKeyNotFound) } func TestRamBlock_Check_KeyNotFound(t *testing.T) { s := testRamBlock(t, nil) hkey := xxhash.Sum64([]byte("nonexistent")) require.False(t, s.Check(hkey)) } func TestRamBlock_RangeHKey(t *testing.T) { s := testRamBlock(t, nil) expected := make(map[uint64]struct{}) for i := 0; i < 100; i++ { e := entry.New() e.SetKey(bkey(i)) e.SetValue(bval(i)) hkey := xxhash.Sum64([]byte(e.Key())) err := s.Put(hkey, e) require.NoError(t, err) expected[hkey] = struct{}{} } collected := make(map[uint64]struct{}) s.RangeHKey(func(hkey uint64) bool { collected[hkey] = struct{}{} return true }) require.Equal(t, expected, collected) } func TestRamBlock_SetConfig(t *testing.T) { kv, err := New(nil) require.NoError(t, err) newConfig := storage.NewConfig(nil) newConfig.Add("tableSize", uint64(2048)) newConfig.Add("maxIdleTableTimeout", defaultMaxIdleTableTimeout) kv.SetConfig(newConfig) raw, err := kv.config.Get("tableSize") require.NoError(t, err) require.Equal(t, uint64(2048), raw) } func TestRamBlock_Fork_CustomConfig(t *testing.T) { s := testRamBlock(t, nil) customConfig := DefaultConfig() customConfig.Add("tableSize", uint64(2048)) child, err := s.Fork(customConfig) require.NoError(t, err) childKV := child.(*RamBlock) require.Equal(t, uint64(2048), childKV.tableSize) require.Equal(t, 1, len(childKV.tables)) require.Equal(t, 0, child.Stats().Length) } func TestRamBlock_MakeTable_RecycledTableReuse(t *testing.T) { s := testRamBlock(t, nil) timestamp := time.Now().UnixNano() // Insert entries with large values to fill multiple tables (default 1MB each). for i := 0; i < 1500; i++ { e := entry.New() e.SetKey(bkey(i)) e.SetTTL(int64(i)) e.SetValue([]byte(fmt.Sprintf("%01000d", i))) e.SetTimestamp(timestamp) hkey := xxhash.Sum64([]byte(e.Key())) err := s.Put(hkey, e) require.NoError(t, err) } // Delete enough entries to exceed the 40% garbage ratio on a table. for i := 0; i < 750; i++ { hkey := xxhash.Sum64([]byte(bkey(i))) err := s.Delete(hkey) require.NoError(t, err) } // Run compaction until done to produce RecycledState tables. for { done, err := s.Compaction() require.NoError(t, err) if done { break } } k := s.(*RamBlock) // Verify at least one recycled table exists. var recycledFound bool for _, tb := range k.tables { if tb.State() == table.RecycledState { recycledFound = true break } } require.True(t, recycledFound, "Expected at least one RecycledState table after compaction") tableCountBefore := len(k.tables) // Fill the current writable table to trigger makeTable via putWithRetry. startIdx := 2000 for i := startIdx; i < startIdx+1500; i++ { e := entry.New() e.SetKey(bkey(i)) e.SetTTL(int64(i)) e.SetValue([]byte(fmt.Sprintf("%01000d", i))) e.SetTimestamp(timestamp) hkey := xxhash.Sum64([]byte(e.Key())) err := s.Put(hkey, e) require.NoError(t, err) } // The recycled table should have been reused: no RecycledState tables remain. for _, tb := range k.tables { require.NotEqual(t, table.RecycledState, tb.State(), "Expected no RecycledState tables after reuse") } // The last table must be writable. lastTable := k.tables[len(k.tables)-1] require.Equal(t, table.ReadWriteState, lastTable.State()) // Table count should not have grown beyond what's needed (recycled table was reused). require.LessOrEqual(t, len(k.tables), tableCountBefore+1, "Expected recycled table reuse to limit table growth") } func TestRamBlock_EvictTable_PutRawError(t *testing.T) { s := testRamBlock(t, nil) timestamp := time.Now().UnixNano() // Insert entries with large values to fill multiple tables. for i := 0; i < 1500; i++ { e := entry.New() e.SetKey(bkey(i)) e.SetTTL(int64(i)) e.SetValue([]byte(fmt.Sprintf("%01000d", i))) e.SetTimestamp(timestamp) hkey := xxhash.Sum64([]byte(e.Key())) err := s.Put(hkey, e) require.NoError(t, err) } // Delete enough entries to exceed the 40% garbage ratio on a table. for i := 0; i < 750; i++ { hkey := xxhash.Sum64([]byte(bkey(i))) err := s.Delete(hkey) require.NoError(t, err) } k := s.(*RamBlock) // Shrink tableSize to force PutRaw to return ErrEntryTooLarge during eviction. k.tableSize = 1 done, err := k.Compaction() require.False(t, done) require.Error(t, err) require.ErrorIs(t, err, storage.ErrEntryTooLarge) } func TestRamBlock_Compaction_NoTables(t *testing.T) { k, err := New(nil) require.NoError(t, err) err = k.Start() require.NoError(t, err) // No tables exist — Compaction should return done=true with no error. done, compactionErr := k.Compaction() require.NoError(t, compactionErr) require.True(t, done) } func TestRamBlock_IsCompactionOK_ExactThreshold(t *testing.T) { // Each entry: key=1 byte + value=70 bytes + metadata=29 bytes = 100 bytes. // Table size 1000 → 9 entries fit. Deleting N entries → garbage = N*100. // maxGarbageRatio = 0.40, threshold = 1000 * 0.40 = 400. tests := []struct { name string deleteCount int expected bool }{ {"ExactBoundary", 4, true}, // garbage=400, ratio=0.40 → >=0.40 → true {"BelowBoundary", 3, false}, // garbage=300, ratio=0.30 → <0.40 → false {"AboveBoundary", 5, true}, // garbage=500, ratio=0.50 → >=0.40 → true } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { config := DefaultConfig() config.Add("tableSize", uint64(1000)) s := testRamBlock(t, config) timestamp := time.Now().UnixNano() // Insert 9 entries of exactly 100 bytes each. for i := 0; i < 9; i++ { e := entry.New() e.SetKey(fmt.Sprintf("%01d", i)) // 1-byte key e.SetValue([]byte(fmt.Sprintf("%070d", i))) // 70-byte value e.SetTimestamp(timestamp) hkey := xxhash.Sum64([]byte(e.Key())) err := s.Put(hkey, e) require.NoError(t, err) } // Delete entries to reach desired garbage level. for i := 0; i < tc.deleteCount; i++ { hkey := xxhash.Sum64([]byte(fmt.Sprintf("%01d", i))) err := s.Delete(hkey) require.NoError(t, err) } k := s.(*RamBlock) tb := k.tables[0] stats := tb.Stats() require.Equal(t, uint64(1000), stats.Allocated) require.Equal(t, uint64(tc.deleteCount*100), stats.Garbage, "Expected garbage = %d", tc.deleteCount*100) result := k.isCompactionOK(tb) require.Equal(t, tc.expected, result) }) } } func TestTransferIterator_Drop_EmptyTables(t *testing.T) { s := testRamBlock(t, nil) // Put a single entry so we have one table with data. e := entry.New() e.SetKey(bkey(0)) e.SetValue(bval(0)) e.SetTimestamp(time.Now().UnixNano()) hkey := xxhash.Sum64([]byte(e.Key())) err := s.Put(hkey, e) require.NoError(t, err) // Drain all tables via Export + Drop. ti := s.TransferIterator() for ti.Next() { _, index, err := ti.Export() require.NoError(t, err) err = ti.Drop(index) require.NoError(t, err) } // Now tables slice is empty. Drop should return an error. err = ti.Drop(0) require.Error(t, err) require.Contains(t, err.Error(), "there is no table to drop") } func TestTransferIterator_Export_SkipsRecycledState(t *testing.T) { s := testRamBlock(t, nil) k := s.(*RamBlock) // Insert enough data to create at least 2 tables. timestamp := time.Now().UnixNano() for i := 0; i < 100000; i++ { e := entry.New() e.SetKey(bkey(i)) e.SetTTL(int64(i)) e.SetValue([]byte(fmt.Sprintf("%01000d", i))) e.SetTimestamp(timestamp) hkey := xxhash.Sum64([]byte(e.Key())) err := s.Put(hkey, e) require.NoError(t, err) } require.Greater(t, len(k.tables), 1, "need at least 2 tables for this test") // Set the first table to RecycledState. k.tables[0].SetState(table.RecycledState) ti := s.TransferIterator() // Export should skip the recycled table and return the next non-recycled one. data, index, err := ti.Export() require.NoError(t, err) require.NotNil(t, data) require.Greater(t, index, 0, "Expected Export to skip index 0 (recycled table)") // Now set ALL tables to RecycledState. for _, tb := range k.tables { tb.SetState(table.RecycledState) } // Export should return io.EOF when all tables are recycled. _, _, err = ti.Export() require.ErrorIs(t, err, io.EOF) } ================================================ FILE: internal/ramblock/table/pack.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package table import ( "github.com/RoaringBitmap/roaring/roaring64" "github.com/vmihailenco/msgpack/v5" ) // Pack is the serializable representation of a Table. It is used by Encode and // Decode to transfer table data between nodes via msgpack serialization. type Pack struct { Offset uint64 Allocated uint64 Inuse uint64 Garbage uint64 RecycledAt int64 State State HKeys map[uint64]uint64 OffsetIndex []byte Memory []byte } // Encode serializes the given Table into a msgpack-encoded byte slice. Only the // active portion of the memory buffer (up to the current offset) is included. func Encode(t *Table) ([]byte, error) { offsetIndex, err := t.offsetIndex.MarshalBinary() if err != nil { return nil, err } p := Pack{ Offset: t.offset, Allocated: t.allocated, Inuse: t.inuse, Garbage: t.garbage, RecycledAt: t.recycledAt, State: t.state, HKeys: t.hkeys, OffsetIndex: offsetIndex, } p.Memory = make([]byte, t.offset) copy(p.Memory, t.memory[:t.offset]) return msgpack.Marshal(p) } // Decode deserializes a msgpack-encoded byte slice into a new Table, restoring // all entries, metadata, and the offset index. func Decode(data []byte) (*Table, error) { p := &Pack{} err := msgpack.Unmarshal(data, p) if err != nil { return nil, err } rb := roaring64.New() err = rb.UnmarshalBinary(p.OffsetIndex) if err != nil { return nil, err } t := New(p.Allocated) t.offset = p.Offset t.inuse = p.Inuse t.garbage = p.Garbage t.recycledAt = p.RecycledAt t.state = p.State t.hkeys = p.HKeys t.offsetIndex = rb copy(t.memory[:t.offset], p.Memory) return t, nil } ================================================ FILE: internal/ramblock/table/pack_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package table import ( "fmt" "testing" "time" "github.com/cespare/xxhash/v2" "github.com/olric-data/olric/internal/ramblock/entry" "github.com/stretchr/testify/require" ) func bkey(i int) string { return fmt.Sprintf("%09d", i) } func bval(i int) []byte { return []byte(fmt.Sprintf("%025d", i)) } func TestTable_Pack_Decode_CorruptData(t *testing.T) { _, err := Decode([]byte("this is not valid msgpack data")) require.Error(t, err) } func TestTable_Pack_EncodeDecode_GarbageAndRecycledAt(t *testing.T) { size := uint64(1 << 16) tb := New(size) for i := 0; i < 20; i++ { e := entry.New() e.SetKey(bkey(i)) e.SetValue(bval(i)) hkey := xxhash.Sum64([]byte(e.Key())) err := tb.Put(hkey, e) require.NoError(t, err) } // Delete some entries to create garbage for i := 0; i < 10; i++ { hkey := xxhash.Sum64([]byte(bkey(i))) err := tb.Delete(hkey) require.NoError(t, err) } statsBefore := tb.Stats() require.Greater(t, statsBefore.Garbage, uint64(0)) // Reset to set recycledAt tb.Reset() statsAfterReset := tb.Stats() require.NotEqual(t, int64(0), statsAfterReset.RecycledAt) // Re-add some entries after reset to have garbage again tb.SetState(ReadWriteState) for i := 100; i < 110; i++ { e := entry.New() e.SetKey(bkey(i)) e.SetValue(bval(i)) hkey := xxhash.Sum64([]byte(e.Key())) err := tb.Put(hkey, e) require.NoError(t, err) } // Delete a few to accumulate garbage for i := 100; i < 105; i++ { hkey := xxhash.Sum64([]byte(bkey(i))) err := tb.Delete(hkey) require.NoError(t, err) } statsBeforeEncode := tb.Stats() require.Greater(t, statsBeforeEncode.Garbage, uint64(0)) require.NotEqual(t, int64(0), statsBeforeEncode.RecycledAt) encoded, err := Encode(tb) require.NoError(t, err) decoded, err := Decode(encoded) require.NoError(t, err) statsAfterDecode := decoded.Stats() require.Equal(t, statsBeforeEncode.Garbage, statsAfterDecode.Garbage) require.Equal(t, statsBeforeEncode.RecycledAt, statsAfterDecode.RecycledAt) require.Equal(t, statsBeforeEncode.Inuse, statsAfterDecode.Inuse) require.Equal(t, statsBeforeEncode.Length, statsAfterDecode.Length) } func TestTable_Pack_EncodeDecode(t *testing.T) { size := uint64(1 << 16) tb := New(size) timestamp := time.Now().UnixNano() for i := 0; i < 100; i++ { e := entry.New() e.SetKey(bkey(i)) e.SetTTL(int64(i)) e.SetValue(bval(i)) e.SetLastAccess(timestamp) hkey := xxhash.Sum64([]byte(e.Key())) err := tb.Put(hkey, e) require.NoError(t, err) } encoded, err := Encode(tb) require.NoError(t, err) newTable, err := Decode(encoded) require.NoError(t, err) for i := 0; i < 100; i++ { hkey := xxhash.Sum64([]byte(bkey(i))) e, err := newTable.Get(hkey) require.NoError(t, err) require.Equal(t, e.Key(), bkey(i)) require.Equal(t, e.Value(), bval(i)) require.Equal(t, e.TTL(), int64(i)) require.NotEqual(t, timestamp, e.LastAccess()) } } ================================================ FILE: internal/ramblock/table/table.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package table import ( "encoding/binary" "fmt" "regexp" "sync" "time" "github.com/RoaringBitmap/roaring/roaring64" "github.com/olric-data/olric/internal/ramblock/entry" "github.com/olric-data/olric/pkg/storage" "github.com/pkg/errors" ) const ( // MaxKeyLength is the maximum allowed key size in bytes. MaxKeyLength = 256 // MetadataLength is the fixed number of bytes used to store per-entry metadata // (TTL + Timestamp + LastAccess + ValueLength + KeyLength = 8+8+8+4+1 = 29). MetadataLength = 29 ) // State represents the operational state of a Table. type State uint8 const ( // ReadWriteState indicates the table accepts both reads and writes. ReadWriteState = State(iota + 1) // ReadOnlyState indicates the table only accepts read operations. ReadOnlyState // RecycledState indicates the table has been reset and is ready for reuse. RecycledState ) var ( // ErrNotEnoughSpace is returned when the table's pre-allocated memory buffer // does not have enough room to store a new entry. ErrNotEnoughSpace = errors.New("not enough space") // ErrHKeyNotFound is returned when the given hash key does not exist in the table. ErrHKeyNotFound = errors.New("hkey not found") ) // Stats holds memory usage statistics and metadata for a Table. type Stats struct { // Allocated is the total size of the pre-allocated memory buffer in bytes. Allocated uint64 // Inuse is the number of bytes currently occupied by active entries. Inuse uint64 // Garbage is the number of bytes occupied by deleted entries that have not been reclaimed. Garbage uint64 // Length is the number of active entries in the table. Length int // RecycledAt is the UnixNano timestamp of the last Reset call, or zero if never recycled. RecycledAt int64 } // Table is an in-memory key-value store backed by a pre-allocated byte slice. // Entries are written sequentially into the buffer using a compact binary layout: // // KEY-LENGTH(uint8) | KEY(bytes) | TTL(uint64) | TIMESTAMP(uint64) | LASTACCESS(uint64) | VALUE-LENGTH(uint32) | VALUE(bytes) // // A hash key (uint64) to offset mapping provides O(1) lookups. Deleted entries // are tracked as garbage but not reclaimed until the table is compacted or recycled. type Table struct { lastAccessMtx sync.RWMutex coefficient uint64 offset uint64 allocated uint64 inuse uint64 garbage uint64 recycledAt int64 state State hkeys map[uint64]uint64 offsetIndex *roaring64.Bitmap memory []byte } // New creates a new Table with a pre-allocated memory buffer of the given size in bytes. func New(size uint64) *Table { t := &Table{ hkeys: make(map[uint64]uint64), allocated: size, offsetIndex: roaring64.New(), state: ReadWriteState, } // From builtin.go: // // The size specifies the length. The capacity of the slice is // equal to its length. A second integer argument may be provided to // specify a different capacity; it must be no smaller than the // length. For example, make([]int, 0, 10) allocates an underlying array // of size 10 and returns a slice of length 0 and capacity 10 that is // backed by this underlying array. t.memory = make([]byte, size) return t } // SetCoefficient sets the coefficient value used for load-balancing and distribution purposes. func (t *Table) SetCoefficient(cf uint64) { t.coefficient = cf } // Coefficient returns the current coefficient value of the table. func (t *Table) Coefficient() uint64 { return t.coefficient } // SetState sets the operational state of the table. func (t *Table) SetState(s State) { t.state = s } // State returns the current operational state of the table. func (t *Table) State() State { return t.state } // PutRaw stores pre-encoded raw bytes into the table under the given hash key. // It copies the value directly into the memory buffer without any metadata encoding. // Returns ErrNotEnoughSpace if the buffer cannot accommodate the value. func (t *Table) PutRaw(hkey uint64, value []byte) error { // Check empty space on the allocated memory area. inuse := uint64(len(value)) if inuse+t.offset >= t.allocated { return ErrNotEnoughSpace } t.hkeys[hkey] = t.offset t.offsetIndex.Add(t.offset) copy(t.memory[t.offset:], value) t.inuse += inuse t.offset += inuse return nil } // Put stores a storage.Entry into the table under the given hash key. It encodes // the entry's key, TTL, timestamp, last access time and value into the memory buffer // using the following binary layout: // // KEY-LENGTH(uint8) | KEY(bytes) | TTL(uint64) | TIMESTAMP(uint64) | LASTACCESS(uint64) | VALUE-LENGTH(uint32) | VALUE(bytes) // // If the hash key already exists, the previous entry is deleted first. Returns // ErrNotEnoughSpace if the buffer cannot accommodate the entry, or storage.ErrKeyTooLarge // if the key exceeds MaxKeyLength. func (t *Table) Put(hkey uint64, value storage.Entry) error { if len(value.Key()) >= MaxKeyLength { return storage.ErrKeyTooLarge } // Check empty space on the allocated memory area. // TTL + Timestamp + LastAccess + + value-Length + key-Length inuse := uint64(len(value.Key()) + len(value.Value()) + MetadataLength) if inuse+t.offset >= t.allocated { return ErrNotEnoughSpace } // If we already have the key, delete it. err := t.Delete(hkey) if errors.Is(err, ErrHKeyNotFound) { err = nil } if err != nil { return err } t.hkeys[hkey] = t.offset t.offsetIndex.Add(t.offset) t.inuse += inuse // Set key length. It's 1 byte. klen := uint8(len(value.Key())) copy(t.memory[t.offset:], []byte{klen}) t.offset++ // Set the key. copy(t.memory[t.offset:], value.Key()) t.offset += uint64(len(value.Key())) // Set the TTL. It's 8 bytes. binary.BigEndian.PutUint64(t.memory[t.offset:], uint64(value.TTL())) t.offset += 8 // Set the Timestamp. It's 8 bytes. binary.BigEndian.PutUint64(t.memory[t.offset:], uint64(value.Timestamp())) t.offset += 8 // Set the last access. It's 8 bytes. binary.BigEndian.PutUint64(t.memory[t.offset:], uint64(time.Now().UnixNano())) t.offset += 8 // Set the value length. It's 4 bytes. binary.BigEndian.PutUint32(t.memory[t.offset:], uint32(len(value.Value()))) t.offset += 4 // Set the value. copy(t.memory[t.offset:], value.Value()) t.offset += uint64(len(value.Value())) return nil } // GetRaw returns the raw byte representation of the entry stored under the given hash key. // The returned slice is a copy and includes the full binary-encoded entry (key, metadata and value). // Returns ErrHKeyNotFound if the hash key does not exist. func (t *Table) GetRaw(hkey uint64) ([]byte, error) { offset, ok := t.hkeys[hkey] if !ok { return nil, ErrHKeyNotFound } start, end := offset, offset // In-memory structure: // 1 | klen | 8 | 8 | 8 | 4 | vlen // KEY-LENGTH(uint8) | KEY(bytes) | TTL(uint64) | TIMESTAMP(uint64) | LASTACCESS(uint64) | VALUE-LENGTH(uint64) | VALUE(bytes) klen := uint64(t.memory[end]) end++ // One byte to keep key length end += klen // key length end += 8 // TTL end += 8 // Timestamp end += 8 // LastAccess vlen := binary.BigEndian.Uint32(t.memory[end : end+4]) end += 4 // 4 bytes to keep value length end += uint64(vlen) // value length // Create a copy of the requested data. rawval := make([]byte, end-start) copy(rawval, t.memory[start:end]) return rawval, nil } // getRawKey reads and returns the raw key bytes from the memory buffer at the given offset. func (t *Table) getRawKey(offset uint64) ([]byte, error) { klen := uint64(t.memory[offset]) offset++ return t.memory[offset : offset+klen], nil } // GetRawKey returns the raw key bytes for the given hash key. // Returns ErrHKeyNotFound if the hash key does not exist. func (t *Table) GetRawKey(hkey uint64) ([]byte, error) { offset, ok := t.hkeys[hkey] if !ok { return nil, ErrHKeyNotFound } return t.getRawKey(offset) } // GetKey returns the key as a string for the given hash key. // Returns ErrHKeyNotFound if the hash key does not exist. func (t *Table) GetKey(hkey uint64) (string, error) { raw, err := t.GetRawKey(hkey) if raw == nil { return "", err } return string(raw), err } // GetTTL returns the TTL value in nanoseconds for the given hash key. // Returns ErrHKeyNotFound if the hash key does not exist. func (t *Table) GetTTL(hkey uint64) (int64, error) { offset, ok := t.hkeys[hkey] if !ok { return 0, ErrHKeyNotFound } klen := uint64(t.memory[offset]) offset++ offset += klen return int64(binary.BigEndian.Uint64(t.memory[offset : offset+8])), nil } // GetLastAccess returns the last access timestamp in nanoseconds for the given hash key. // Returns ErrHKeyNotFound if the hash key does not exist. func (t *Table) GetLastAccess(hkey uint64) (int64, error) { offset, ok := t.hkeys[hkey] if !ok { return 0, ErrHKeyNotFound } klen := uint64(t.memory[offset]) offset++ // Key length offset += klen // Key's itself offset += 8 // TTL offset += 8 // Timestamp return int64(binary.BigEndian.Uint64(t.memory[offset : offset+8])), nil } // get decodes a storage.Entry from the memory buffer at the given offset and updates // the entry's last access time to the current time. It is used internally by Scan methods. func (t *Table) get(offset uint64) storage.Entry { e := &entry.Entry{} // In-memory structure: // // KEY-LENGTH(uint8) | KEY(bytes) | TTL(uint64) | TIMESTAMP(uint64) | LASTACCESS(uint64) | VALUE-LENGTH(uint32) | VALUE(bytes) klen := uint64(t.memory[offset]) offset++ e.SetKey(string(t.memory[offset : offset+klen])) offset += klen e.SetTTL(int64(binary.BigEndian.Uint64(t.memory[offset : offset+8]))) offset += 8 e.SetTimestamp(int64(binary.BigEndian.Uint64(t.memory[offset : offset+8]))) offset += 8 // Every SCAN call updates the last access time. We have to serialize the access to that field. t.lastAccessMtx.RLock() e.SetLastAccess(int64(binary.BigEndian.Uint64(t.memory[offset : offset+8]))) t.lastAccessMtx.RUnlock() // Update the last access field lastAccess := uint64(time.Now().UnixNano()) t.lastAccessMtx.Lock() binary.BigEndian.PutUint64(t.memory[offset:], lastAccess) t.lastAccessMtx.Unlock() offset += 8 vlen := binary.BigEndian.Uint32(t.memory[offset : offset+4]) offset += 4 e.SetValue(t.memory[offset : offset+uint64(vlen)]) return e } // Get retrieves the storage.Entry for the given hash key and updates the entry's // last access time. Returns ErrHKeyNotFound if the hash key does not exist. func (t *Table) Get(hkey uint64) (storage.Entry, error) { offset, ok := t.hkeys[hkey] if !ok { return nil, ErrHKeyNotFound } return t.get(offset), nil } // Delete removes the entry associated with the given hash key from the table. // The occupied memory is marked as garbage but not reclaimed. Returns // ErrHKeyNotFound if the hash key does not exist. func (t *Table) Delete(hkey uint64) error { offset, ok := t.hkeys[hkey] if !ok { // Try the previous tables. return ErrHKeyNotFound } var garbage uint64 // key, 1 byte for key size, klen for key's actual length. klen := uint64(t.memory[offset]) // Delete the offset from offsetIndex t.offsetIndex.Remove(offset) offset += 1 + klen garbage += 1 + klen // TTL, skip it. offset += 8 garbage += 8 // Timestamp, skip it. offset += 8 garbage += 8 // LastAccess, skip it. offset += 8 garbage += 8 // value len and its header. vlen := binary.BigEndian.Uint32(t.memory[offset : offset+4]) garbage += 4 + uint64(vlen) // Delete it from metadata delete(t.hkeys, hkey) t.garbage += garbage t.inuse -= garbage return nil } // UpdateTTL updates the TTL and timestamp fields of the entry identified by the // given hash key in-place, and refreshes its last access time. Returns // ErrHKeyNotFound if the hash key does not exist. func (t *Table) UpdateTTL(hkey uint64, value storage.Entry) error { offset, ok := t.hkeys[hkey] if !ok { return ErrHKeyNotFound } // key, 1 byte for key size, klen for key's actual length. klen := uint64(t.memory[offset]) offset += 1 + klen // Set the new TTL. It's 8 bytes. binary.BigEndian.PutUint64(t.memory[offset:], uint64(value.TTL())) offset += 8 // Set the new Timestamp. It's 8 bytes. binary.BigEndian.PutUint64(t.memory[offset:], uint64(value.Timestamp())) offset += 8 // Update the last access field binary.BigEndian.PutUint64(t.memory[offset:], uint64(time.Now().UnixNano())) return nil } // Check reports whether the given hash key exists in the table. func (t *Table) Check(hkey uint64) bool { _, ok := t.hkeys[hkey] return ok } // Stats returns the current memory usage statistics for the table. func (t *Table) Stats() Stats { return Stats{ Allocated: t.allocated, Inuse: t.inuse, Garbage: t.garbage, Length: len(t.hkeys), RecycledAt: t.recycledAt, } } // Range iterates over all entries in the table, calling f for each one. // If f returns false, iteration stops. The iteration order is non-deterministic. func (t *Table) Range(f func(hkey uint64, e storage.Entry) bool) { for hkey := range t.hkeys { e, err := t.Get(hkey) if errors.Is(err, ErrHKeyNotFound) { panic(fmt.Errorf("hkey: %d found in index, but Get could not find it", hkey)) } if !f(hkey, e) { break } } } // RangeHKey iterates over all hash keys in the table without decoding entries. // If f returns false, iteration stops. The iteration order is non-deterministic. func (t *Table) RangeHKey(f func(hkey uint64) bool) { for hkey := range t.hkeys { if !f(hkey) { break } } } // Reset clears all entries and metadata, resets memory usage counters, and // transitions the table to RecycledState. The underlying memory buffer is // retained for reuse. func (t *Table) Reset() { if len(t.hkeys) != 0 { t.hkeys = make(map[uint64]uint64) } t.offsetIndex = roaring64.New() t.SetState(RecycledState) t.inuse = 0 t.garbage = 0 t.offset = 0 t.coefficient = 0 t.recycledAt = time.Now().UnixNano() } // Scan performs a cursor-based iteration over the table entries. Starting from // the given cursor position, it calls f for up to count entries. It returns // the next cursor to resume scanning, or 0 when all entries have been visited. // If f returns false, iteration stops early. func (t *Table) Scan(cursor uint64, count int, f func(e storage.Entry) bool) (uint64, error) { it := t.offsetIndex.Iterator() if cursor != 0 { it.AdvanceIfNeeded(cursor) } var num int for it.HasNext() && num < count { offset := it.Next() e := t.get(offset) if !f(e) { break } cursor = offset + 1 num++ } if !it.HasNext() { // end of the scan cursor = 0 } return cursor, nil } // ScanRegexMatch performs a cursor-based iteration like Scan, but only yields // entries whose keys match the given regular expression. Returns the next cursor // to resume scanning, or 0 when all entries have been visited. Returns an error // if the regular expression is invalid. func (t *Table) ScanRegexMatch(cursor uint64, expr string, count int, f func(e storage.Entry) bool) (uint64, error) { r, err := regexp.Compile(expr) if err != nil { return 0, err } it := t.offsetIndex.Iterator() if cursor != 0 { it.AdvanceIfNeeded(cursor) } var num int for it.HasNext() && num < count { offset := it.Next() key, _ := t.getRawKey(offset) if !r.Match(key) { continue } e := t.get(offset) if !f(e) { break } cursor = offset + 1 num++ } if !it.HasNext() { // end of the scan cursor = 0 } return cursor, nil } ================================================ FILE: internal/ramblock/table/table_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package table import ( "fmt" "strconv" "strings" "testing" "time" "github.com/cespare/xxhash/v2" "github.com/olric-data/olric/internal/ramblock/entry" "github.com/olric-data/olric/pkg/storage" "github.com/stretchr/testify/require" ) var key = "foobar" const hkey uint64 = 18071988 func setupTable() (*Table, storage.Entry) { tb := New(1024) e := entry.New() e.SetKey(key) e.SetValue([]byte("foobar-value")) return tb, e } func TestTable_Put(t *testing.T) { tb, e := setupTable() err := tb.Put(hkey, e) require.NoError(t, err) } func TestTable_Get(t *testing.T) { tb, e := setupTable() err := tb.Put(hkey, e) require.NoError(t, err) value, err := tb.Get(hkey) require.NoError(t, err) require.Equal(t, e.Key(), value.Key()) require.Equal(t, e.Value(), value.Value()) require.Equal(t, e.TTL(), value.TTL()) require.Equal(t, int64(0), e.LastAccess()) require.NotEqual(t, int64(0), value.LastAccess()) } func TestTable_Delete(t *testing.T) { tb, e := setupTable() err := tb.Put(hkey, e) require.NoError(t, err) err = tb.Delete(hkey) require.NoError(t, err) _, err = tb.Get(hkey) require.ErrorIs(t, ErrHKeyNotFound, err) } func TestTable_Check(t *testing.T) { tb, e := setupTable() err := tb.Put(hkey, e) require.NoError(t, err) require.True(t, tb.Check(hkey)) err = tb.Delete(hkey) require.NoError(t, err) require.False(t, tb.Check(hkey)) } func TestTable_PutRaw(t *testing.T) { tb, e := setupTable() err := tb.PutRaw(hkey, e.Encode()) require.NoError(t, err) value, err := tb.Get(hkey) require.NoError(t, err) require.Equal(t, e, value) } func TestTable_GetRaw(t *testing.T) { tb, e := setupTable() err := tb.Put(hkey, e) require.NoError(t, err) raw, err := tb.GetRaw(hkey) require.NoError(t, err) extracted := entry.New() extracted.Decode(raw) require.Equal(t, e.Key(), extracted.Key()) require.Equal(t, e.Value(), extracted.Value()) require.Equal(t, e.TTL(), extracted.TTL()) require.Equal(t, int64(0), e.LastAccess()) require.NotEqual(t, int64(0), extracted.LastAccess()) } func TestTable_GetRawKey(t *testing.T) { tb, e := setupTable() err := tb.Put(hkey, e) require.NoError(t, err) rawKey, err := tb.GetRawKey(hkey) require.NoError(t, err) require.Equal(t, key, string(rawKey)) } func TestTable_GetKey(t *testing.T) { tb, e := setupTable() err := tb.Put(hkey, e) require.NoError(t, err) k, err := tb.GetKey(hkey) require.NoError(t, err) require.Equal(t, key, k) } func TestTable_SetState(t *testing.T) { tb, _ := setupTable() tb.SetState(ReadOnlyState) require.Equal(t, ReadOnlyState, tb.State()) } func TestTable_GetTTL(t *testing.T) { tb, e := setupTable() ttl := time.Now().UnixNano() e.SetTTL(ttl) err := tb.Put(hkey, e) require.NoError(t, err) value, err := tb.GetTTL(hkey) require.NoError(t, err) require.Equal(t, ttl, value) } func TestTable_GetLastAccess(t *testing.T) { tb, e := setupTable() err := tb.Put(hkey, e) require.NoError(t, err) value, err := tb.GetLastAccess(hkey) require.NoError(t, err) require.NotEqual(t, 0, value) } func TestTable_UpdateTTL(t *testing.T) { tb, e := setupTable() ttl := time.Now().UnixNano() e.SetTTL(ttl) err := tb.Put(hkey, e) require.NoError(t, err) e.SetTTL(ttl + 1000) err = tb.UpdateTTL(hkey, e) require.NoError(t, err) value, err := tb.GetTTL(hkey) require.NoError(t, err) require.Equal(t, ttl+1000, value) } func TestTable_UpdateTTL_Update_LastAccess(t *testing.T) { tb, e := setupTable() err := tb.Put(hkey, e) require.NoError(t, err) lastAccessOne, err := tb.GetLastAccess(hkey) require.NoError(t, err) <-time.After(time.Millisecond) ttl := time.Now().UnixNano() + 1000 e.SetTTL(ttl) err = tb.UpdateTTL(hkey, e) require.NoError(t, err) lastAccessTwo, err := tb.GetLastAccess(hkey) require.NoError(t, err) require.Greater(t, lastAccessTwo, lastAccessOne) } func TestTable_State(t *testing.T) { tb, _ := setupTable() require.Equal(t, ReadWriteState, tb.State()) } func TestTable_Range(t *testing.T) { data := make(map[uint64]storage.Entry) tb := New(1 << 20) for i := 0; i < 100; i++ { e := entry.New() ikey := fmt.Sprintf("key-%d", i) idata := []byte(fmt.Sprintf("value-%d", i)) ihkey := xxhash.Sum64String(ikey) e.SetKey(ikey) e.SetValue(idata) data[ihkey] = e err := tb.Put(ihkey, e) require.NoError(t, err) } tb.Range(func(hk uint64, e storage.Entry) bool { item, ok := data[hk] require.True(t, ok) require.Equal(t, item.Key(), e.Key()) require.Equal(t, item.Value(), e.Value()) require.Equal(t, item.TTL(), e.TTL()) require.Equal(t, int64(0), item.LastAccess()) require.NotEqual(t, int64(0), e.LastAccess()) return true }) } func TestTable_Stats(t *testing.T) { tb := New(1 << 20) for i := 0; i < 100; i++ { e := entry.New() ikey := fmt.Sprintf("key-%d", i) idata := []byte(fmt.Sprintf("value-%d", i)) ihkey := xxhash.Sum64String(ikey) e.SetKey(ikey) e.SetValue(idata) err := tb.Put(ihkey, e) require.NoError(t, err) } s := tb.Stats() require.Equal(t, uint64(1<<20), s.Allocated) require.Equal(t, 100, s.Length) require.Equal(t, uint64(4280), s.Inuse) require.Equal(t, uint64(0), s.Garbage) for i := 0; i < 100; i++ { ikey := fmt.Sprintf("key-%d", i) ihkey := xxhash.Sum64String(ikey) err := tb.Delete(ihkey) require.NoError(t, err) } s = tb.Stats() require.Equal(t, uint64(1<<20), s.Allocated) require.Equal(t, 0, s.Length) require.Equal(t, uint64(0), s.Inuse) require.Equal(t, uint64(4280), s.Garbage) } func TestTable_Reset(t *testing.T) { tb := New(1 << 20) for i := 0; i < 100; i++ { e := entry.New() ikey := fmt.Sprintf("key-%d", i) idata := []byte(fmt.Sprintf("value-%d", i)) ihkey := xxhash.Sum64String(ikey) e.SetKey(ikey) e.SetValue(idata) err := tb.Put(ihkey, e) require.NoError(t, err) } tb.Reset() stats := tb.Stats() require.Equal(t, RecycledState, tb.State()) require.Equal(t, uint64(0), stats.Garbage) require.Equal(t, uint64(0), stats.Inuse) require.Equal(t, tb.allocated, stats.Allocated) require.Equal(t, 0, stats.Length) // Verify Scan returns no entries after Reset var count int cursor, err := tb.Scan(0, 100, func(e storage.Entry) bool { count++ return true }) require.NoError(t, err) require.Equal(t, uint64(0), cursor) require.Equal(t, 0, count) } func TestTable_Reset_RecycleScenario(t *testing.T) { tb := New(1 << 20) // Phase 1: Put initial entries for i := 0; i < 50; i++ { e := entry.New() ikey := fmt.Sprintf("old-key-%d", i) idata := []byte(fmt.Sprintf("old-value-%d", i)) ihkey := xxhash.Sum64String(ikey) e.SetKey(ikey) e.SetValue(idata) err := tb.Put(ihkey, e) require.NoError(t, err) } // Phase 2: Reset (simulates compaction recycling) tb.Reset() tb.SetState(ReadWriteState) // mirrors makeTable() behavior // Phase 3: Put new entries into the recycled table newKeys := make(map[string]string) for i := 0; i < 10; i++ { e := entry.New() ikey := fmt.Sprintf("new-key-%d", i) idata := fmt.Sprintf("new-value-%d", i) ihkey := xxhash.Sum64String(ikey) e.SetKey(ikey) e.SetValue([]byte(idata)) err := tb.Put(ihkey, e) require.NoError(t, err) newKeys[ikey] = idata } // Phase 4: Scan and verify ONLY new entries are returned scannedKeys := make(map[string]string) var cursor uint64 var err error for { cursor, err = tb.Scan(cursor, 10, func(e storage.Entry) bool { scannedKeys[e.Key()] = string(e.Value()) return true }) require.NoError(t, err) if cursor == 0 { break } } require.Equal(t, len(newKeys), len(scannedKeys)) for k, v := range newKeys { sv, ok := scannedKeys[k] require.True(t, ok, "expected key %s not found in scan results", k) require.Equal(t, v, sv) } } func TestTable_Scan(t *testing.T) { tb := New(1 << 20) for i := 0; i < 100; i++ { e := entry.New() key := fmt.Sprintf("key-%d", i) data := []byte(fmt.Sprintf("value-%d", i)) hkey := xxhash.Sum64String(key) e.SetKey(key) e.SetValue(data) err := tb.Put(hkey, e) require.NoError(t, err) } var err error var cursor uint64 for { cursor, err = tb.Scan(cursor, 10, func(e storage.Entry) bool { return true }) require.NoError(t, err) if cursor == 0 { break } } } func TestTable_ScanRegexMatch(t *testing.T) { tb := New(1 << 20) for i := 0; i < 100; i++ { if i%2 == 0 { key = "even:" + strconv.Itoa(i) } else { key = "odd:" + strconv.Itoa(i) } e := entry.New() data := []byte(fmt.Sprintf("value-%d", i)) hkey := xxhash.Sum64String(key) e.SetKey(key) e.SetValue(data) err := tb.Put(hkey, e) require.NoError(t, err) } var err error var num int var count int var cursor uint64 for { num++ cursor, err = tb.ScanRegexMatch(cursor, "even:", 10, func(e storage.Entry) bool { count++ return true }) require.NoError(t, err) if cursor == 0 { break } } require.Equal(t, 6, num) require.Equal(t, 50, count) } func TestTable_Put_ErrKeyTooLarge(t *testing.T) { tb := New(1 << 20) e := entry.New() longKey := strings.Repeat("k", MaxKeyLength) e.SetKey(longKey) e.SetValue([]byte("value")) err := tb.Put(hkey, e) require.ErrorIs(t, err, storage.ErrKeyTooLarge) } func TestTable_Put_OverwriteExistingKey(t *testing.T) { tb := New(1024) e1 := entry.New() e1.SetKey("mykey") e1.SetValue([]byte("old-value")) err := tb.Put(hkey, e1) require.NoError(t, err) statsBefore := tb.Stats() e2 := entry.New() e2.SetKey("mykey") e2.SetValue([]byte("new-value")) err = tb.Put(hkey, e2) require.NoError(t, err) got, err := tb.Get(hkey) require.NoError(t, err) require.Equal(t, "new-value", string(got.Value())) require.Equal(t, "mykey", got.Key()) statsAfter := tb.Stats() require.Equal(t, 1, statsAfter.Length) require.Greater(t, statsAfter.Garbage, statsBefore.Garbage) } func TestTable_PutRaw_ErrNotEnoughSpace(t *testing.T) { tb := New(16) data := make([]byte, 32) err := tb.PutRaw(hkey, data) require.ErrorIs(t, err, ErrNotEnoughSpace) } func TestTable_Put_ErrNotEnoughSpace(t *testing.T) { tb := New(16) e := entry.New() e.SetKey("mykey") e.SetValue([]byte("some-value-that-is-too-large")) err := tb.Put(hkey, e) require.ErrorIs(t, err, ErrNotEnoughSpace) } func TestTable_GetRaw_ErrHKeyNotFound(t *testing.T) { tb := New(1024) _, err := tb.GetRaw(hkey) require.ErrorIs(t, err, ErrHKeyNotFound) } func TestTable_Delete_ErrHKeyNotFound(t *testing.T) { tb := New(1024) err := tb.Delete(hkey) require.ErrorIs(t, err, ErrHKeyNotFound) } func TestTable_UpdateTTL_ErrHKeyNotFound(t *testing.T) { tb := New(1024) e := entry.New() e.SetTTL(time.Now().UnixNano()) err := tb.UpdateTTL(hkey, e) require.ErrorIs(t, err, ErrHKeyNotFound) } func TestTable_RangeHKey(t *testing.T) { tb := New(1 << 20) expected := make(map[uint64]struct{}) for i := 0; i < 50; i++ { e := entry.New() ikey := fmt.Sprintf("key-%d", i) ihkey := xxhash.Sum64String(ikey) e.SetKey(ikey) e.SetValue([]byte(fmt.Sprintf("value-%d", i))) err := tb.Put(ihkey, e) require.NoError(t, err) expected[ihkey] = struct{}{} } collected := make(map[uint64]struct{}) tb.RangeHKey(func(hk uint64) bool { collected[hk] = struct{}{} return true }) require.Equal(t, expected, collected) // Test early stop: callback returns false after first call var count int tb.RangeHKey(func(hk uint64) bool { count++ return false }) require.Equal(t, 1, count) } func TestTable_ScanRegexMatch_InvalidRegex(t *testing.T) { tb := New(1024) e := entry.New() e.SetKey("test") e.SetValue([]byte("value")) err := tb.Put(hkey, e) require.NoError(t, err) _, err = tb.ScanRegexMatch(0, "[invalid", 10, func(e storage.Entry) bool { return true }) require.Error(t, err) } func TestTable_Coefficient(t *testing.T) { tb := New(1024) require.Equal(t, uint64(0), tb.Coefficient()) tb.SetCoefficient(42) require.Equal(t, uint64(42), tb.Coefficient()) tb.SetCoefficient(0) require.Equal(t, uint64(0), tb.Coefficient()) } func TestTable_ScanRegexMatch_SingleMatch(t *testing.T) { tb := New(1 << 20) for i := 0; i < 100; i++ { e := entry.New() key := fmt.Sprintf("key-%d", i) data := []byte(fmt.Sprintf("value-%d", i)) hkey := xxhash.Sum64String(key) e.SetKey(key) e.SetValue(data) err := tb.Put(hkey, e) require.NoError(t, err) } e := entry.New() e.SetKey("even:200") e.SetTTL(123123) e.SetValue([]byte("my-value")) e.SetTimestamp(time.Now().UnixNano()) hkey := xxhash.Sum64([]byte(e.Key())) err := tb.Put(hkey, e) require.NoError(t, err) var num int var count int var cursor uint64 for { num++ cursor, err = tb.ScanRegexMatch(cursor, "even:", 10, func(e storage.Entry) bool { count++ require.Equal(t, "even:200", e.Key()) require.Equal(t, "my-value", string(e.Value())) return true }) require.NoError(t, err) if cursor == 0 { break } } require.Equal(t, 1, num) require.Equal(t, 1, count) } ================================================ FILE: internal/ramblock/transport.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package ramblock import ( "fmt" "io" "github.com/olric-data/olric/internal/ramblock/table" "github.com/olric-data/olric/pkg/storage" ) type transferIterator struct { storage *RamBlock } func (t *transferIterator) Next() bool { return len(t.storage.tables) != 0 } func (t *transferIterator) Drop(index int) error { if len(t.storage.tables) == 0 { return fmt.Errorf("there is no table to drop") } tb := t.storage.tables[index] t.storage.tables = append(t.storage.tables[:index], t.storage.tables[index+1:]...) delete(t.storage.tablesByCoefficient, tb.Coefficient()) return nil } func (t *transferIterator) Export() ([]byte, int, error) { for index, t := range t.storage.tables { if t.State() == table.RecycledState { continue } data, err := table.Encode(t) if err != nil { return nil, 0, err } return data, index, nil } return nil, 0, io.EOF } func (rb *RamBlock) Import(data []byte, f func(uint64, storage.Entry) error) error { tb, err := table.Decode(data) if err != nil { return err } tb.Range(func(hkey uint64, e storage.Entry) bool { return f(hkey, e) == nil }) return err } func (rb *RamBlock) TransferIterator() storage.TransferIterator { return &transferIterator{ storage: rb, } } ================================================ FILE: internal/resp/encoder.go ================================================ // Copyright (c) 2013 The github.com/go-redis/redis Authors. // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package resp import ( "encoding" "fmt" "io" "strconv" "time" "github.com/olric-data/olric/internal/util" ) type encoder interface { io.Writer io.ByteWriter WriteString(s string) (n int, err error) } type Encoder struct { encoder lenBuf []byte numBuf []byte } func New(e encoder) *Encoder { return &Encoder{ encoder: e, lenBuf: make([]byte, 64), numBuf: make([]byte, 64), } } func (e *Encoder) Encode(v interface{}) error { switch v := v.(type) { case nil: return e.string("") case string: return e.string(v) case []byte: return e.bytes(v) case int: return e.int(int64(v)) case int8: return e.int(int64(v)) case int16: return e.int(int64(v)) case int32: return e.int(int64(v)) case int64: return e.int(v) case uint: return e.uint(uint64(v)) case uint8: return e.uint(uint64(v)) case uint16: return e.uint(uint64(v)) case uint32: return e.uint(uint64(v)) case uint64: return e.uint(v) case float32: return e.float(float64(v)) case float64: return e.float(v) case bool: if v { return e.int(1) } return e.int(0) case time.Time: e.numBuf = v.AppendFormat(e.numBuf[:0], time.RFC3339Nano) return e.bytes(e.numBuf) case time.Duration: return e.int(v.Nanoseconds()) case encoding.BinaryMarshaler: b, err := v.MarshalBinary() if err != nil { return err } return e.bytes(b) default: return fmt.Errorf( "olric: can't marshal %T (implement encoding.BinaryMarshaler)", v) } } func (e *Encoder) bytes(b []byte) error { if _, err := e.Write(b); err != nil { return err } return nil } func (e *Encoder) string(s string) error { return e.bytes(util.StringToBytes(s)) } func (e *Encoder) uint(n uint64) error { e.numBuf = strconv.AppendUint(e.numBuf[:0], n, 10) return e.bytes(e.numBuf) } func (e *Encoder) int(n int64) error { e.numBuf = strconv.AppendInt(e.numBuf[:0], n, 10) return e.bytes(e.numBuf) } func (e *Encoder) float(f float64) error { e.numBuf = strconv.AppendFloat(e.numBuf[:0], f, 'f', -1, 64) return e.bytes(e.numBuf) } ================================================ FILE: internal/resp/encoder_test.go ================================================ package resp import ( "bytes" "encoding" "fmt" "testing" "time" "github.com/stretchr/testify/require" ) type MyType struct{} var _ encoding.BinaryMarshaler = (*MyType)(nil) func (t *MyType) MarshalBinary() ([]byte, error) { return []byte("hello"), nil } func (t *MyType) UnmarshalBinary(data []byte) error { if !bytes.Equal([]byte("hello"), data) { return fmt.Errorf("not equal") } return nil } func TestWriter_WriteArg(t *testing.T) { buf := bytes.NewBuffer(nil) w := New(buf) t.Run("uint64", func(t *testing.T) { defer buf.Reset() value := uint64(345353) err := w.Encode(value) require.NoError(t, err) scannedValue := new(uint64) err = Scan(buf.Bytes(), scannedValue) require.NoError(t, err) require.Equal(t, uint64(345353), *scannedValue) }) t.Run("nil", func(t *testing.T) { defer buf.Reset() err := w.Encode(nil) require.NoError(t, err) scannedValue := new(string) err = Scan(buf.Bytes(), scannedValue) require.NoError(t, err) require.Equal(t, "", *scannedValue) }) t.Run("string", func(t *testing.T) { defer buf.Reset() err := w.Encode("foobar") require.NoError(t, err) scannedValue := new(string) err = Scan(buf.Bytes(), scannedValue) require.NoError(t, err) require.Equal(t, "foobar", *scannedValue) }) t.Run("byte slice", func(t *testing.T) { defer buf.Reset() err := w.Encode([]byte("foobar")) require.NoError(t, err) scannedValue := new([]byte) err = Scan(buf.Bytes(), scannedValue) require.NoError(t, err) require.Equal(t, []byte("foobar"), *scannedValue) }) t.Run("int", func(t *testing.T) { defer buf.Reset() value := 345353 err := w.Encode(value) require.NoError(t, err) scannedValue := new(int) err = Scan(buf.Bytes(), scannedValue) require.NoError(t, err) require.Equal(t, 345353, *scannedValue) }) t.Run("int8", func(t *testing.T) { defer buf.Reset() value := int8(2) err := w.Encode(value) require.NoError(t, err) scannedValue := new(int8) err = Scan(buf.Bytes(), scannedValue) require.NoError(t, err) require.Equal(t, int8(2), *scannedValue) }) t.Run("int16", func(t *testing.T) { defer buf.Reset() value := int16(2) err := w.Encode(value) require.NoError(t, err) scannedValue := new(int16) err = Scan(buf.Bytes(), scannedValue) require.NoError(t, err) require.Equal(t, int16(2), *scannedValue) }) t.Run("int32", func(t *testing.T) { defer buf.Reset() value := int32(2) err := w.Encode(value) require.NoError(t, err) scannedValue := new(int32) err = Scan(buf.Bytes(), scannedValue) require.NoError(t, err) require.Equal(t, int32(2), *scannedValue) }) t.Run("int64", func(t *testing.T) { defer buf.Reset() value := int64(2) err := w.Encode(value) require.NoError(t, err) scannedValue := new(int64) err = Scan(buf.Bytes(), scannedValue) require.NoError(t, err) require.Equal(t, int64(2), *scannedValue) }) t.Run("uint", func(t *testing.T) { defer buf.Reset() value := uint(2) err := w.Encode(value) require.NoError(t, err) scannedValue := new(uint) err = Scan(buf.Bytes(), scannedValue) require.NoError(t, err) require.Equal(t, uint(2), *scannedValue) }) t.Run("uint8", func(t *testing.T) { defer buf.Reset() value := uint8(2) err := w.Encode(value) require.NoError(t, err) scannedValue := new(uint8) err = Scan(buf.Bytes(), scannedValue) require.NoError(t, err) require.Equal(t, uint8(2), *scannedValue) }) t.Run("uint16", func(t *testing.T) { defer buf.Reset() value := uint16(2) err := w.Encode(value) require.NoError(t, err) scannedValue := new(uint16) err = Scan(buf.Bytes(), scannedValue) require.NoError(t, err) require.Equal(t, uint16(2), *scannedValue) }) t.Run("uint32", func(t *testing.T) { defer buf.Reset() value := uint32(2) err := w.Encode(value) require.NoError(t, err) scannedValue := new(uint32) err = Scan(buf.Bytes(), scannedValue) require.NoError(t, err) require.Equal(t, uint32(2), *scannedValue) }) t.Run("uint64", func(t *testing.T) { defer buf.Reset() value := uint64(2) err := w.Encode(value) require.NoError(t, err) scannedValue := new(uint64) err = Scan(buf.Bytes(), scannedValue) require.NoError(t, err) require.Equal(t, uint64(2), *scannedValue) }) t.Run("float32", func(t *testing.T) { defer buf.Reset() value := float32(2) err := w.Encode(value) require.NoError(t, err) scannedValue := new(float32) err = Scan(buf.Bytes(), scannedValue) require.NoError(t, err) require.Equal(t, float32(2), *scannedValue) }) t.Run("float64", func(t *testing.T) { defer buf.Reset() value := float64(2) err := w.Encode(value) require.NoError(t, err) scannedValue := new(float64) err = Scan(buf.Bytes(), scannedValue) require.NoError(t, err) require.Equal(t, float64(2), *scannedValue) }) t.Run("bool", func(t *testing.T) { defer buf.Reset() value := true err := w.Encode(value) require.NoError(t, err) scannedValue := new(bool) err = Scan(buf.Bytes(), scannedValue) require.NoError(t, err) require.Equal(t, true, *scannedValue) }) t.Run("time.Time", func(t *testing.T) { defer buf.Reset() value := time.Now() err := w.Encode(value) require.NoError(t, err) scannedValue := new(time.Time) err = Scan(buf.Bytes(), scannedValue) require.NoError(t, err) }) t.Run("time.Duration", func(t *testing.T) { defer buf.Reset() value := time.Second err := w.Encode(value) require.NoError(t, err) scannedValue := new(time.Duration) err = Scan(buf.Bytes(), scannedValue) require.NoError(t, err) require.Equal(t, time.Second, *scannedValue) }) t.Run("encoding.BinaryMarshaler", func(t *testing.T) { defer buf.Reset() var value encoding.BinaryMarshaler = &MyType{} err := w.Encode(value) require.NoError(t, err) scannedValue := new(MyType) err = Scan(buf.Bytes(), scannedValue) require.NoError(t, err) require.Equal(t, MyType{}, *scannedValue) }) } ================================================ FILE: internal/resp/scan.go ================================================ // Copyright (c) 2013 The github.com/go-redis/redis Authors. // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package resp import ( "encoding" "fmt" "time" "github.com/olric-data/olric/internal/util" ) // Scan parses bytes `b` to `v` with appropriate type. // //nolint:gocyclo func Scan(b []byte, v interface{}) error { switch v := v.(type) { case nil: return fmt.Errorf("olric: Scan(nil)") case *string: *v = util.BytesToString(b) return nil case *[]byte: *v = b return nil case *int: var err error *v, err = util.Atoi(b) return err case *int8: n, err := util.ParseInt(b, 10, 8) if err != nil { return err } *v = int8(n) return nil case *int16: n, err := util.ParseInt(b, 10, 16) if err != nil { return err } *v = int16(n) return nil case *int32: n, err := util.ParseInt(b, 10, 32) if err != nil { return err } *v = int32(n) return nil case *int64: n, err := util.ParseInt(b, 10, 64) if err != nil { return err } *v = n return nil case *uint: n, err := util.ParseUint(b, 10, 64) if err != nil { return err } *v = uint(n) return nil case *uint8: n, err := util.ParseUint(b, 10, 8) if err != nil { return err } *v = uint8(n) return nil case *uint16: n, err := util.ParseUint(b, 10, 16) if err != nil { return err } *v = uint16(n) return nil case *uint32: n, err := util.ParseUint(b, 10, 32) if err != nil { return err } *v = uint32(n) return nil case *uint64: n, err := util.ParseUint(b, 10, 64) if err != nil { return err } *v = n return nil case *float32: n, err := util.ParseFloat(b, 32) if err != nil { return err } *v = float32(n) return err case *float64: var err error *v, err = util.ParseFloat(b, 64) return err case *bool: *v = len(b) == 1 && b[0] == '1' return nil case *time.Time: var err error *v, err = time.Parse(time.RFC3339Nano, util.BytesToString(b)) return err case *time.Duration: n, err := util.ParseInt(b, 10, 64) if err != nil { return err } *v = time.Duration(n) return nil case encoding.BinaryUnmarshaler: return v.UnmarshalBinary(b) default: return fmt.Errorf( "olric: can't unmarshal %T (consider implementing BinaryUnmarshaler)", v) } } ================================================ FILE: internal/roundrobin/round_robin.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package roundrobin import ( "errors" "fmt" "sync" ) // ErrEmptyInstance denotes that there is nothing in the round-robin instance to schedule. var ErrEmptyInstance = errors.New("empty round-robin instance") // RoundRobin implements quite simple round-robin scheduling algorithm to distribute load fairly between servers. type RoundRobin struct { // Mutual exclusion lock is required here because the Get method // is called concurrently by the client component, and it modifies the state // in every call. mtx sync.RWMutex current int items []string } // New returns a new RoundRobin instance. func New(items []string) *RoundRobin { return &RoundRobin{ current: 0, items: items, } } // Get returns an item. func (r *RoundRobin) Get() (string, error) { // Acquire the lock here. This function modifies the internal state. r.mtx.Lock() defer r.mtx.Unlock() if len(r.items) == 0 { return "", ErrEmptyInstance } if r.current >= len(r.items) { r.current %= len(r.items) } if r.current >= len(r.items) { return "", fmt.Errorf("round-robin: corrupted internal state") } item := r.items[r.current] r.current++ return item, nil } // Add adds a new item to the Round-Robin scheduler. func (r *RoundRobin) Add(item string) { r.mtx.Lock() defer r.mtx.Unlock() r.items = append(r.items, item) } // Delete deletes an item from the Round-Robin scheduler. func (r *RoundRobin) Delete(item string) { r.mtx.Lock() defer r.mtx.Unlock() for i := 0; i < len(r.items); i++ { if r.items[i] == item { r.items = append(r.items[:i], r.items[i+1:]...) i-- } } } // Length returns the count of items func (r *RoundRobin) Length() int { r.mtx.RLock() defer r.mtx.RUnlock() return len(r.items) } ================================================ FILE: internal/roundrobin/round_robin_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package roundrobin import ( "testing" "github.com/stretchr/testify/require" ) func TestRoundRobin(t *testing.T) { items := []string{"127.0.0.1:2323", "127.0.0.1:4556", "127.0.0.1:7889"} r := New(items) t.Run("Get", func(t *testing.T) { items := make(map[string]int) for i := 0; i < r.Length(); i++ { item, err := r.Get() require.NoError(t, err) items[item]++ } if len(items) != r.Length() { t.Fatalf("Expected item count: %d. Got: %d", r.Length(), len(items)) } }) t.Run("Add", func(t *testing.T) { item := "127.0.0.1:3320" r.Add(item) items := make(map[string]int) for i := 0; i < r.Length(); i++ { item, err := r.Get() require.NoError(t, err) items[item]++ } if _, ok := items[item]; !ok { t.Fatalf("Item not processed: %s", item) } if len(items) != r.Length() { t.Fatalf("Expected item count: %d. Got: %d", r.Length(), len(items)) } }) t.Run("Delete", func(t *testing.T) { item := "127.0.0.1:7889" r.Delete(item) items := make(map[string]int) for i := 0; i < r.Length(); i++ { item, err := r.Get() require.NoError(t, err) items[item]++ } if _, ok := items[item]; ok { t.Fatalf("Item stil exists: %s", item) } if len(items) != r.Length() { t.Fatalf("Expected item count: %d. Got: %d", r.Length(), len(items)) } }) } func TestRoundRobin_Delete_NonExistent(t *testing.T) { items := []string{"127.0.0.1:2323", "127.0.0.1:4556", "127.0.0.1:7889"} r := New(items) var fresh []string fresh = append(fresh, items...) for i, item := range fresh { if i+1 == len(items) { r.Delete(item) } else { r.Delete(item) } } } ================================================ FILE: internal/server/client.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "context" "errors" "fmt" "sync" "github.com/olric-data/olric/config" "github.com/olric-data/olric/internal/roundrobin" "github.com/redis/go-redis/v9" ) type Client struct { mu sync.RWMutex config *config.Client clients map[string]*redis.Client roundRobin *roundrobin.RoundRobin } func NewClient(c *config.Client) *Client { if c == nil { c = config.NewClient() err := c.Sanitize() if err != nil { panic(fmt.Sprintf("failed to sanitize client config: %s", err)) } } return &Client{ config: c, clients: make(map[string]*redis.Client), roundRobin: roundrobin.New(nil), } } func (c *Client) Addresses() map[string]struct{} { c.mu.RLock() defer c.mu.RUnlock() addresses := make(map[string]struct{}) for address := range c.clients { addresses[address] = struct{}{} } return addresses } func (c *Client) Get(addr string) *redis.Client { c.mu.RLock() rc, ok := c.clients[addr] if ok { c.mu.RUnlock() return rc } c.mu.RUnlock() // Need the lock for writing, we modify c.clients map and the round-robin // implementation updates its internal state. c.mu.Lock() defer c.mu.Unlock() // Need to check again, because another goroutine may have updated clients // between our calls to RUnlock and Lock. if rc, ok = c.clients[addr]; ok { return rc } opt := c.config.RedisOptions() opt.Protocol = 2 opt.Addr = addr rc = redis.NewClient(opt) c.clients[addr] = rc c.roundRobin.Add(addr) return rc } func (c *Client) pickNodeRoundRobin() (string, error) { c.mu.RLock() defer c.mu.RUnlock() addr, err := c.roundRobin.Get() if errors.Is(err, roundrobin.ErrEmptyInstance) { return "", fmt.Errorf("no available client found") } if err != nil { return "", err } return addr, nil } func (c *Client) Pick() (*redis.Client, error) { addr, err := c.pickNodeRoundRobin() if err != nil { return nil, err } return c.Get(addr), nil } func (c *Client) Close(addr string) error { c.mu.Lock() defer c.mu.Unlock() rc, ok := c.clients[addr] if ok { err := rc.Close() if err != nil { return err } c.roundRobin.Delete(addr) delete(c.clients, addr) } return nil } func (c *Client) Shutdown(ctx context.Context) error { c.mu.Lock() defer c.mu.Unlock() for addr, rc := range c.clients { select { case <-ctx.Done(): return ctx.Err() default: } if err := rc.Close(); err != nil { return err } delete(c.clients, addr) c.roundRobin.Delete(addr) } return nil } ================================================ FILE: internal/server/client_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "context" "net" "strconv" "testing" "github.com/olric-data/olric/config" "github.com/olric-data/olric/internal/protocol" "github.com/stretchr/testify/require" "github.com/tidwall/redcon" ) func TestServer_Client_Get(t *testing.T) { srv := newServer(t) srv.ServeMux().HandleFunc(protocol.Generic.Ping, func(conn redcon.Conn, cmd redcon.Command) { conn.WriteBulkString("pong") }) <-srv.StartedCtx.Done() addr := net.JoinHostPort(srv.config.BindAddr, strconv.Itoa(srv.config.BindPort)) c := config.NewClient() require.NoError(t, c.Sanitize()) cs := NewClient(c) rc := cs.Get(addr) ctx := context.Background() cmd := protocol.NewPing().Command(ctx) err := rc.Process(ctx, cmd) require.NoError(t, err) result, err := cmd.Result() require.NoError(t, err) require.Equal(t, "pong", result) t.Run("Fetch cached client", func(t *testing.T) { newClient := cs.Get(addr) require.Equal(t, rc, newClient) }) } func TestServer_Client_Pick(t *testing.T) { servers := make(map[string]*Server) for i := 0; i < 10; i++ { srv := newServer(t) srv.ServeMux().HandleFunc(protocol.Generic.Ping, func(conn redcon.Conn, cmd redcon.Command) { conn.WriteBulkString("pong") }) addr := net.JoinHostPort(srv.config.BindAddr, strconv.Itoa(srv.config.BindPort)) servers[addr] = srv } c := config.NewClient() require.NoError(t, c.Sanitize()) cs := NewClient(c) for addr, srv := range servers { <-srv.StartedCtx.Done() cs.Get(addr) } // All the servers have been started. clients := make(map[string]struct{}) for i := 0; i < 100; i++ { rc, err := cs.Pick() require.NoError(t, err) ctx := context.Background() cmd := protocol.NewPing().Command(ctx) err = rc.Process(ctx, cmd) require.NoError(t, err) result, err := cmd.Result() require.NoError(t, err) require.Equal(t, "pong", result) clients[rc.String()] = struct{}{} } require.Greater(t, len(clients), 1) } func TestServer_Client_Close(t *testing.T) { srv := newServer(t) <-srv.StartedCtx.Done() c := config.NewClient() require.NoError(t, c.Sanitize()) addr := net.JoinHostPort(srv.config.BindAddr, strconv.Itoa(srv.config.BindPort)) cs := NewClient(c) rc1 := cs.Get(addr) require.NoError(t, cs.Close(addr)) rc2 := cs.Get(addr) require.NotEqual(t, rc1, rc2) require.Len(t, cs.clients, 1) require.Equal(t, 1, cs.roundRobin.Length()) } func TestServer_Client_Shutdown(t *testing.T) { servers := make(map[string]*Server) for i := 0; i < 10; i++ { srv := newServer(t) srv.ServeMux().HandleFunc(protocol.Generic.Ping, func(conn redcon.Conn, cmd redcon.Command) { conn.WriteBulkString("pong") }) addr := net.JoinHostPort(srv.config.BindAddr, strconv.Itoa(srv.config.BindPort)) servers[addr] = srv } c := config.NewClient() require.NoError(t, c.Sanitize()) cs := NewClient(c) for addr, srv := range servers { <-srv.StartedCtx.Done() cs.Get(addr) } // All the servers have been started. err := cs.Shutdown(context.Background()) require.NoError(t, err) require.Empty(t, cs.clients) require.Equal(t, 0, cs.roundRobin.Length()) } ================================================ FILE: internal/server/handler.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "fmt" "github.com/olric-data/olric/internal/protocol" "github.com/olric-data/olric/internal/util" "github.com/tidwall/redcon" ) type ServeMuxWrapper struct { mux *ServeMux precond func(conn redcon.Conn, cmd redcon.Command) bool } // The HandlerFunc type is an adapter to allow the use of // ordinary functions as RESP handlers. If f is a function // with the appropriate signature, HandlerFunc(f) is a // Handler that calls f. type HandlerFunc func(conn redcon.Conn, cmd redcon.Command) type Handler struct { handler func(conn redcon.Conn, cmd redcon.Command) precondition func(conn redcon.Conn, cmd redcon.Command) bool } // ServeRESP calls f(w, r) func (h Handler) ServeRESP(conn redcon.Conn, cmd redcon.Command) { CommandsTotal.Increase(1) if len(cmd.Args) == 0 { // A client may form a bad message, prevent panicking. h.handler(conn, cmd) return } command := util.BytesToString(cmd.Args[0]) if command == "pubsub" || command == "PUBSUB" { command = fmt.Sprintf("%s %s", command, util.BytesToString(cmd.Args[1])) } // Do not call precondition function for the following commands: // * Internal.UpdateRouting // * Generic.Auth if command == protocol.Internal.UpdateRouting || command == protocol.Generic.Auth { h.handler(conn, cmd) return } if h.precondition == nil { // No precondition h.handler(conn, cmd) return } if h.precondition(conn, cmd) { h.handler(conn, cmd) } } // HandleFunc registers the handler function for the given command. func (m *ServeMuxWrapper) HandleFunc(command string, handler func(conn redcon.Conn, cmd redcon.Command)) { if handler == nil { panic("server: nil handler") } m.mux.Handle(command, Handler{ handler: handler, precondition: m.precond, }) } ================================================ FILE: internal/server/handler_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "context" "crypto/rand" "sync/atomic" "testing" "github.com/olric-data/olric/internal/protocol" "github.com/redis/go-redis/v9" "github.com/stretchr/testify/require" "github.com/tidwall/redcon" ) func respEcho(t *testing.T, s *Server) { data := make([]byte, 8) _, err := rand.Read(data) require.NoError(t, err) s.ServeMux().HandleFunc(protocol.DMap.Get, func(conn redcon.Conn, cmd redcon.Command) { conn.WriteBulk(data) }) <-s.StartedCtx.Done() rdb := redis.NewClient(defaultRedisOptions(s.config)) ctx := context.Background() cmd := protocol.NewGet("mydmap", "mykey").Command(ctx) err = rdb.Process(ctx, cmd) require.NoError(t, err) result, err := cmd.Bytes() require.NoError(t, err) require.Equal(t, data, result) } func TestHandler_ServeRESP_PreCondition(t *testing.T) { var precond int32 s := newServerWithPreConditionFunc(t, func(conn redcon.Conn, cmd redcon.Command) bool { atomic.AddInt32(&precond, 1) return true }) defer func() { require.NoError(t, s.Shutdown(context.Background())) }() respEcho(t, s) require.Equal(t, int32(1), atomic.LoadInt32(&precond)) } func TestHandler_ServeRESP_PreCondition_DontCheck(t *testing.T) { var precond int32 s := newServerWithPreConditionFunc(t, func(conn redcon.Conn, cmd redcon.Command) bool { atomic.AddInt32(&precond, 1) return true }) defer func() { require.NoError(t, s.Shutdown(context.Background())) }() data := make([]byte, 8) _, err := rand.Read(data) require.NoError(t, err) // The node is bootstrapped by UpdateRoutingCmd. Don't check any preconditions to run that command. s.ServeMux().HandleFunc(protocol.Internal.UpdateRouting, func(conn redcon.Conn, cmd redcon.Command) { conn.WriteBulk(data) }) <-s.StartedCtx.Done() rdb := redis.NewClient(defaultRedisOptions(s.config)) ctx := context.Background() cmd := protocol.NewUpdateRouting([]byte("dummy-data"), 1).Command(ctx) err = rdb.Process(ctx, cmd) require.NoError(t, err) result, err := cmd.Bytes() require.NoError(t, err) require.Equal(t, data, result) require.Equal(t, int32(0), atomic.LoadInt32(&precond)) } ================================================ FILE: internal/server/mux.go ================================================ // The MIT License (MIT) // // Copyright (c) 2016 Josh Baker // // Permission is hereby granted, free of charge, to any person obtaining a copy of // this software and associated documentation files (the "Software"), to deal in // the Software without restriction, including without limitation the rights to // use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of // the Software, and to permit persons to whom the Software is furnished to do so, // subject to the following conditions: // // The above copyright notice and this permission notice shall be included in all // copies or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS // FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR // COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER // IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN // CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. package server import ( "errors" "fmt" "strings" "github.com/olric-data/olric/internal/protocol" "github.com/olric-data/olric/internal/util" "github.com/tidwall/redcon" ) // errAuthRequired represents an error indicating that authentication is required to access the requested resource or operation. var errAuthRequired = errors.New("Authentication required.") // ServeMux is an RESP command multiplexer. type ServeMux struct { config *Config handlers map[string]redcon.Handler } // NewServeMux allocates and returns a new ServeMux. func NewServeMux(c *Config) *ServeMux { protocol.SetError("NOAUTH", errAuthRequired) return &ServeMux{ config: c, handlers: make(map[string]redcon.Handler), } } // HandleFunc registers the handler function for the given command. func (m *ServeMux) HandleFunc(command string, handler redcon.Handler) { if handler == nil { panic("olric: nil handler") } m.Handle(command, handler) } // Handle registers the handler for the given command. // If a handler already exists for command, Handle panics. func (m *ServeMux) Handle(command string, handler redcon.Handler) { if command == "" { panic("olric: invalid command") } if handler == nil { panic("olric: nil handler") } if _, exist := m.handlers[command]; exist { panic("olric: multiple registrations for " + command) } m.handlers[command] = handler } // ServeRESP dispatches the command to the handler. func (m *ServeMux) ServeRESP(conn redcon.Conn, cmd redcon.Command) { command := strings.ToLower(util.BytesToString(cmd.Args[0])) if m.config.RequireAuth && command != protocol.Generic.Auth { ctx := conn.Context().(*ConnContext) if !ctx.IsAuthenticated() { protocol.WriteError(conn, errAuthRequired) return } } if handler, ok := m.handlers[command]; ok { handler.ServeRESP(conn, cmd) return } if command == protocol.PubSub.PubSub { if len(cmd.Args) < 2 { protocol.WriteError(conn, fmt.Errorf("wrong number of arguments for '%s' command", command)) return } command = fmt.Sprintf("%s %s", command, util.BytesToString(cmd.Args[1])) } if handler, ok := m.handlers[command]; ok { handler.ServeRESP(conn, cmd) return } protocol.WriteError(conn, fmt.Errorf("unknown command '%s'", command)) } ================================================ FILE: internal/server/mux_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "context" "math/rand" "testing" "github.com/olric-data/olric/internal/protocol" "github.com/redis/go-redis/v9" "github.com/stretchr/testify/require" "github.com/tidwall/redcon" ) func TestMux_PubSub_Command(t *testing.T) { s := newServer(t) data := make([]byte, 8) _, err := rand.Read(data) require.NoError(t, err) s.ServeMux().HandleFunc(protocol.PubSub.PubSubNumpat, func(conn redcon.Conn, cmd redcon.Command) { conn.WriteInt(10) }) <-s.StartedCtx.Done() rdb := redis.NewClient(defaultRedisOptions(s.config)) ctx := context.Background() var args []interface{} args = append(args, "pubsub") args = append(args, "numpat") cmd := redis.NewIntCmd(ctx, args...) err = rdb.Process(ctx, cmd) require.NoError(t, err) num, err := cmd.Result() require.NoError(t, err) require.Equal(t, int64(10), num) } ================================================ FILE: internal/server/server.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "context" "net" "strconv" "sync" "time" "github.com/olric-data/olric/internal/checkpoint" "github.com/olric-data/olric/internal/stats" "github.com/olric-data/olric/pkg/flog" "github.com/tidwall/redcon" ) var ( // CommandsTotal is the total number of all requests broken down by command (get, put, etc.) and status. CommandsTotal = stats.NewInt64Counter() // ConnectionsTotal is the total number of connections opened since the server started running. ConnectionsTotal = stats.NewInt64Counter() // CurrentConnections is the current number of open connections. CurrentConnections = stats.NewInt64Gauge() // WrittenBytesTotal is the total number of bytes sent by this server to network. WrittenBytesTotal = stats.NewInt64Counter() // ReadBytesTotal is the total number of bytes read by this server from network. ReadBytesTotal = stats.NewInt64Counter() ) // Config is a composite type to bundle configuration parameters. type Config struct { BindAddr string BindPort int KeepAlivePeriod time.Duration IdleClose time.Duration RequireAuth bool } // ConnContext represents the context for a connection with authentication state management. type ConnContext struct { mtx sync.RWMutex // authenticated indicates whether the connection is successfully authenticated. authenticated bool } // NewConnContext initializes and returns a new instance of ConnContext for managing connection states like authentication. func NewConnContext() *ConnContext { return &ConnContext{} } // SetAuthenticated sets the authentication state of the connection to the specified value. It is thread-safe. func (c *ConnContext) SetAuthenticated(authenticated bool) { c.mtx.Lock() defer c.mtx.Unlock() c.authenticated = authenticated } // IsAuthenticated checks if the connection is authenticated. It is thread-safe and returns true if authenticated. func (c *ConnContext) IsAuthenticated() bool { c.mtx.RLock() defer c.mtx.RUnlock() return c.authenticated } // ConnWrapper is a wrapper around net.Conn that enables tracking of read and written bytes. type ConnWrapper struct { net.Conn } // Write sends data over the underlying connection and updates the total written bytes counter. // It returns the number of bytes written and any error encountered. func (cw *ConnWrapper) Write(b []byte) (n int, err error) { nr, err := cw.Conn.Write(b) if err != nil { return 0, err } WrittenBytesTotal.Increase(int64(nr)) return nr, nil } // Read reads data into the provided byte slice, updates the read bytes counter, and returns the number of bytes read. func (cw *ConnWrapper) Read(b []byte) (n int, err error) { nr, err := cw.Conn.Read(b) if err != nil { return 0, err } ReadBytesTotal.Increase(int64(nr)) return nr, nil } // ListenerWrapper is a wrapper around net.Listener that supports setting a TCP keep-alive period for accepted connections. type ListenerWrapper struct { net.Listener keepAlivePeriod time.Duration } // Accept waits for and returns the next connection to the ListenerWrapper, applying TCP keep-alive settings if specified. func (lw *ListenerWrapper) Accept() (net.Conn, error) { conn, err := lw.Listener.Accept() if err != nil { return nil, err } if tcpConn, ok := conn.(*net.TCPConn); ok { if lw.keepAlivePeriod != 0 { if keepAliveErr := tcpConn.SetKeepAlive(true); keepAliveErr != nil { return nil, keepAliveErr } if keepAliveErr := tcpConn.SetKeepAlivePeriod(lw.keepAlivePeriod); keepAliveErr != nil { return nil, keepAliveErr } } } return &ConnWrapper{conn}, nil } // Server is a TCP server struct that manages configurations, logging, and connection handling for RESP-based protocols. type Server struct { config *Config mux *ServeMux wmux *ServeMuxWrapper server *redcon.Server log *flog.Logger listener *ListenerWrapper StartedCtx context.Context started context.CancelFunc ctx context.Context cancel context.CancelFunc wg sync.WaitGroup // some components of the TCP server should be closed after the listener stopped chan struct{} } // New initializes and returns a new Server configured with the specified Config and Logger. func New(c *Config, l *flog.Logger) *Server { // The server has to be started properly before accepting connections. checkpoint.Add() ctx, cancel := context.WithCancel(context.Background()) startedCtx, started := context.WithCancel(context.Background()) s := &Server{ config: c, mux: NewServeMux(c), log: l, started: started, StartedCtx: startedCtx, stopped: make(chan struct{}), ctx: ctx, cancel: cancel, } s.wmux = &ServeMuxWrapper{mux: s.mux} return s } // SetPreConditionFunc sets a precondition function to be executed before serving each command on the server. func (s *Server) SetPreConditionFunc(f func(conn redcon.Conn, cmd redcon.Command) bool) { select { case <-s.StartedCtx.Done(): // It's already started. return default: } s.wmux.precond = f } func (s *Server) ServeMux() *ServeMuxWrapper { return s.wmux } // ListenAndServe starts the TCP server, initializes internal components, and begins accepting connections. func (s *Server) ListenAndServe() error { addr := net.JoinHostPort(s.config.BindAddr, strconv.Itoa(s.config.BindPort)) listener, err := net.Listen("tcp", addr) if err != nil { return err } lw := &ListenerWrapper{ Listener: listener, keepAlivePeriod: s.config.KeepAlivePeriod, } defer close(s.stopped) s.listener = lw srv := redcon.NewServer(addr, s.mux.ServeRESP, func(conn redcon.Conn) bool { conn.SetContext(NewConnContext()) ConnectionsTotal.Increase(1) CurrentConnections.Increase(1) return true }, func(conn redcon.Conn, err error) { CurrentConnections.Increase(-1) }, ) if s.config.IdleClose != 0 { srv.SetIdleClose(s.config.IdleClose) } s.server = srv // The TCP server has been started s.started() checkpoint.Pass() return s.server.Serve(lw) } // Shutdown gracefully shuts down the server without interrupting any active connections. // Shutdown works by first closing all open listeners, then closing all idle connections, // and then waiting indefinitely for connections to return to idle and then shut down. // If the provided context expires before the shutdown is complete, Shutdown returns // the context's error; otherwise it returns any error returned from closing the Server's // underlying Listener(s). func (s *Server) Shutdown(ctx context.Context) error { select { case <-s.ctx.Done(): // It's already closed. return nil default: } s.cancel() if s.server == nil { // There is nothing to close. return nil } var latestError error err := s.server.Close() if err != nil { s.log.V(2).Printf("[ERROR] Failed to close listener: %v", err) latestError = err } // Listener is closed successfully. Now we can await for closing // other components of the TCP server. <-s.stopped done := make(chan struct{}) go func() { s.wg.Wait() close(done) }() select { case <-ctx.Done(): err = ctx.Err() if err != nil { s.log.V(2).Printf("[ERROR] Context has an error: %v", err) latestError = err } case <-done: } return latestError } ================================================ FILE: internal/server/server_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "context" "log" "net" "os" "strconv" "testing" "time" "github.com/olric-data/olric/pkg/flog" "github.com/redis/go-redis/v9" "github.com/stretchr/testify/require" "github.com/tidwall/redcon" ) // getFreePort copied from testutil package to prevent cycle import. func getFreePort() (int, error) { addr, err := net.ResolveTCPAddr("tcp", "127.0.0.1:0") if err != nil { return 0, err } l, err := net.ListenTCP("tcp", addr) if err != nil { return 0, err } port := l.Addr().(*net.TCPAddr).Port if err := l.Close(); err != nil { return 0, err } return port, nil } func newServerWithPreConditionFunc(t *testing.T, precond func(conn redcon.Conn, cmd redcon.Command) bool) *Server { bindPort, err := getFreePort() if err != nil { t.Fatalf("Expected nil. Got: %v", err) } l := log.New(os.Stdout, "server-test: ", log.LstdFlags) fl := flog.New(l) fl.SetLevel(6) fl.ShowLineNumber(1) c := &Config{ BindAddr: "127.0.0.1", BindPort: bindPort, KeepAlivePeriod: time.Second, } s := New(c, fl) s.SetPreConditionFunc(precond) go func() { err := s.ListenAndServe() if err != nil { t.Errorf("Expected nil. Got: %v", err) } }() t.Cleanup(func() { err = s.Shutdown(context.Background()) if err != nil { t.Fatalf("Expected nil. Got: %v", err) } }) return s } func newServer(t *testing.T) *Server { srv := newServerWithPreConditionFunc(t, nil) t.Cleanup(func() { require.NoError(t, srv.Shutdown(context.Background())) }) return srv } func defaultRedisOptions(c *Config) *redis.Options { return &redis.Options{ Addr: net.JoinHostPort(c.BindAddr, strconv.Itoa(c.BindPort)), } } func TestServer_RESP(t *testing.T) { s := newServer(t) respEcho(t, s) } func TestServer_RESP_Stats(t *testing.T) { s := newServer(t) respEcho(t, s) require.NotEqual(t, int64(0), CommandsTotal.Read()) require.NotEqual(t, int64(0), ConnectionsTotal.Read()) require.NotEqual(t, int64(0), CurrentConnections.Read()) require.NotEqual(t, int64(0), WrittenBytesTotal.Read()) require.NotEqual(t, int64(0), ReadBytesTotal.Read()) } func TestConnContext_Authentication(t *testing.T) { ctx := NewConnContext() require.False(t, ctx.IsAuthenticated()) t.Run("Authenticated", func(t *testing.T) { ctx.SetAuthenticated(true) require.True(t, ctx.IsAuthenticated()) }) } ================================================ FILE: internal/service/service.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package service import ( "context" ) type Service interface { Start() error RegisterHandlers() Shutdown(ctx context.Context) error } ================================================ FILE: internal/stats/stats.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package stats import "sync/atomic" // Int64Counter is a cumulative metric that represents a single monotonically // increasing counter whose value can only increase or be reset to zero on restart. type Int64Counter struct { counter int64 } // NewInt64Counter returns a new Int64Counter func NewInt64Counter() *Int64Counter { return &Int64Counter{} } // Increase increases the counter by delta. func (c *Int64Counter) Increase(delta int64) { atomic.AddInt64(&c.counter, delta) } // Read returns the current value of counter. func (c *Int64Counter) Read() int64 { return atomic.LoadInt64(&c.counter) } // Reset sets zero to the underlying counter. func (c *Int64Counter) Reset() { atomic.StoreInt64(&c.counter, 0) } // Int64Gauge is a metric that represents a single numerical value that can // arbitrarily go up and down. type Int64Gauge struct { gauge int64 } // NewInt64Gauge returns a new Int64Gauge func NewInt64Gauge() *Int64Gauge { return &Int64Gauge{} } // Increase increases the gauge by delta. func (c *Int64Gauge) Increase(delta int64) { atomic.AddInt64(&c.gauge, delta) } // Decrease decreases the counter by delta. func (c *Int64Gauge) Decrease(delta int64) { atomic.AddInt64(&c.gauge, -1*delta) } // Read returns the current value of gauge. func (c *Int64Gauge) Read() int64 { return atomic.LoadInt64(&c.gauge) } // Reset sets zero to the underlying gauge. func (c *Int64Gauge) Reset() { atomic.StoreInt64(&c.gauge, 0) } ================================================ FILE: internal/stats/stats_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package stats import ( "sync" "testing" "github.com/stretchr/testify/require" ) func TestUint64Counter(t *testing.T) { c := NewInt64Counter() var wg sync.WaitGroup for i := 0; i < 100; i++ { wg.Add(1) go func() { defer wg.Done() c.Increase(1) }() } wg.Wait() require.Equal(t, int64(100), c.Read()) } func TestUint64Gauge(t *testing.T) { g := NewInt64Gauge() var wg sync.WaitGroup for i := 0; i < 100; i++ { wg.Add(1) go func() { defer wg.Done() g.Increase(1) }() } for i := 0; i < 20; i++ { wg.Add(1) go func() { defer wg.Done() g.Decrease(1) }() } wg.Wait() require.Equal(t, int64(80), g.Read()) } ================================================ FILE: internal/testcluster/testcluster.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package testcluster import ( "context" "fmt" "net" "strconv" "sync" "github.com/olric-data/olric/config" "github.com/olric-data/olric/internal/cluster/balancer" "github.com/olric-data/olric/internal/cluster/partitions" "github.com/olric-data/olric/internal/cluster/routingtable" "github.com/olric-data/olric/internal/environment" "github.com/olric-data/olric/internal/locker" "github.com/olric-data/olric/internal/server" "github.com/olric-data/olric/internal/service" "github.com/olric-data/olric/internal/testutil" "golang.org/x/sync/errgroup" ) type TestCluster struct { mu sync.Mutex environments []*environment.Environment memberPorts []int constructor func(e *environment.Environment) (service.Service, error) errGr errgroup.Group ctx context.Context cancel context.CancelFunc } func NewEnvironment(c *config.Config) *environment.Environment { if c == nil { c = testutil.NewConfig() } e := environment.New() e.Set("config", c) e.Set("logger", testutil.NewFlogger(c)) e.Set("client", server.NewClient(c.Client)) e.Set("primary", partitions.New(c.PartitionCount, partitions.PRIMARY)) e.Set("backup", partitions.New(c.PartitionCount, partitions.BACKUP)) e.Set("locker", locker.New()) e.Set("server", testutil.NewServer(c)) return e } func (t *TestCluster) newService(e *environment.Environment) service.Service { rt := routingtable.New(e) e.Set("routingtable", rt) b := balancer.New(e) e.Set("balancer", b) t.errGr.Go(func() error { <-t.ctx.Done() return b.Shutdown(context.Background()) }) srv := e.Get("server").(*server.Server) go func() { err := srv.ListenAndServe() if err != nil { panic(fmt.Sprintf("ListenAndServe returned an error: %v", err)) } }() t.errGr.Go(func() error { <-t.ctx.Done() return srv.Shutdown(context.Background()) }) <-srv.StartedCtx.Done() s, err := t.constructor(e) if err != nil { panic(fmt.Sprintf("failed to start DMap service: %v", err)) } return s } func New(constructor func(e *environment.Environment) (service.Service, error)) *TestCluster { ctx, cancel := context.WithCancel(context.Background()) return &TestCluster{ constructor: constructor, ctx: ctx, cancel: cancel, } } func (t *TestCluster) syncCluster() { // Update routing table on the cluster before running balancer for _, e := range t.environments { rt := e.Get("routingtable").(*routingtable.RoutingTable) if rt.Discovery().IsCoordinator() { // The coordinator pushes the routing table immediately. // Normally, this is triggered by every cluster event but we don't want to // do this asynchronously to avoid randomness in tests. rt.UpdateEagerly() } } // Normally, balancer is triggered by routing table after a successful update, but we don't want to // balance the test cluster asynchronously. So we balance the partitions here explicitly. for _, e := range t.environments { e.Get("balancer").(*balancer.Balancer).BalanceEagerly() } } func (t *TestCluster) AddMember(e *environment.Environment) service.Service { t.mu.Lock() defer t.mu.Unlock() if e == nil { e = NewEnvironment(nil) } c := e.Get("config").(*config.Config) partitions.SetHashFunc(c.Hasher) port, err := testutil.GetFreePort() if err != nil { panic(fmt.Sprintf("failed to a random port: %v", err)) } c.MemberlistConfig.BindPort = port var peers []string for _, peerPort := range t.memberPorts { peers = append(peers, net.JoinHostPort("127.0.0.1", strconv.Itoa(peerPort))) } c.Peers = peers s := t.newService(e) rt := e.Get("routingtable").(*routingtable.RoutingTable) err = rt.Join() if err != nil { panic(fmt.Sprintf("failed to join the Olric cluster: %v", err)) } err = rt.Start() if err != nil { panic(fmt.Sprintf("failed to start the routing table: %v", err)) } t.errGr.Go(func() error { <-t.ctx.Done() return rt.Shutdown(context.Background()) }) t.errGr.Go(func() error { return s.Start() }) t.errGr.Go(func() error { <-t.ctx.Done() return s.Shutdown(context.Background()) }) t.environments = append(t.environments, e) t.memberPorts = append(t.memberPorts, port) t.syncCluster() return s } func (t *TestCluster) Shutdown() { t.cancel() err := t.errGr.Wait() if err != nil { panic(fmt.Sprintf("failed to shutdown the cluster: %v", err)) } } ================================================ FILE: internal/testutil/mockfragment/mockfragment.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mockfragment import ( "crypto/rand" "fmt" mrand "math/rand" "sync" "github.com/olric-data/olric/internal/cluster/partitions" "github.com/olric-data/olric/internal/discovery" "github.com/olric-data/olric/pkg/storage" ) type Result struct { Name string Owners []discovery.Member } type MockFragment struct { sync.RWMutex m map[string]interface{} result map[partitions.Kind]map[uint64]Result } func New() *MockFragment { return &MockFragment{ m: make(map[string]interface{}), result: make(map[partitions.Kind]map[uint64]Result), } } func (f *MockFragment) Stats() storage.Stats { f.Lock() defer f.Unlock() return storage.Stats{ Length: len(f.m), } } func (f *MockFragment) Name() string { return "Mock-DMap" } func (f *MockFragment) Put(key string, value interface{}) { f.Lock() defer f.Unlock() f.m[key] = value } func (f *MockFragment) Get(key string) interface{} { f.Lock() defer f.Unlock() return f.m[key] } func (f *MockFragment) Delete(key string) { f.Lock() defer f.Unlock() delete(f.m, key) } func (f *MockFragment) Fill() { n := 5 b := make([]byte, n) randKey := func() string { if _, err := rand.Read(b); err != nil { panic(err) } return fmt.Sprintf("%X", b) } num := mrand.Intn(100) for i := 0; i < num; i++ { f.Put(randKey(), i) } } func (f *MockFragment) Result() map[partitions.Kind]map[uint64]Result { return f.result } func (f *MockFragment) Move(part *partitions.Partition, name string, owners []discovery.Member) error { f.Lock() defer f.Unlock() f.result[part.Kind()] = map[uint64]Result{ part.ID(): { Name: name, Owners: owners, }, } for key := range f.m { delete(f.m, key) } return nil } func (f *MockFragment) Compaction() (bool, error) { return false, nil } func (f *MockFragment) Destroy() error { f.Lock() defer f.Unlock() f.m = make(map[string]interface{}) return nil } func (f *MockFragment) Close() error { return nil } var _ partitions.Fragment = (*MockFragment)(nil) ================================================ FILE: internal/testutil/testutil.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package testutil import ( "fmt" "net" "strconv" "testing" "time" "github.com/hashicorp/memberlist" "github.com/olric-data/olric/config" "github.com/olric-data/olric/internal/server" "github.com/olric-data/olric/pkg/flog" "github.com/stretchr/testify/require" ) func GetFreePort() (int, error) { addr, err := net.ResolveTCPAddr("tcp", "127.0.0.1:0") if err != nil { return 0, err } l, err := net.ListenTCP("tcp", addr) if err != nil { return 0, err } port := l.Addr().(*net.TCPAddr).Port if err := l.Close(); err != nil { return 0, err } return port, nil } func NewFlogger(c *config.Config) *flog.Logger { flogger := flog.New(c.Logger) flogger.SetLevel(c.LogVerbosity) if c.LogLevel == "DEBUG" { flogger.ShowLineNumber(1) } return flogger } func NewEngineConfig(t *testing.T) *config.Engine { e := config.NewEngine() require.NoError(t, e.Sanitize()) require.NoError(t, e.Validate()) return e } func NewConfig() *config.Config { c := config.New("local") c.PartitionCount = 7 mc := memberlist.DefaultLocalConfig() mc.BindAddr = "127.0.0.1" mc.BindPort = 0 c.MemberlistConfig = mc port, err := GetFreePort() if err != nil { panic(fmt.Sprintf("GetFreePort returned an error: %v", err)) } c.BindAddr = "127.0.0.1" c.BindPort = port c.MemberlistConfig.Name = net.JoinHostPort(c.BindAddr, strconv.Itoa(c.BindPort)) c.LeaveTimeout = 500 * time.Millisecond if err := c.Sanitize(); err != nil { panic(fmt.Sprintf("failed to sanitize default config: %v", err)) } return c } func NewServer(c *config.Config) *server.Server { sc := &server.Config{ BindAddr: c.BindAddr, BindPort: c.BindPort, KeepAlivePeriod: time.Second, } l := NewFlogger(c) return server.New(sc, l) } func TryWithInterval(max int, interval time.Duration, f func() error) error { ticker := time.NewTicker(interval) defer ticker.Stop() var err error err = f() if err == nil { // Done. No need to try with interval return nil } var count = 1 for count < max { <-ticker.C count++ err = f() if err == nil { break } } return err } func ToKey(i int) string { return fmt.Sprintf("%09d", i) } func ToVal(i int) []byte { return []byte(fmt.Sprintf("%010d", i)) } ================================================ FILE: internal/util/safe.go ================================================ // Copyright (c) 2013 The github.com/go-redis/redis Authors. // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. //go:build appengine // +build appengine package util func BytesToString(b []byte) string { return string(b) } func StringToBytes(s string) []byte { return []byte(s) } ================================================ FILE: internal/util/strconv.go ================================================ // Copyright (c) 2013 The github.com/go-redis/redis Authors. // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package util import "strconv" func Atoi(b []byte) (int, error) { return strconv.Atoi(BytesToString(b)) } func ParseInt(b []byte, base int, bitSize int) (int64, error) { return strconv.ParseInt(BytesToString(b), base, bitSize) } func ParseUint(b []byte, base int, bitSize int) (uint64, error) { return strconv.ParseUint(BytesToString(b), base, bitSize) } func ParseFloat(b []byte, bitSize int) (float64, error) { return strconv.ParseFloat(BytesToString(b), bitSize) } ================================================ FILE: internal/util/unsafe.go ================================================ // Copyright (c) 2013 The github.com/go-redis/redis Authors. // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. //go:build !appengine // +build !appengine package util import ( "unsafe" ) // BytesToString converts byte slice to string. func BytesToString(b []byte) string { return *(*string)(unsafe.Pointer(&b)) } // StringToBytes converts string to byte slice. func StringToBytes(s string) []byte { return *(*[]byte)(unsafe.Pointer( &struct { string Cap int }{s, len(s)}, )) } ================================================ FILE: olric-server-docker.yaml ================================================ server: # BindAddr denotes the address that Olric will bind to for communication # with other Olric nodes. bindAddr: 0.0.0.0 # BindPort denotes the address that Olric will bind to for communication # with other Olric nodes. bindPort: 3320 # KeepAlivePeriod denotes whether the operating system should send # keep-alive messages on the connection. keepAlivePeriod: 300s # IdleClose will automatically close idle connections after the specified duration. # Use zero to disable this feature. # idleClose: 300s # Timeout for bootstrap control # # An Olric node checks operation status before taking any action for the # cluster events, responding incoming requests and running API functions. # Bootstrapping status is one of the most important checkpoints for an # "operable" Olric node. BootstrapTimeout sets a deadline to check # bootstrapping status without blocking indefinitely. bootstrapTimeout: 5s # PartitionCount is 271, by default. partitionCount: 271 # ReplicaCount is 1, by default. replicaCount: 1 # Minimum number of successful writes to return a response for a write request. writeQuorum: 1 # Minimum number of successful reads to return a response for a read request. readQuorum: 1 # Switch to control read-repair algorithm which helps to reduce entropy. readRepair: false # Default value is SyncReplicationMode. replicationMode: 0 # sync mode. for async, set 1 # Minimum number of members to form a cluster and run any query on the cluster. memberCountQuorum: 1 # Coordinator member pushes the routing table to cluster members in the case of # node join or left events. It also pushes the table periodically. routingTablePushInterval # is the interval between subsequent calls. Default is 1 minute. routingTablePushInterval: 1m # Olric can send push cluster events to cluster.events channel. Available cluster events: # # * node-join-event # * node-left-event # * fragment-migration-event # * fragment-received-event # # If you want to receive these events, set true to EnableClusterEventsChannel and subscribe to # cluster.events channel. Default is false. enableClusterEventsChannel: true #authentication: # password: "your-password" client: # Timeout for TCP dial. # # The timeout includes name resolution, if required. When using TCP, and the host in the address parameter # resolves to multiple IP addresses, the timeout is spread over each consecutive dial, such that each is # given an appropriate fraction of the time to connect. dialTimeout: 5s # Timeout for socket reads. If reached, commands will fail # with a timeout instead of blocking. Use value -1 for no timeout and 0 for default. # Default is DefaultReadTimeout readTimeout: 3s # Timeout for socket writes. If reached, commands will fail # with a timeout instead of blocking. # Default is DefaultWriteTimeout writeTimeout: 3s # Maximum number of retries before giving up. # Default is 3 retries; -1 (not 0) disables retries. #maxRetries: 3 # Minimum backoff between each retry. # Default is 8 milliseconds; -1 disables backoff. #minRetryBackoff: 8ms # Maximum backoff between each retry. # Default is 512 milliseconds; -1 disables backoff. #maxRetryBackoff: 512ms # Type of connection pool. # true for FIFO pool, false for LIFO pool. # Note that fifo has higher overhead compared to lifo. #poolFIFO: false # Maximum number of socket connections. # Default is 10 connections per every available CPU as reported by runtime.GOMAXPROCS. #poolSize: 0 # Minimum number of idle connections which is useful when establishing # new connection is slow. #minIdleConns: # Connection age at which client retires (closes) the connection. # Default is to not close aged connections. #maxConnAge: # Amount of time client waits for connection if all connections are busy before # returning an error. Default is ReadTimeout + 1 second. #poolTimeout: 3s # Amount of time after which client closes idle connections. # Should be less than server's timeout. # Default is 5 minutes. -1 disables idle timeout check. idleTimeout: 5m # Frequency of idle checks made by idle connections reaper. # Default is 1 minute. -1 disables idle connections reaper, # but idle connections are still discarded by the client # if IdleTimeout is set. idleCheckFrequency: 1m logging: # DefaultLogVerbosity denotes default log verbosity level. # # * 1 - Generally useful for this to ALWAYS be visible to an operator # * Programmer errors # * Logging extra info about a panic # * CLI argument handling # * 2 - A reasonable default log level if you don't want verbosity. # * Information about config (listening on X, watching Y) # * Errors that repeat frequently that relate to conditions that can be # corrected # * 3 - Useful steady state information about the service and # important log messages that may correlate to # significant changes in the system. This is the recommended default log # level for most systems. # * Logging HTTP requests and their exit code # * System state changing # * Controller state change events # * Scheduler log messages # * 4 - Extended information about changes # * More info about system state changes # * 5 - Debug level verbosity # * Logging in particularly thorny parts of code where you may want to come # back later and check it # * 6 - Trace level verbosity # * Context to understand the steps leading up to neterrors and warnings # * More information for troubleshooting reported issues verbosity: 3 # Default LogLevel is DEBUG. Available levels: "DEBUG", "WARN", "ERROR", "INFO" level: WARN output: stderr memberlist: environment: lan # Configuration related to what address to bind to and ports to # listen on. The port is used for both UDP and TCP gossip. It is # assumed other nodes are running on this port, but they do not need # to. bindAddr: 0.0.0.0 bindPort: 3322 # EnableCompression is used to control message compression. This can # be used to reduce bandwidth usage at the cost of slightly more CPU # utilization. This is only available starting at protocol version 1. enableCompression: false # JoinRetryInterval is the time gap between attempts to join an existing # cluster. joinRetryInterval: 1ms # MaxJoinAttempts denotes the maximum number of attemps to join an existing # cluster before forming a new one. maxJoinAttempts: 1 # See service discovery plugins #peers: # - "localhost:3325" #advertiseAddr: "" #advertisePort: 3322 #suspicionMaxTimeoutMult: 6 #disableTCPPings: false #awarenessMaxMultiplier: 8 #gossipNodes: 3 #gossipVerifyIncoming: true #gossipVerifyOutgoing: true #dnsConfigPath: "/etc/resolv.conf" #handoffQueueDepth: 1024 #udpBufferSize: 1400 dmaps: engine: name: ramblock config: tableSize: 524288 # bytes # checkEmptyFragmentsInterval: 1m # triggerCompactionInterval: 10m # numEvictionWorkers: 1 # maxIdleDuration: "" # ttlDuration: "100s" # maxKeys: 100000 # maxInuse: 1000000 # lRUSamples: 10 # evictionPolicy: "LRU" # custom: # foobar: # maxIdleDuration: "60s" # ttlDuration: "300s" # maxKeys: 500000 # lRUSamples: 20 # evictionPolicy: "NONE" #serviceDiscovery: # # path is a required property and used by Olric. It has to be a full path. # path: "/home/burak/go/src/github.com/olric-data/olric-consul-plugin/consul.so" # # # provider is just informal, # provider: "consul" # # # Plugin specific configuration # # Consul server, used by the plugin. It's required # address: "http://127.0.0.1:8500" # # # Specifies that the server should return only nodes with all checks in the passing state. # passingOnly: true # # # Missing health checks from the request will be deleted from the agent. Using this parameter # # allows to idempotently register a service and its checks without having to manually deregister # # checks. # replaceExistingChecks: true # # # InsecureSkipVerify controls whether a client verifies the # # server's certificate chain and host name. # # If InsecureSkipVerify is true, TLS accepts any certificate # # presented by the server and any host name in that certificate. # # In this mode, TLS is susceptible to man-in-the-middle attacks. # # This should be used only for testing. # insecureSkipVerify: true # # # service record # payload: ' # { # "Name": "olric-cluster", # "ID": "olric-node-1", # "Tags": [ # "primary", # "v1" # ], # "Address": "localhost", # "Port": 3322, # "EnableTagOverride": false, # "check": { # "name": "Olric node on 3322", # "tcp": "0.0.0.0:3322", # "interval": "10s", # "timeout": "1s" # } # } #' # # #serviceDiscovery: # provider: "k8s" # path: "/Users/buraksezer/go/src/github.com/olric-data/olric-cloud-plugin/olric-cloud-plugin.so" # args: 'label_selector="app = olric-server"' ================================================ FILE: olric.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. /* Package olric provides a distributed cache and in-memory key/value data store. It can be used both as an embedded Go library and as a language-independent service. With Olric, you can instantly create a fast, scalable, shared pool of RAM across a cluster of computers. Olric is designed to be a distributed cache. But it also provides Publish/Subscribe, data replication, failure detection and simple anti-entropy services. So it can be used as an ordinary key/value data store to scale your cloud application. */ package olric import ( "context" "fmt" "net" "runtime" "strconv" "strings" "sync" "time" "github.com/hashicorp/logutils" "github.com/olric-data/olric/config" "github.com/olric-data/olric/hasher" "github.com/olric-data/olric/internal/checkpoint" "github.com/olric-data/olric/internal/cluster/balancer" "github.com/olric-data/olric/internal/cluster/partitions" "github.com/olric-data/olric/internal/cluster/routingtable" "github.com/olric-data/olric/internal/dmap" "github.com/olric-data/olric/internal/environment" "github.com/olric-data/olric/internal/locker" "github.com/olric-data/olric/internal/protocol" "github.com/olric-data/olric/internal/pubsub" "github.com/olric-data/olric/internal/server" "github.com/olric-data/olric/pkg/flog" "github.com/pkg/errors" "github.com/tidwall/redcon" "golang.org/x/sync/errgroup" ) // ReleaseVersion is the current stable version of Olric const ReleaseVersion string = "0.7.3" var ( // ErrOperationTimeout is returned when an operation times out. ErrOperationTimeout = errors.New("operation timeout") // ErrServerGone means that a cluster member is closed unexpectedly. ErrServerGone = errors.New("server is gone") // ErrKeyNotFound means that returned when a key could not be found. ErrKeyNotFound = errors.New("key not found") // ErrKeyFound means that the requested key found in the cluster. ErrKeyFound = errors.New("key found") // ErrWriteQuorum means that write quorum cannot be reached to operate. ErrWriteQuorum = errors.New("write quorum cannot be reached") // ErrReadQuorum means that read quorum cannot be reached to operate. ErrReadQuorum = errors.New("read quorum cannot be reached") // ErrLockNotAcquired is returned when the requested lock could not be acquired ErrLockNotAcquired = errors.New("lock not acquired") // ErrNoSuchLock is returned when the requested lock does not exist ErrNoSuchLock = errors.New("no such lock") // ErrClusterQuorum means that the cluster could not reach a healthy numbers of members to operate. ErrClusterQuorum = errors.New("failed to find enough peers to create quorum") // ErrKeyTooLarge means that the given key is too large to process. // The maximum length of a key is 256 bytes. ErrKeyTooLarge = errors.New("key too large") // ErrEntryTooLarge returned if the required space for an entry is bigger than table size. ErrEntryTooLarge = errors.New("entry too large for the configured table size") // ErrConnRefused returned if the target node refused a connection request. // It is good to call RefreshMetadata to update the underlying data structures. ErrConnRefused = errors.New("connection refused") // ErrWrongPass indicates that the provided password is incorrect during authentication. ErrWrongPass = errors.New("wrong password") ) // Olric implements a distributed cache and in-memory key/value data store. // It can be used both as an embedded Go library and as a language-independent // service. type Olric struct { // name is BindAddr:BindPort. It defines servers unique name in the cluster. name string env *environment.Environment config *config.Config log *flog.Logger hashFunc hasher.Hasher // Logical units to store data primary *partitions.Partitions backup *partitions.Partitions // RESP server and clients. server *server.Server client *server.Client rt *routingtable.RoutingTable balancer *balancer.Balancer pubsub *pubsub.Service dmap *dmap.Service // Structures for flow control ctx context.Context cancel context.CancelFunc wg sync.WaitGroup // Callback function. Olric calls this after // the server is ready to accept new connections. started func() } func prepareConfig(c *config.Config) (*config.Config, error) { if c == nil { return nil, fmt.Errorf("config cannot be nil") } err := c.Sanitize() if err != nil { return nil, err } err = c.Validate() if err != nil { return nil, err } err = c.SetupNetworkConfig() if err != nil { return nil, err } c.MemberlistConfig.Name = net.JoinHostPort(c.BindAddr, strconv.Itoa(c.BindPort)) filter := &logutils.LevelFilter{ Levels: []logutils.LogLevel{"DEBUG", "WARN", "ERROR", "INFO"}, MinLevel: logutils.LogLevel(strings.ToUpper(c.LogLevel)), Writer: c.Logger.Writer(), } c.Logger.SetOutput(filter) return c, nil } func initializeServices(db *Olric) error { db.rt = routingtable.New(db.env) db.env.Set("routingtable", db.rt) db.balancer = balancer.New(db.env) // Add Services dt, err := pubsub.NewService(db.env) if err != nil { return err } db.pubsub = dt.(*pubsub.Service) dm, err := dmap.NewService(db.env) if err != nil { return err } db.dmap = dm.(*dmap.Service) return nil } // New creates a new Olric instance, otherwise returns an error. func New(c *config.Config) (*Olric, error) { var err error c, err = prepareConfig(c) if err != nil { return nil, err } e := environment.New() e.Set("config", c) // Set the hash function. Olric distributes keys over partitions by hashing. partitions.SetHashFunc(c.Hasher) flogger := flog.New(c.Logger) flogger.SetLevel(c.LogVerbosity) if c.LogLevel == "DEBUG" { flogger.ShowLineNumber(1) } e.Set("logger", flogger) if c.Authentication.Enabled() { c.Client.Authentication = c.Authentication } client := server.NewClient(c.Client) e.Set("client", client) e.Set("primary", partitions.New(c.PartitionCount, partitions.PRIMARY)) e.Set("backup", partitions.New(c.PartitionCount, partitions.BACKUP)) e.Set("locker", locker.New()) ctx, cancel := context.WithCancel(context.Background()) db := &Olric{ name: c.MemberlistConfig.Name, env: e, log: flogger, config: c, hashFunc: c.Hasher, client: client, primary: e.Get("primary").(*partitions.Partitions), backup: e.Get("backup").(*partitions.Partitions), started: c.Started, ctx: ctx, cancel: cancel, } // Create a Redcon server instance rc := &server.Config{ BindAddr: c.BindAddr, BindPort: c.BindPort, KeepAlivePeriod: c.KeepAlivePeriod, RequireAuth: c.Authentication.Enabled(), } srv := server.New(rc, flogger) srv.SetPreConditionFunc(db.preconditionFunc) db.server = srv e.Set("server", srv) err = initializeServices(db) if err != nil { return nil, err } db.registerCommandHandlers() registerErrors() return db, nil } func (db *Olric) preconditionFunc(conn redcon.Conn, _ redcon.Command) bool { err := db.isOperable() if err != nil { protocol.WriteError(conn, err) return false } return true } func (db *Olric) registerCommandHandlers() { db.server.ServeMux().HandleFunc(protocol.Generic.Ping, db.pingCommandHandler) db.server.ServeMux().HandleFunc(protocol.Cluster.RoutingTable, db.clusterRoutingTableCommandHandler) db.server.ServeMux().HandleFunc(protocol.Generic.Stats, db.statsCommandHandler) db.server.ServeMux().HandleFunc(protocol.Cluster.Members, db.clusterMembersCommandHandler) db.server.ServeMux().HandleFunc(protocol.Generic.Auth, db.authCommandHandler) } // callStartedCallback checks passed checkpoint count and calls the callback // function. func (db *Olric) callStartedCallback() { defer db.wg.Done() timer := time.NewTimer(10 * time.Millisecond) defer timer.Stop() for { timer.Reset(10 * time.Millisecond) select { case <-timer.C: if checkpoint.AllPassed() { if db.started != nil { db.started() } return } case <-db.ctx.Done(): return } } } func convertClusterError(err error) error { switch { case errors.Is(err, routingtable.ErrClusterQuorum): return ErrClusterQuorum case errors.Is(err, routingtable.ErrServerGone): return ErrServerGone case errors.Is(err, routingtable.ErrOperationTimeout): return ErrOperationTimeout default: return err } } // isOperable controls bootstrapping status and cluster quorum to prevent split-brain syndrome. func (db *Olric) isOperable() error { if err := db.rt.CheckMemberCountQuorum(); err != nil { return convertClusterError(err) } // An Olric node has to be bootstrapped to function properly. return db.rt.CheckBootstrap() } // Start starts background servers and joins the cluster. You still must call Shutdown // method if Start function returns an early error. func (db *Olric) Start() error { db.log.V(1).Printf("[INFO] Olric %s on %s/%s %s", ReleaseVersion, runtime.GOOS, runtime.GOARCH, runtime.Version()) // This error group is responsible to run the TCP server at background and report errors. errGr, ctx := errgroup.WithContext(context.Background()) errGr.Go(func() error { return db.server.ListenAndServe() }) select { case <-db.server.StartedCtx.Done(): // TCP server has been started case <-ctx.Done(): // TCP server could not be started due to an error. There is no need to run // Olric.Shutdown here because we could not start anything. return errGr.Wait() } // Balancer works periodically to balance partition data across the cluster. if err := db.balancer.Start(); err != nil { if err != nil { db.log.V(2).Printf("[ERROR] Failed to run the balancer subsystem: %v", err) } return err } // First, we need to join the cluster. Then, the routing table has been started. if err := db.rt.Join(); err != nil { if err != nil { db.log.V(2).Printf("[ERROR] Failed to join the Olric cluster: %v", err) } return err } // Start routing table service and member discovery subsystem. if err := db.rt.Start(); err != nil { if err != nil { db.log.V(2).Printf("[ERROR] Failed to run the routing table subsystem: %v", err) } return err } // Start publish-subscribe service if err := db.pubsub.Start(); err != nil { if err != nil { db.log.V(2).Printf("[ERROR] Failed to run the Publish-Subscribe service: %v", err) } return err } // Start distributed map service if err := db.dmap.Start(); err != nil { if err != nil { db.log.V(2).Printf("[ERROR] Failed to run the Distributed Map service: %v", err) } return err } // Warn the user about his/her choice of configuration if db.config.ReplicationMode == config.AsyncReplicationMode && db.config.WriteQuorum > 1 { db.log.V(2). Printf("[WARN] Olric is running in async replication mode. WriteQuorum (%d) is ineffective", db.config.WriteQuorum) } if db.started != nil { db.wg.Add(1) go db.callStartedCallback() } db.log.V(2).Printf("[INFO] Node name in the cluster: %s", db.name) if db.config.Interface != "" { db.log.V(2).Printf("[INFO] Olric uses interface: %s", db.config.Interface) } db.log.V(2).Printf("[INFO] Olric bindAddr: %s, bindPort: %d", db.config.BindAddr, db.config.BindPort) db.log.V(2).Printf("[INFO] Replication count is %d", db.config.ReplicaCount) // Wait for the TCP server. return errGr.Wait() } // Shutdown stops background servers and leaves the cluster. func (db *Olric) Shutdown(ctx context.Context) error { select { case <-db.ctx.Done(): // Shutdown only once. return nil default: } db.cancel() var latestError error if err := db.pubsub.Shutdown(ctx); err != nil { db.log.V(2).Printf("[ERROR] Failed to shutdown PubSub service: %v", err) latestError = err } if err := db.dmap.Shutdown(ctx); err != nil { db.log.V(2).Printf("[ERROR] Failed to shutdown DMap service: %v", err) latestError = err } if err := db.balancer.Shutdown(ctx); err != nil { db.log.V(2).Printf("[ERROR] Failed to shutdown balancer service: %v", err) latestError = err } if err := db.rt.Shutdown(ctx); err != nil { db.log.V(2).Printf("[ERROR] Failed to shutdown routing table service: %v", err) latestError = err } // Shutdown Redcon server if err := db.server.Shutdown(ctx); err != nil { db.log.V(2).Printf("[ERROR] Failed to shutdown RESP server: %v", err) latestError = err } done := make(chan struct{}) go func() { defer func() { close(done) }() db.wg.Wait() }() select { case <-ctx.Done(): case <-done: } // db.name will be shown as empty string, if the program is killed before // bootstrapping. db.log.V(2).Printf("[INFO] %s is gone", db.name) return latestError } func convertDMapError(err error) error { switch { case errors.Is(err, dmap.ErrKeyFound): return ErrKeyFound case errors.Is(err, dmap.ErrKeyNotFound): return ErrKeyNotFound case errors.Is(err, dmap.ErrDMapNotFound): return ErrKeyNotFound case errors.Is(err, dmap.ErrLockNotAcquired): return ErrLockNotAcquired case errors.Is(err, dmap.ErrNoSuchLock): return ErrNoSuchLock case errors.Is(err, dmap.ErrReadQuorum): return ErrReadQuorum case errors.Is(err, dmap.ErrWriteQuorum): return ErrWriteQuorum case errors.Is(err, dmap.ErrServerGone): return ErrServerGone case errors.Is(err, dmap.ErrKeyTooLarge): return ErrKeyTooLarge case errors.Is(err, dmap.ErrEntryTooLarge): return ErrEntryTooLarge default: return convertClusterError(err) } } // registerErrors registers application-specific errors with their corresponding prefixes in the error management system. func registerErrors() { protocol.SetError("WRONGPASS", ErrWrongPass) } ================================================ FILE: olric_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package olric import ( "context" "fmt" "sync" "testing" "time" "github.com/hashicorp/memberlist" "github.com/olric-data/olric/config" "github.com/olric-data/olric/internal/testutil" "github.com/olric-data/olric/stats" "github.com/stretchr/testify/require" ) // newTestOlricWithConfig creates a new Olric instance with the given configuration. // This function is intended for internal use. Please use testOlricCluster and its // methods to form a cluster in tests. func newTestOlricWithConfig(t *testing.T, c *config.Config) *Olric { port, err := testutil.GetFreePort() require.NoError(t, err) if c.MemberlistConfig == nil { c.MemberlistConfig = memberlist.DefaultLocalConfig() } c.MemberlistConfig.BindPort = 0 c.BindAddr = "127.0.0.1" c.BindPort = port err = c.Sanitize() require.NoError(t, err) err = c.Validate() require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) c.Started = func() { cancel() } db, err := New(c) require.NoError(t, err) go func() { if err := db.Start(); err != nil { panic(fmt.Sprintf("Failed to run Olric: %v", err)) } }() select { case <-time.After(time.Second): t.Fatalf("Olric cannot be started in one second") case <-ctx.Done(): // everything is fine } return db } type testOlricCluster struct { mtx sync.Mutex members map[string]*Olric } func newTestOlricCluster(t *testing.T) *testOlricCluster { cl := &testOlricCluster{members: make(map[string]*Olric)} t.Cleanup(func() { cl.mtx.Lock() defer cl.mtx.Unlock() for _, member := range cl.members { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) err := member.Shutdown(ctx) cancel() require.NoError(t, err) } }) return cl } func (cl *testOlricCluster) addMemberWithConfig(t *testing.T, c *config.Config) *Olric { cl.mtx.Lock() defer cl.mtx.Unlock() if c == nil { c = testutil.NewConfig() } for _, member := range cl.members { c.Peers = append(c.Peers, member.rt.Discovery().LocalNode().Address()) } db := newTestOlricWithConfig(t, c) cl.members[db.rt.This().String()] = db t.Logf("A new cluster member has been created: %s", db.rt.This()) return db } func (cl *testOlricCluster) addMember(t *testing.T) *Olric { return cl.addMemberWithConfig(t, nil) } func TestOlric_StartAndShutdown(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) err := db.Shutdown(context.Background()) require.NoError(t, err) } func TestOlricCluster_StartAndShutdown(t *testing.T) { cluster := newTestOlricCluster(t) cluster.addMember(t) db := cluster.addMember(t) require.Len(t, cluster.members, 2) e := db.NewEmbeddedClient() st, err := e.Stats(context.Background(), db.rt.This().String()) require.NoError(t, err) require.Len(t, st.ClusterMembers, 2) for _, member := range cluster.members { require.Contains(t, st.ClusterMembers, stats.MemberID(member.rt.This().ID)) } } ================================================ FILE: ping.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package olric import ( "context" "strings" "github.com/olric-data/olric/internal/protocol" "github.com/tidwall/redcon" ) const DefaultPingResponse = "PONG" func (db *Olric) ping(ctx context.Context, addr, message string) ([]byte, error) { message = strings.TrimSpace(message) pingCmd := protocol.NewPing() if message != "" { pingCmd = pingCmd.SetMessage(message) } cmd := pingCmd.Command(ctx) rc := db.client.Get(addr) err := rc.Process(ctx, cmd) if err != nil { return nil, err } return cmd.Bytes() } func (db *Olric) pingCommandHandler(conn redcon.Conn, cmd redcon.Command) { pingCmd, err := protocol.ParsePingCommand(cmd) if err != nil { protocol.WriteError(conn, err) return } if pingCmd.Message != "" { conn.WriteString(pingCmd.Message) return } conn.WriteString(DefaultPingResponse) } ================================================ FILE: ping_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package olric import ( "context" "testing" "github.com/stretchr/testify/require" ) func TestOlric_Ping(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) result, err := db.ping(context.Background(), db.rt.This().String(), "") require.NoError(t, err) require.Equal(t, []byte(DefaultPingResponse), result) } func TestOlric_PingWithMessage(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) msg := "Olric rocks!" response, err := db.ping(context.Background(), db.rt.This().String(), msg) require.NoError(t, err) require.Equal(t, []byte(msg), response) } ================================================ FILE: pipeline.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package olric import ( "bytes" "context" "errors" "runtime" "strconv" "sync" "time" "github.com/olric-data/olric/internal/cluster/partitions" "github.com/olric-data/olric/internal/dmap" "github.com/olric-data/olric/internal/protocol" "github.com/olric-data/olric/internal/resp" "github.com/redis/go-redis/v9" "golang.org/x/sync/errgroup" "golang.org/x/sync/semaphore" ) var ( // ErrNotReady denotes that the Future instance you hold is not ready to read the response yet. ErrNotReady = errors.New("not ready yet") // ErrPipelineClosed denotes that the underlying pipeline is closed, and it's impossible to operate. ErrPipelineClosed = errors.New("pipeline is closed") // ErrPipelineExecuted denotes that Exec was already called on the underlying pipeline. ErrPipelineExecuted = errors.New("pipeline already executed") ) // DMapPipeline implements a pipeline for the following methods of the DMap API: // // * Put // * Get // * Delete // * Incr // * Decr // * GetPut // * IncrByFloat // // DMapPipeline enables batch operations on DMap data. type DMapPipeline struct { mtx sync.Mutex dm *ClusterDMap commands map[uint64][]redis.Cmder result map[uint64][]redis.Cmder ctx context.Context cancel context.CancelFunc closedCtx context.Context // used to detect if the pipeline is closed / discarded closedCancel context.CancelFunc concurrency int // defaults to runtime.NumCPU() } func (dp *DMapPipeline) addCommand(key string, cmd redis.Cmder) (uint64, int) { dp.mtx.Lock() defer dp.mtx.Unlock() hkey := partitions.HKey(dp.dm.name, key) partID := hkey % dp.dm.clusterClient.partitionCount cmds, ok := dp.commands[partID] if !ok { // if there are no existing commands, get a new slice from the pool cmds = getPipelineCmdsFromPool() } dp.commands[partID] = append(cmds, cmd) return partID, len(dp.commands[partID]) - 1 } // FuturePut is used to read the result of a pipelined Put command. type FuturePut struct { dp *DMapPipeline partID uint64 index int ctx context.Context closedCtx context.Context } // Result returns a response for the pipelined Put command. func (f *FuturePut) Result() error { // this select is separate from the one below on purpose, since select is non-deterministic if multiple // cases are available, and we need to guarantee this check first. select { case <-f.closedCtx.Done(): return ErrPipelineClosed default: } select { case <-f.ctx.Done(): cmd := f.dp.result[f.partID][f.index] return processProtocolError(cmd.Err()) default: return ErrNotReady } } // Put queues a Put command. The parameters are identical to the DMap.Put, // but it returns FuturePut to read the batched response. func (dp *DMapPipeline) Put(ctx context.Context, key string, value interface{}, options ...PutOption) (*FuturePut, error) { buf := bytes.NewBuffer(nil) enc := resp.New(buf) err := enc.Encode(value) if err != nil { return nil, err } var pc dmap.PutConfig for _, opt := range options { opt(&pc) } cmd := dp.dm.writePutCommand(&pc, key, buf.Bytes()).Command(ctx) partID, index := dp.addCommand(key, cmd) return &FuturePut{ dp: dp, partID: partID, index: index, ctx: dp.ctx, closedCtx: dp.closedCtx, }, nil } // FutureGet is used to read result of a pipelined Get command. type FutureGet struct { dp *DMapPipeline partID uint64 index int ctx context.Context closedCtx context.Context } // Result returns a response for the pipelined Get command. func (f *FutureGet) Result() (*GetResponse, error) { // this select is separate from the one below on purpose, since select is non-deterministic if multiple // cases are available, and we need to guarantee this check first. select { case <-f.closedCtx.Done(): return nil, ErrPipelineClosed default: } select { case <-f.ctx.Done(): cmd := f.dp.result[f.partID][f.index] if cmd.Err() != nil { return nil, processProtocolError(cmd.Err()) } stringCmd := redis.NewStringCmd(context.Background(), cmd.Args()...) stringCmd.SetVal(cmd.(*redis.Cmd).Val().(string)) return f.dp.dm.makeGetResponse(stringCmd) default: return nil, ErrNotReady } } // Get queues a Get command. The parameters are identical to the DMap.Get, // but it returns FutureGet to read the batched response. func (dp *DMapPipeline) Get(ctx context.Context, key string) *FutureGet { cmd := protocol.NewGet(dp.dm.name, key).SetRaw().Command(ctx) partID, index := dp.addCommand(key, cmd) return &FutureGet{ dp: dp, partID: partID, index: index, ctx: dp.ctx, closedCtx: dp.closedCtx, } } // FutureDelete is used to read the result of a pipelined Delete command. type FutureDelete struct { dp *DMapPipeline partID uint64 index int ctx context.Context closedCtx context.Context } // Result returns a response for the pipelined Delete command. func (f *FutureDelete) Result() (int, error) { // this select is separate from the one below on purpose, since select is non-deterministic if multiple // cases are available, and we need to guarantee this check first. select { case <-f.closedCtx.Done(): return 0, ErrPipelineClosed default: } select { case <-f.ctx.Done(): cmd := f.dp.result[f.partID][f.index] if cmd.Err() != nil { return 0, processProtocolError(cmd.Err()) } return int(cmd.(*redis.Cmd).Val().(int64)), nil default: return 0, ErrNotReady } } // Delete queues a Delete command. The parameters are identical to the DMap.Delete, // but it returns FutureDelete to read the batched response. func (dp *DMapPipeline) Delete(ctx context.Context, key string) *FutureDelete { cmd := protocol.NewDel(dp.dm.name, []string{key}...).Command(ctx) partID, index := dp.addCommand(key, cmd) return &FutureDelete{ dp: dp, partID: partID, index: index, ctx: dp.ctx, closedCtx: dp.closedCtx, } } // FutureExpire is used to read the result of a pipelined Expire command. type FutureExpire struct { dp *DMapPipeline partID uint64 index int ctx context.Context closedCtx context.Context } // Result returns a response for the pipelined Expire command. func (f *FutureExpire) Result() error { // this select is separate from the one below on purpose, since select is non-deterministic if multiple // cases are available, and we need to guarantee this check first. select { case <-f.closedCtx.Done(): return ErrPipelineClosed default: } select { case <-f.ctx.Done(): cmd := f.dp.result[f.partID][f.index] return processProtocolError(cmd.Err()) default: return ErrNotReady } } // Expire queues an Expire command. The parameters are identical to the DMap.Expire, // but it returns FutureExpire to read the batched response. func (dp *DMapPipeline) Expire(ctx context.Context, key string, timeout time.Duration) (*FutureExpire, error) { cmd := protocol.NewExpire(dp.dm.name, key, timeout).Command(ctx) partID, index := dp.addCommand(key, cmd) return &FutureExpire{ dp: dp, partID: partID, index: index, ctx: dp.ctx, closedCtx: dp.closedCtx, }, nil } // FutureIncr is used to read the result of a pipelined Incr command. type FutureIncr struct { dp *DMapPipeline partID uint64 index int ctx context.Context closedCtx context.Context } // Result returns a response for the pipelined Incr command. func (f *FutureIncr) Result() (int, error) { // this select is separate from the one below on purpose, since select is non-deterministic if multiple // cases are available, and we need to guarantee this check first. select { case <-f.closedCtx.Done(): return 0, ErrPipelineClosed default: } select { case <-f.ctx.Done(): cmd := f.dp.result[f.partID][f.index] if cmd.Err() != nil { return 0, processProtocolError(cmd.Err()) } return int(cmd.(*redis.Cmd).Val().(int64)), nil default: return 0, ErrNotReady } } // Incr queues an Incr command. The parameters are identical to the DMap.Incr, // but it returns FutureIncr to read the batched response. func (dp *DMapPipeline) Incr(ctx context.Context, key string, delta int) (*FutureIncr, error) { cmd := protocol.NewIncr(dp.dm.name, key, delta).Command(ctx) partID, index := dp.addCommand(key, cmd) return &FutureIncr{ dp: dp, partID: partID, index: index, ctx: dp.ctx, closedCtx: dp.closedCtx, }, nil } // FutureDecr is used to read the result of a pipelined Decr command. type FutureDecr struct { dp *DMapPipeline partID uint64 index int ctx context.Context closedCtx context.Context } // Result returns a response for the pipelined Decr command. func (f *FutureDecr) Result() (int, error) { // this select is separate from the one below on purpose, since select is non-deterministic if multiple // cases are available, and we need to guarantee this check first. select { case <-f.closedCtx.Done(): return 0, ErrPipelineClosed default: } select { case <-f.ctx.Done(): cmd := f.dp.result[f.partID][f.index] if cmd.Err() != nil { return 0, processProtocolError(cmd.Err()) } return int(cmd.(*redis.Cmd).Val().(int64)), nil default: return 0, ErrNotReady } } // Decr queues a Decr command. The parameters are identical to the DMap.Decr, // but it returns FutureDecr to read the batched response. func (dp *DMapPipeline) Decr(ctx context.Context, key string, delta int) (*FutureDecr, error) { cmd := protocol.NewDecr(dp.dm.name, key, delta).Command(ctx) partID, index := dp.addCommand(key, cmd) return &FutureDecr{ dp: dp, partID: partID, index: index, ctx: dp.ctx, closedCtx: dp.closedCtx, }, nil } // FutureGetPut is used to read the result of a pipelined GetPut command. type FutureGetPut struct { dp *DMapPipeline partID uint64 index int ctx context.Context closedCtx context.Context } // Result returns a response for the pipelined GetPut command. func (f *FutureGetPut) Result() (*GetResponse, error) { // this select is separate from the one below on purpose, since select is non-deterministic if multiple // cases are available, and we need to guarantee this check first. select { case <-f.closedCtx.Done(): return nil, ErrPipelineClosed default: } select { case <-f.ctx.Done(): cmd := f.dp.result[f.partID][f.index] if cmd.Err() == redis.Nil { // This should be the first run. return nil, nil } if cmd.Err() != nil { return nil, processProtocolError(cmd.Err()) } stringCmd := redis.NewStringCmd(context.Background(), cmd.Args()...) stringCmd.SetVal(cmd.(*redis.Cmd).Val().(string)) return f.dp.dm.makeGetResponse(stringCmd) default: return nil, ErrNotReady } } // GetPut queues a GetPut command. The parameters are identical to the DMap.GetPut, // but it returns FutureGetPut to read the batched response. func (dp *DMapPipeline) GetPut(ctx context.Context, key string, value interface{}) (*FutureGetPut, error) { buf := bytes.NewBuffer(nil) enc := resp.New(buf) err := enc.Encode(value) if err != nil { return nil, err } cmd := protocol.NewGetPut(dp.dm.name, key, buf.Bytes()).SetRaw().Command(ctx) partID, index := dp.addCommand(key, cmd) return &FutureGetPut{ dp: dp, partID: partID, index: index, ctx: dp.ctx, closedCtx: dp.closedCtx, }, nil } // FutureIncrByFloat is used to read the result of a pipelined IncrByFloat command. type FutureIncrByFloat struct { dp *DMapPipeline partID uint64 index int ctx context.Context closedCtx context.Context } // Result returns a response for the pipelined IncrByFloat command. func (f *FutureIncrByFloat) Result() (float64, error) { // this select is separate from the one below on purpose, since select is non-deterministic if multiple // cases are available, and we need to guarantee this check first. select { case <-f.closedCtx.Done(): return 0, ErrPipelineClosed default: } select { case <-f.ctx.Done(): cmd := f.dp.result[f.partID][f.index] if cmd.Err() != nil { return 0, processProtocolError(cmd.Err()) } stringRes := cmd.(*redis.Cmd).Val().(string) return strconv.ParseFloat(stringRes, 64) default: return 0, ErrNotReady } } // IncrByFloat queues an IncrByFloat command. The parameters are identical to the DMap.IncrByFloat, // but it returns FutureIncrByFloat to read the batched response. func (dp *DMapPipeline) IncrByFloat(ctx context.Context, key string, delta float64) (*FutureIncrByFloat, error) { cmd := protocol.NewIncrByFloat(dp.dm.name, key, delta).Command(ctx) partID, index := dp.addCommand(key, cmd) return &FutureIncrByFloat{ dp: dp, partID: partID, index: index, ctx: dp.ctx, closedCtx: dp.closedCtx, }, nil } func (dp *DMapPipeline) execOnPartition(ctx context.Context, partID uint64) error { rc, err := dp.dm.clusterClient.clientByPartID(partID) if err != nil { return err } // There is no need to protect dp.commands map and its content. // It's already filled before running Exec, and it's now a read-only // data structure commands := dp.commands[partID] pipe := rc.Pipeline() for _, cmd := range commands { pipe.Do(ctx, cmd.Args()...) } // Exec executes all previously queued commands using one // client-server roundtrip. // // Exec always returns list of commands and error of the first failed // command if any. result, _ := pipe.Exec(ctx) dp.mtx.Lock() dp.result[partID] = result dp.mtx.Unlock() return nil } // Exec executes all queued commands using one client-server roundtrip per partition. func (dp *DMapPipeline) Exec(ctx context.Context) error { // this select is separate from the one below on purpose, since select is non-deterministic if multiple // cases are available, and we need to guarantee this check first. select { case <-dp.closedCtx.Done(): return ErrPipelineClosed default: } // this checks to see if Exec has already run. While Exec should only be called once, it is possible that // the user could call Exec multiple times. If we stored the result of errGr.Wait on the pipeline, we could // return that error and make Exec idempotent. select { case <-dp.ctx.Done(): return ErrPipelineExecuted default: } defer dp.cancel() var errGr errgroup.Group sem := semaphore.NewWeighted(int64(dp.concurrency)) for i := uint64(0); i < dp.dm.clusterClient.partitionCount; i++ { err := sem.Acquire(ctx, 1) if err != nil { return err } partID := i errGr.Go(func() error { defer sem.Release(1) // If execOnPartition returns an error, it will eventually stop // all flush operation. return dp.execOnPartition(ctx, partID) }) } return errGr.Wait() } // Discard discards the pipelined commands and resets all internal states. // A pipeline can be reused after calling Discard. func (dp *DMapPipeline) Discard() error { select { case <-dp.closedCtx.Done(): return ErrPipelineClosed default: } dp.closedCancel() dp.mtx.Lock() defer dp.mtx.Unlock() // return all command slices to the pool for _, v := range dp.commands { putPipelineCmdsIntoPool(v) } for _, v := range dp.result { putPipelineCmdsIntoPool(v) } // the deletes below are purposefully not combined with the loops above, as these are recognized and optimized // by the compiler. https://go-review.googlesource.com/c/go/+/110055 for k := range dp.commands { delete(dp.commands, k) } for k := range dp.result { delete(dp.result, k) } dp.initContexts() return nil } // Close closes the pipeline and frees the allocated resources. You shouldn't try to // reuse a closed pipeline. func (dp *DMapPipeline) Close() { dp.closedCancel() } // Pipeline is a mechanism to realise Redis Pipeline technique. // // Pipelining is a technique to extremely speed up processing by packing // operations to batches, send them at once to Redis and read a replies in a // singe step. // See https://redis.io/topics/pipelining // // Pay attention, that Pipeline is not a transaction, so you can get unexpected // results in case of big pipelines and small read/write timeouts. // Redis client has retransmission logic in case of timeouts, pipeline // can be retransmitted and commands can be executed more than once. func (dm *ClusterDMap) Pipeline(opts ...PipelineOption) (*DMapPipeline, error) { dp := &DMapPipeline{ dm: dm, commands: make(map[uint64][]redis.Cmder), result: make(map[uint64][]redis.Cmder), concurrency: runtime.NumCPU(), } for _, opt := range opts { opt(dp) } dp.initContexts() return dp, nil } // initContexts sets up chained contexts for the pipeline. The base is closedCtx, which is closed either in // Close or Discard. ctx is a child of closedCtx, as we want to cancel the pipeline if it is closed. It is // canceled in Exec, and used to block FutureXXX.Result() calls until Exec has completed. func (dp *DMapPipeline) initContexts() { dp.closedCtx, dp.closedCancel = context.WithCancel(context.Background()) dp.ctx, dp.cancel = context.WithCancel(dp.closedCtx) } // This stores a slice of commands for each partition. There is a possibility that a single // large slice could be allocated with an unusually large number of commands in a single pipeline that // are very unbalanced across partitions, but that is unlikely to be a problem in practice. // // It does not store a pointer to the slice as recommended by staticcheck because that is harder to reason // about, and a single allocation is not a big deal compared to the slices we're able to reuse. // https://staticcheck.io/docs/checks#SA6002 // https://github.com/dominikh/go-tools/issues/1336#issuecomment-1331206290 var pipelineCmdPool = sync.Pool{ New: func() interface{} { return make([]redis.Cmder, 0) }, } func getPipelineCmdsFromPool() []redis.Cmder { return pipelineCmdPool.Get().([]redis.Cmder) } func putPipelineCmdsIntoPool(cmds []redis.Cmder) { // remove references to underlying commands so they can be GCed for i := range cmds { cmds[i] = nil } cmds = cmds[:0] pipelineCmdPool.Put(cmds) } ================================================ FILE: pipeline_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package olric import ( "context" "fmt" "testing" "time" "github.com/olric-data/olric/internal/testutil" "github.com/stretchr/testify/require" ) func TestDMapPipeline_Put(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) futures := make(map[int]*FuturePut) pipe, err := dm.Pipeline() require.NoError(t, err) defer pipe.Close() for i := 0; i < 100; i++ { fp, err := pipe.Put(ctx, testutil.ToKey(i), testutil.ToVal(i)) require.NoError(t, err) futures[i] = fp } err = pipe.Exec(ctx) require.NoError(t, err) for _, fp := range futures { require.NoError(t, fp.Result()) } for i := 0; i < 100; i++ { key := testutil.ToKey(i) gr, err := dm.Get(ctx, key) require.NoError(t, err) value, err := gr.Byte() require.NoError(t, err) require.Equal(t, testutil.ToVal(i), value) } } func TestDMapPipeline_Get(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) for i := 0; i < 100; i++ { err = dm.Put(ctx, testutil.ToKey(i), testutil.ToVal(i)) require.NoError(t, err) } pipe, err := dm.Pipeline() require.NoError(t, err) defer pipe.Close() futures := make(map[int]*FutureGet) for i := 0; i < 100; i++ { fg := pipe.Get(ctx, testutil.ToKey(i)) futures[i] = fg } err = pipe.Exec(ctx) require.NoError(t, err) for i, fg := range futures { gr, err := fg.Result() require.NoError(t, err) value, err := gr.Byte() require.NoError(t, err) require.Equal(t, testutil.ToVal(i), value) } } func TestDMapPipeline_Delete(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) for i := 0; i < 100; i++ { err = dm.Put(ctx, testutil.ToKey(i), testutil.ToVal(i)) require.NoError(t, err) } pipe, err := dm.Pipeline() require.NoError(t, err) defer pipe.Close() futures := make(map[int]*FutureDelete) for i := 0; i < 100; i++ { fd := pipe.Delete(ctx, testutil.ToKey(i)) futures[i] = fd } err = pipe.Exec(ctx) require.NoError(t, err) for _, fd := range futures { num, err := fd.Result() require.NoError(t, err) require.Equal(t, 1, num) } } func TestDMapPipeline_Expire(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) for i := 0; i < 100; i++ { err = dm.Put(ctx, testutil.ToKey(i), testutil.ToVal(i)) require.NoError(t, err) } pipe, err := dm.Pipeline() require.NoError(t, err) defer pipe.Close() futures := make(map[int]*FutureExpire) for i := 0; i < 100; i++ { fd, err := pipe.Expire(ctx, testutil.ToKey(i), time.Hour) require.NoError(t, err) futures[i] = fd } err = pipe.Exec(ctx) require.NoError(t, err) for _, fd := range futures { err := fd.Result() require.NoError(t, err) } for i := 0; i < 100; i++ { gr, err := dm.Get(ctx, testutil.ToKey(i)) require.NoError(t, err) require.NotEqual(t, int64(0), gr.TTL()) } } func TestDMapPipeline_Incr(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) futures := make(map[int]*FutureIncr) pipe, err := dm.Pipeline() require.NoError(t, err) defer pipe.Close() for i := 0; i < 100; i++ { fi, err := pipe.Incr(ctx, "mykey", 1) require.NoError(t, err) futures[i] = fi } err = pipe.Exec(ctx) require.NoError(t, err) for i, fp := range futures { num, err := fp.Result() require.NoError(t, err) require.Equal(t, i+1, num) } } func TestDMapPipeline_Decr(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) futures := make(map[int]*FutureDecr) pipe, err := dm.Pipeline() require.NoError(t, err) defer pipe.Close() for i := 0; i < 100; i++ { fi, err := pipe.Decr(ctx, "mykey", 1) require.NoError(t, err) futures[i] = fi } err = pipe.Exec(ctx) require.NoError(t, err) for i, fp := range futures { num, err := fp.Result() require.NoError(t, err) require.Equal(t, -1*(i+1), num) } } func TestDMapPipeline_GetPut(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) futures := make(map[int]*FutureGetPut) pipe, err := dm.Pipeline() require.NoError(t, err) defer pipe.Close() for i := 0; i < 100; i++ { fi, err := pipe.GetPut(ctx, "key", testutil.ToVal(i)) require.NoError(t, err) futures[i] = fi } err = pipe.Exec(ctx) require.NoError(t, err) for _, fp := range futures { gr, err := fp.Result() require.NoError(t, err) if gr != nil { fmt.Println(gr.String()) } } } func TestDMapPipeline_IncrByFloat(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) futures := make(map[int]*FutureIncrByFloat) pipe, err := dm.Pipeline() require.NoError(t, err) defer pipe.Close() for i := 0; i < 100; i++ { fi, err := pipe.IncrByFloat(ctx, "mykey", 1.2) require.NoError(t, err) futures[i] = fi } err = pipe.Exec(ctx) require.NoError(t, err) for _, fp := range futures { _, err := fp.Result() require.NoError(t, err) } } func TestDMapPipeline_Discard(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) futures := make(map[int]*FuturePut) pipe, err := dm.Pipeline() require.NoError(t, err) for i := 0; i < 100; i++ { fp, err := pipe.Put(ctx, testutil.ToKey(i), testutil.ToVal(i)) require.NoError(t, err) futures[i] = fp } // Discard all pipelined DM.PUT requests. err = pipe.Discard() require.NoError(t, err) err = pipe.Exec(ctx) require.NoError(t, err) for i := 0; i < 100; i++ { key := testutil.ToKey(i) _, err := dm.Get(ctx, key) require.ErrorIs(t, err, ErrKeyNotFound) } } func TestDMapPipeline_Close(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) futures := make(map[int]*FuturePut) pipe, err := dm.Pipeline() require.NoError(t, err) for i := 0; i < 100; i++ { fp, err := pipe.Put(ctx, testutil.ToKey(i), testutil.ToVal(i)) require.NoError(t, err) futures[i] = fp } pipe.Close() err = pipe.Exec(ctx) require.ErrorIs(t, err, ErrPipelineClosed) } func TestDMapPipeline_ErrNotReady(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) pipe, err := dm.Pipeline() require.NoError(t, err) defer pipe.Close() t.Run("Put", func(t *testing.T) { fp, err := pipe.Put(ctx, "key", "value") require.NoError(t, err) require.ErrorIs(t, ErrNotReady, fp.Result()) }) t.Run("Get", func(t *testing.T) { fp := pipe.Get(ctx, "key") _, err := fp.Result() require.ErrorIs(t, ErrNotReady, err) }) t.Run("Delete", func(t *testing.T) { fp := pipe.Delete(ctx, "key") _, err := fp.Result() require.ErrorIs(t, ErrNotReady, err) }) t.Run("Expire", func(t *testing.T) { fp, err := pipe.Expire(ctx, "key", time.Second) require.NoError(t, err) err = fp.Result() require.ErrorIs(t, ErrNotReady, err) }) t.Run("Incr", func(t *testing.T) { fp, err := pipe.Incr(ctx, "key", 1) require.NoError(t, err) _, err = fp.Result() require.ErrorIs(t, ErrNotReady, err) }) t.Run("Decr", func(t *testing.T) { fp, err := pipe.Decr(ctx, "key", 1) require.NoError(t, err) _, err = fp.Result() require.ErrorIs(t, ErrNotReady, err) }) t.Run("GetPut", func(t *testing.T) { fp, err := pipe.GetPut(ctx, "key", "value") require.NoError(t, err) _, err = fp.Result() require.ErrorIs(t, ErrNotReady, err) }) t.Run("IncrByFloat", func(t *testing.T) { fp, err := pipe.IncrByFloat(ctx, "key", 1) require.NoError(t, err) _, err = fp.Result() require.ErrorIs(t, ErrNotReady, err) }) } func TestDMapPipeline_EmbeddedClient(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c := db.NewEmbeddedClient() defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) futures := make(map[int]*FuturePut) pipe, err := dm.Pipeline() require.NoError(t, err) defer pipe.Close() for i := 0; i < 100; i++ { fp, err := pipe.Put(ctx, testutil.ToKey(i), testutil.ToVal(i)) require.NoError(t, err) futures[i] = fp } err = pipe.Exec(ctx) require.NoError(t, err) for _, fp := range futures { require.NoError(t, fp.Result()) } for i := 0; i < 100; i++ { key := testutil.ToKey(i) gr, err := dm.Get(ctx, key) require.NoError(t, err) value, err := gr.Byte() require.NoError(t, err) require.Equal(t, testutil.ToVal(i), value) } } func TestDMapPipeline_setOrGetClusterClient(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c := db.NewEmbeddedClient() defer func() { require.NoError(t, c.Close(ctx)) }() dm, err := c.NewDMap("mydmap") require.NoError(t, err) pipeOne, err := dm.Pipeline() require.NoError(t, err) defer pipeOne.Close() require.NotNil(t, dm.(*EmbeddedDMap).clusterClient) } ================================================ FILE: pkg/flog/flog.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. /*Package flog is a simple wrapper around Golang's log package which adds verbosity support.*/ package flog // import "github.com/olric-data/olric/pkg/flog" import ( "fmt" "log" "path" "runtime" "sync/atomic" ) /* Derived from kubernetes/klog: * flog.V(1) - Generally useful for this to ALWAYS be visible to an operator * Programmer errors: * Logging extra info about a panic * CLI argument handling * flog.V(2) - A reasonable default log level if you don't want verbosity. * Information about config (listening on X, watching Y) * Errors that repeat frequently that relate to conditions that can be corrected (pod detected as unhealthy) * flog.V(3) - Useful steady state information about the service and important log messages that may correlate to significant changes in the system. This is the recommended default log level for most systems. * Logging HTTP requests and their exit code * System state changing (killing pod) * Controller state change events (starting pods) * Scheduler log messages * flog.V(4) - Extended information about changes * More info about system state changes * flog.V(5) - Debug level verbosity * Logging in particularly thorny parts of code where you may want to come back later and check it * flog.V(6) - Trace level verbosity * Context to understand the steps leading up to errors and warnings * More information for troubleshooting reported issues The practical default level is V(2). Developers and QE environments may wish to run at V(3) or V(4). */ // A Logger represents an active logging instance that generates lines of // output to an io.Writer. Each logging operation makes a single call to // the Writer's Encode method. A Logger can be used simultaneously from // multiple goroutines; it guarantees to serialize access to the Writer. type Logger struct { logger *log.Logger showLineNum int32 level int32 } // New returns a new Logger func New(logger *log.Logger) *Logger { return &Logger{ logger: logger, } } // SetLevel sets verbosity level. func (f *Logger) SetLevel(level int32) { if level < 0 { return } atomic.StoreInt32(&f.level, level) } // ShowLineNumber enables line number support if show is bigger than zero. func (f *Logger) ShowLineNumber(show int32) { if show < 0 { return } atomic.StoreInt32(&f.showLineNum, show) } // Verbose is a type that implements Printf and Println with verbosity support. type Verbose struct { ok bool f *Logger } // V reports whether verbosity at the call site is at least the requested level. The returned value is a struct // of type Verbose, which implements Printf and Println func (f *Logger) V(level int32) Verbose { return Verbose{ ok: atomic.LoadInt32(&f.level) >= level, f: f, } } // Ok will return true if this log level is enabled, guarded by the value of verbosity level. func (v Verbose) Ok() bool { return v.ok } // Printf calls v.f.logger.Printf to print to the logger. // Arguments are handled in the manner of fmt.Printf. func (v Verbose) Printf(format string, i ...interface{}) { if !v.ok { return } if atomic.LoadInt32(&v.f.showLineNum) != 1 { v.f.logger.Printf(format, i...) } else { _, fn, line, _ := runtime.Caller(1) v.f.logger.Printf(fmt.Sprintf("%s => %s:%d", format, path.Base(fn), line), i...) } } // Println calls v.f.logger.Println to print to the logger. // Arguments are handled in the manner of fmt.Println. func (v Verbose) Println(i ...interface{}) { if !v.ok { return } if atomic.LoadInt32(&v.f.showLineNum) != 1 { v.f.logger.Println(i...) } else { _, fn, line, _ := runtime.Caller(1) v.f.logger.Println(fmt.Sprintf("%s => %s:%d", fmt.Sprint(i...), path.Base(fn), line)) } } ================================================ FILE: pkg/neterrors/errors.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package neterrors import ( "errors" ) var ( ErrInvalidArgument = errors.New("invalid argument") ErrUnknownOperation = errors.New("unknown operation") ErrInternalFailure = errors.New("internal failure") ErrNotImplemented = errors.New("not implemented") ErrOperationTimeout = errors.New("operation timeout") ) ================================================ FILE: pkg/service_discovery/service_discovery.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. /*Package service_discovery provides ServiceDiscovery interface for plugins*/ package service_discovery // import "github.com/olric-data/olric/pkg/service_discovery" import "log" // ServiceDiscovery represents an interface for discovering, registering nodes within an Olric cluster. type ServiceDiscovery interface { // Initialize prepares the service discovery plugin for use and ensures it is ready for further operations. Initialize() error // SetConfig sets the configuration for the service discovery plugin using the provided map of settings. SetConfig(c map[string]interface{}) error // SetLogger assigns a custom logger to the service discovery instance for logging operations. SetLogger(l *log.Logger) // Register registers the current node in the service discovery directory, enabling it to participate in the cluster. Register() error // Deregister removes the current node from the service discovery directory and stops its participation in the cluster. Deregister() error // DiscoverPeers retrieves a list of available peers in the cluster and returns their addresses or an error if any occurs. DiscoverPeers() ([]string, error) // Close gracefully terminates all operations and releases resources associated with the service discovery instance. Close() error } ================================================ FILE: pkg/storage/config.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package storage import ( "fmt" "sync" ) // Config defines a new storage engine configuration type Config struct { m map[string]interface{} sync.RWMutex } // NewConfig returns a new Config func NewConfig(cfg map[string]interface{}) *Config { if cfg == nil { cfg = make(map[string]interface{}) } return &Config{ m: cfg, } } // Add adds a new key/value pair to Config func (c *Config) Add(key string, value interface{}) { c.Lock() defer c.Unlock() c.m[key] = value } // Get loads a configuration variable with its key, otherwise it returns an error. func (c *Config) Get(key string) (interface{}, error) { c.Lock() defer c.Unlock() value, ok := c.m[key] if !ok { return nil, fmt.Errorf("not found: %s", key) } return value, nil } // Delete deletes a configuration variable with its key. func (c *Config) Delete(key string) { c.Lock() defer c.Unlock() delete(c.m, key) } // Copy creates a thread-safe copy of the existing Config struct. func (c *Config) Copy() *Config { c.Lock() defer c.Unlock() n := &Config{ m: make(map[string]interface{}), } for key, value := range c.m { n.m[key] = value } return n } // ToMap casts Config to map[string]interface{} type. func (c *Config) ToMap() map[string]interface{} { return c.Copy().m } ================================================ FILE: pkg/storage/config_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package storage import ( "reflect" "testing" ) func Test_Config(t *testing.T) { c := NewConfig(nil) c.Add("string-key", "string-value") c.Add("integer-key", 65786) t.Run("Get", func(t *testing.T) { sv, err := c.Get("string-key") if err != nil { t.Fatalf("Expected nil. Got %v", err) } if sv.(string) != "string-value" { t.Fatalf("Expected string-value. Got %v", sv) } iv, err := c.Get("integer-key") if err != nil { t.Fatalf("Expected nil. Got %v", err) } if iv.(int) != 65786 { t.Fatalf("Expected integer-value. Got %v", iv) } }) t.Run("Delete", func(t *testing.T) { c.Delete("string-key") _, err := c.Get("string-key") if err == nil { t.Fatalf("Expected an error. Got %v", err) } }) t.Run("Copy", func(t *testing.T) { copied := c.Copy() if copied == c { t.Fatalf("New config is the same with the previous one") } if !reflect.DeepEqual(c, copied) { t.Fatalf("New config is not idential with the previous one") } }) } ================================================ FILE: pkg/storage/engine.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package storage import ( "errors" "log" ) // ErrKeyTooLarge is an error that indicates the given key is larger than the determined key size. // The current maximum key length is 256. var ErrKeyTooLarge = errors.New("key too large") // ErrEntryTooLarge returned if required space for an entry is bigger than table size. var ErrEntryTooLarge = errors.New("entry too large for the configured table size") // ErrKeyNotFound is an error that indicates that the requested key could not be found in the DB. var ErrKeyNotFound = errors.New("key not found") // ErrNotImplemented means that the interface implementation does not support // the functionality required to fulfill the request. var ErrNotImplemented = errors.New("not implemented yet") // TransferIterator is an interface to implement iterators to encode and transfer // the underlying tables to another Olric member. type TransferIterator interface { // Next returns true if there are more tables to Export in the storage instance. // Otherwise, it returns false. Next() bool // Export encodes a table and returns result. This encoded table can be moved to another Olric node. Export() ([]byte, int, error) // Drop drops a table with its index from the storage engine instance and frees allocated resources. Drop(int) error } // Engine defines methods for a storage engine implementation. type Engine interface { // SetConfig sets a storage engine configuration. nil can be accepted, but // it depends on the implementation. SetConfig(*Config) // SetLogger sets a logger. nil can be accepted, but it depends on the implementation. SetLogger(*log.Logger) // Start can be used to run background services before starting operation. Start() error // NewEntry returns a new Entry interface implemented by the current storage // engine implementation. NewEntry() Entry // Name returns name of the current storage engine implementation. Name() string // Fork creates an empty instance of an online engine by using the current // configuration. Fork(*Config) (Engine, error) // PutRaw inserts an encoded entry into the storage engine. PutRaw(uint64, []byte) error // Put inserts a new Entry into the storage engine. Put(uint64, Entry) error // GetRaw reads an encoded entry from the storage engine. GetRaw(uint64) ([]byte, error) // Get reads an entry from the storage engine. Get(uint64) (Entry, error) // GetTTL extracts TTL of an entry. GetTTL(uint64) (int64, error) // GetLastAccess extracts LastAccess of an entry. GetLastAccess(uint64) (int64, error) // GetKey extracts key of an entry. GetKey(uint64) (string, error) // Delete deletes an entry from the storage engine. Delete(uint64) error // UpdateTTL updates TTL of an entry. It returns ErrKeyNotFound, // if the key doesn't exist. UpdateTTL(uint64, Entry) error // TransferIterator returns a new TransferIterator instance to the caller. TransferIterator() TransferIterator // Import imports an encoded table of the storage engine implementation and // calls f for every Entry item in that table. Import(data []byte, f func(uint64, Entry) error) error // Stats returns metrics for an online storage engine. Stats() Stats // Check returns true, if the key exists. Check(uint64) bool // Range implements a loop over the storage engine Range(func(uint64, Entry) bool) // RangeHKey implements a loop for hashed keys(HKeys). RangeHKey(func(uint64) bool) // Scan implements an iterator. The caller starts iterating from the cursor. "count" is the number of entries // that will be returned during the iteration. Scan calls the function "f" on Entry items for every iteration. //It returns the next cursor if everything is okay. Otherwise, it returns an error. Scan(cursor uint64, count int, f func(Entry) bool) (uint64, error) // ScanRegexMatch is the same with the Scan method, but it supports regular expressions on keys. ScanRegexMatch(cursor uint64, match string, count int, f func(Entry) bool) (uint64, error) // Compaction reorganizes storage tables and reclaims wasted resources. Compaction() (bool, error) // Close stops an online storage engine instance. It may free some of allocated // resources. A storage engine implementation should be started again, but it // depends on the implementation. Close() error // Destroy stops an online storage engine instance and frees allocated resources. // It should not be possible to reuse a destroyed storage engine. Destroy() error } ================================================ FILE: pkg/storage/entry.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package storage // Entry interface defines methods for a storage entry. type Entry interface { // SetKey accepts strings as a key and inserts the key into the underlying // data structure. SetKey(string) // Key returns the key as string Key() string // SetValue accepts a byte slice as a value and inserts the value into the // underlying data structure. SetValue([]byte) // Value returns the value as a byte slice. Value() []byte // SetTTL sets TTL to an entry. SetTTL(int64) // TTL returns the current TTL for an entry. TTL() int64 // SetTimestamp sets the current timestamp to an entry. SetTimestamp(int64) // Timestamp returns the current timestamp for an entry. Timestamp() int64 SetLastAccess(int64) LastAccess() int64 // Encode encodes an entry into a binary form and returns the result. Encode() []byte // Decode decodes a byte slice into an Entry. Decode([]byte) } ================================================ FILE: pkg/storage/stats.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package storage // Stats defines metrics exposed by a storage engine implementation. type Stats struct { // Currently allocated memory by the engine. Allocated int // Used portion of allocated memory Inuse int // Deleted portions of allocated memory. Garbage int // Total number of keys hosted by the engine instance. Length int // Number of tables hosted by the engine instance. NumTables int // Any other metrics that's specific to an engine implementation. Extras map[string]interface{} } ================================================ FILE: pubsub.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package olric import ( "context" "strings" "github.com/olric-data/olric/internal/server" "github.com/redis/go-redis/v9" ) type PubSub struct { config *pubsubConfig rc *redis.Client client *server.Client } func newPubSub(client *server.Client, options ...PubSubOption) (*PubSub, error) { var ( err error rc *redis.Client pc pubsubConfig ) for _, opt := range options { opt(&pc) } addr := strings.Trim(pc.Address, " ") if addr != "" { rc = client.Get(addr) } else { rc, err = client.Pick() if err != nil { return nil, err } } return &PubSub{ config: &pc, rc: rc, client: client, }, nil } func (ps *PubSub) Subscribe(ctx context.Context, channels ...string) *redis.PubSub { return ps.rc.Subscribe(ctx, channels...) } func (ps *PubSub) PSubscribe(ctx context.Context, channels ...string) *redis.PubSub { return ps.rc.PSubscribe(ctx, channels...) } func (ps *PubSub) Publish(ctx context.Context, channel string, message interface{}) (int64, error) { return ps.rc.Publish(ctx, channel, message).Result() } func (ps *PubSub) PubSubChannels(ctx context.Context, pattern string) ([]string, error) { return ps.rc.PubSubChannels(ctx, pattern).Result() } func (ps *PubSub) PubSubNumSub(ctx context.Context, channels ...string) (map[string]int64, error) { return ps.rc.PubSubNumSub(ctx, channels...).Result() } func (ps *PubSub) PubSubNumPat(ctx context.Context) (int64, error) { return ps.rc.PubSubNumPat(ctx).Result() } ================================================ FILE: pubsub_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package olric import ( "context" "fmt" "testing" "time" "github.com/redis/go-redis/v9" "github.com/stretchr/testify/require" ) func pubsubTestRunner(t *testing.T, ps *PubSub, kind, channel string) { ctx := context.Background() var rp *redis.PubSub switch kind { case "subscribe": rp = ps.Subscribe(ctx, channel) case "psubscribe": rp = ps.PSubscribe(ctx, channel) } defer func() { require.NoError(t, rp.Close()) }() // Wait for confirmation that subscription is created before publishing anything. msgi, err := rp.ReceiveTimeout(ctx, time.Second) require.NoError(t, err) subs := msgi.(*redis.Subscription) require.Equal(t, kind, subs.Kind) require.Equal(t, channel, subs.Channel) require.Equal(t, 1, subs.Count) // Go channel which receives messages. ch := rp.Channel() expected := make(map[string]struct{}) for i := 0; i < 10; i++ { msg := fmt.Sprintf("my-message-%d", i) count, err := ps.Publish(ctx, "my-channel", msg) require.Equal(t, int64(1), count) require.NoError(t, err) expected[msg] = struct{}{} } consumed := make(map[string]struct{}) L: for { select { case msg := <-ch: require.Equal(t, "my-channel", msg.Channel) consumed[msg.Payload] = struct{}{} if len(consumed) == 10 { // It would be OK break L } case <-time.After(5 * time.Second): // Enough. Break it and check the consumed items. break L } } require.Equal(t, expected, consumed) } func TestPubSub_Publish_Subscribe(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() ps, err := c.NewPubSub(ToAddress(db.rt.This().String())) require.NoError(t, err) pubsubTestRunner(t, ps, "subscribe", "my-channel") } func TestPubSub_Publish_PSubscribe(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() ps, err := c.NewPubSub(ToAddress(db.rt.This().String())) require.NoError(t, err) pubsubTestRunner(t, ps, "psubscribe", "my-*") } func TestPubSub_PubSubChannels(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() ps, err := c.NewPubSub(ToAddress(db.rt.This().String())) require.NoError(t, err) rp := ps.Subscribe(ctx, "my-channel") defer func() { require.NoError(t, rp.Close()) }() // Wait for confirmation that subscription is created before publishing anything. _, err = rp.ReceiveTimeout(ctx, time.Second) require.NoError(t, err) channels, err := ps.PubSubChannels(ctx, "my-*") require.NoError(t, err) require.Equal(t, []string{"my-channel"}, channels) } func TestPubSub_PubSubNumSub(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() ps, err := c.NewPubSub(ToAddress(db.rt.This().String())) require.NoError(t, err) rp := ps.Subscribe(ctx, "my-channel") defer func() { require.NoError(t, rp.Close()) }() // Wait for confirmation that subscription is created before publishing anything. _, err = rp.ReceiveTimeout(ctx, time.Second) require.NoError(t, err) numsub, err := ps.PubSubNumSub(ctx, "my-channel", "foobar") require.NoError(t, err) expected := map[string]int64{ "foobar": 0, "my-channel": 1, } require.Equal(t, expected, numsub) } func TestPubSub_PubSubNumPat(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) ctx := context.Background() c, err := NewClusterClient([]string{db.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() ps, err := c.NewPubSub(ToAddress(db.rt.This().String())) require.NoError(t, err) rp := ps.PSubscribe(ctx, "my-*") defer func() { require.NoError(t, rp.Close()) }() // Wait for confirmation that subscription is created before publishing anything. _, err = rp.ReceiveTimeout(ctx, time.Second) require.NoError(t, err) numpat, err := ps.PubSubNumPat(ctx) require.NoError(t, err) require.Equal(t, int64(1), numpat) } func TestPubSub_Cluster(t *testing.T) { cluster := newTestOlricCluster(t) db1 := cluster.addMember(t) db2 := cluster.addMember(t) // Create a subscriber ctx := context.Background() c, err := NewClusterClient([]string{db1.name}) require.NoError(t, err) defer func() { require.NoError(t, c.Close(ctx)) }() ps1, err := c.NewPubSub(ToAddress(db1.rt.This().String())) require.NoError(t, err) rp := ps1.Subscribe(ctx, "my-channel") defer func() { require.NoError(t, rp.Close()) }() // Wait for confirmation that subscription is created before publishing anything. _, err = rp.ReceiveTimeout(ctx, time.Second) require.NoError(t, err) receiveChan := rp.Channel() // Create a publisher e := db2.NewEmbeddedClient() ps2, err := e.NewPubSub(ToAddress(db2.rt.This().String())) require.NoError(t, err) expected := make(map[string]struct{}) for i := 0; i < 10; i++ { msg := fmt.Sprintf("my-message-%d", i) count, err := ps2.Publish(ctx, "my-channel", msg) require.Equal(t, int64(1), count) require.NoError(t, err) expected[msg] = struct{}{} } consumed := make(map[string]struct{}) L: for { select { case msg := <-receiveChan: require.Equal(t, "my-channel", msg.Channel) consumed[msg.Payload] = struct{}{} if len(consumed) == 10 { // It would be OK break L } case <-time.After(5 * time.Second): // Enough. Break it and check the consumed items. break L } } } ================================================ FILE: stats/stats.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. /*Package stats exposes internal data structures for Stat command*/ package stats import "runtime" type ( // PartitionID denotes ID of a partition in the cluster. PartitionID uint64 // MemberID denotes ID of a member in the cluster. MemberID uint64 ) // SlabInfo denotes memory usage of the storage engine(a hash indexed, append only byte slice). type SlabInfo struct { // Total allocated space by the append-only byte slice. Allocated int `json:"allocated"` // Total inuse memory space in the append-only byte slice. Inuse int `json:"inuse"` // Total garbage(deleted key/value pairs) space in the append-only byte slice. Garbage int `json:"garbage"` } // DMap denotes a distributed map instance on the cluster. type DMap struct { // Number of keys in the DMap. Length int `json:"length"` // Statistics about memory representation of a DMap. SlabInfo SlabInfo `json:"slab_info"` // Number of tables in a storage instance. NumTables int `json:"num_tables"` } // Partition denotes a partition and its metadata in the cluster. type Partition struct { // PreviousOwners is a list of members whose still owns some fragments. PreviousOwners []Member `json:"previous_owners"` // Backups is a list of members whose holds replicas of this partition. Backups []Member `json:"backups"` // Total number of entries in the partition. Length int `json:"length"` // DMaps is a map that contains statistics of DMaps in this partition. DMaps map[string]DMap `json:"dmaps"` } // Runtime exposes memory stats and various metrics from Go runtime. type Runtime struct { // GOOS is the running program's operating system target GOOS string `json:"goos"` // GOARCH is the running program's architecture target GOARCH string `json:"goarch"` // Version returns the Go tree's version string. Version string `json:"version"` // NumCPU returns the number of logical CPUs usable by the current process. NumCPU int `json:"num_cpu"` // NumGoroutine returns the number of goroutines that currently exist. NumGoroutine int `json:"num_goroutine"` // MemStats records statistics about the memory allocator. MemStats runtime.MemStats `json:"mem_stats"` } // Member denotes a cluster member. type Member struct { // Name is name of the node in the cluster. Name string `json:"name"` // ID is the unique identifier of this node in the cluster. It's derived // from Name and Birthdate. ID uint64 `json:"id"` // Birthdate is UNIX time in nanoseconds. Birthdate int64 `json:"birthdate"` } // String returns the member name. func (m Member) String() string { return m.Name } // Network holds network statistics. type Network struct { // ConnectionsTotal is total number of connections opened since the server started running. ConnectionsTotal int64 `json:"connections_total"` // CurrentConnections is current number of open connections. CurrentConnections int64 `json:"current_connections"` // WrittenBytesTotal is total number of bytes sent by this server to network. WrittenBytesTotal int64 `json:"written_bytes_total"` // ReadBytesTotal is total number of bytes read by this server from network. ReadBytesTotal int64 `json:"read_bytes_total"` // CommandsTotal is total number of all requests (get, put, etc.). CommandsTotal int64 `json:"commands_total"` } // DMaps holds global DMap statistics. type DMaps struct { // EntriesTotal is the total number of entries(including replicas) stored during the life of this instance. EntriesTotal int64 `json:"entries_total"` // DeleteHits is the number of deletion reqs resulting in an item being removed. DeleteHits int64 `json:"delete_hits"` // DeleteMisses is the number of deletions reqs for missing keys DeleteMisses int64 `json:"delete_misses"` // GetMisses is the number of entries that have been requested and not found GetMisses int64 `json:"get_misses"` // GetHits is the number of entries that have been requested and found present GetHits int64 `json:"get_hits"` // EvictedTotal is the number of entries removed from cache to free memory for new entries. EvictedTotal int64 `json:"evicted_total"` } // PubSub holds global Pub/Sub statistics. type PubSub struct { // PublishedTotal is the total number of published messages to PubSub during the life of this instance. PublishedTotal int64 `json:"published_total"` // CurrentSubscribers is the current number of Pub/Sub listeners of PubSub. CurrentSubscribers int64 `json:"current_subscribers"` // SubscribersTotal is the total number of registered Pub/Sub listeners during the life of this instance. SubscribersTotal int64 `json:"subscribers_total"` // CurrentSubscribers is the current number of Pub/Sub listeners of PubSub. CurrentPSubscribers int64 `json:"current_psubscribers"` // SubscribersTotal is the total number of registered Pub/Sub listeners during the life of this instance. PSubscribersTotal int64 `json:"psubscribers_total"` } // Stats is a struct that exposes statistics about the current state of a member. type Stats struct { // Cmdline holds the command-line arguments, starting with the program name. Cmdline []string `json:"cmdline"` // ReleaseVersion is the current Olric version ReleaseVersion string `json:"release_version"` // UptimeSeconds is number of seconds since the server started. UptimeSeconds int64 `json:"uptime_seconds"` // Stats from Golang runtime Runtime *Runtime `json:"runtime"` // ClusterCoordinator is the current cluster coordinator. ClusterCoordinator Member `json:"cluster_coordinator"` // Member denotes the current member. Member Member `json:"member"` // Partitions is a map that contains partition statistics. Partitions map[PartitionID]Partition `json:"partitions"` // Backups is a map that contains backup partition statistics. Backups map[PartitionID]Partition `json:"backups"` // ClusterMembers is a map that contains bootstrapped cluster members ClusterMembers map[MemberID]Member `json:"cluster_members"` // Network holds network statistics. Network Network `json:"network"` // DMaps holds global DMap statistics. DMaps DMaps `json:"dmaps"` // PubSub holds global Pub/Sub statistics. PubSub PubSub `json:"pub_sub"` } ================================================ FILE: stats/stats_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package stats import ( "testing" "time" "github.com/stretchr/testify/require" ) func TestMember_String(t *testing.T) { m := Member{ Name: "foobar", ID: 123345645678, Birthdate: time.Now().UnixNano(), } require.Equal(t, "foobar", m.String()) } ================================================ FILE: stats.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package olric import ( "encoding/json" "os" "runtime" "strings" "github.com/olric-data/olric/internal/cluster/partitions" "github.com/olric-data/olric/internal/discovery" "github.com/olric-data/olric/internal/dmap" "github.com/olric-data/olric/internal/protocol" "github.com/olric-data/olric/internal/pubsub" "github.com/olric-data/olric/internal/server" "github.com/olric-data/olric/stats" "github.com/tidwall/redcon" ) func toMember(member discovery.Member) stats.Member { return stats.Member{ Name: member.Name, ID: member.ID, Birthdate: member.Birthdate, } } func toMembers(members []discovery.Member) []stats.Member { var _stats []stats.Member for _, m := range members { _stats = append(_stats, toMember(m)) } return _stats } func (db *Olric) collectPartitionMetrics(partID uint64, part *partitions.Partition) stats.Partition { owners := part.Owners() p := stats.Partition{ Backups: toMembers(db.backup.PartitionOwnersByID(partID)), Length: part.Length(), DMaps: make(map[string]stats.DMap), } if len(owners) > 0 { p.PreviousOwners = toMembers(owners[:len(owners)-1]) } part.Map().Range(func(name, item interface{}) bool { f := item.(partitions.Fragment) st := f.Stats() tmp := stats.DMap{ Length: st.Length, NumTables: st.NumTables, } tmp.SlabInfo.Allocated = st.Allocated tmp.SlabInfo.Garbage = st.Garbage tmp.SlabInfo.Inuse = st.Inuse dmapName := strings.TrimPrefix(name.(string), "dmap.") p.DMaps[dmapName] = tmp return true }) return p } func (db *Olric) checkPartitionOwnership(part *partitions.Partition) bool { owners := part.Owners() for _, owner := range owners { if owner.CompareByID(db.rt.This()) { return true } } return false } func (db *Olric) stats(cfg statsConfig) stats.Stats { s := stats.Stats{ Cmdline: os.Args, ReleaseVersion: ReleaseVersion, UptimeSeconds: discovery.UptimeSeconds.Read(), ClusterCoordinator: toMember(db.rt.Discovery().GetCoordinator()), Member: toMember(db.rt.This()), Partitions: make(map[stats.PartitionID]stats.Partition), Backups: make(map[stats.PartitionID]stats.Partition), ClusterMembers: make(map[stats.MemberID]stats.Member), Network: stats.Network{ ConnectionsTotal: server.ConnectionsTotal.Read(), CurrentConnections: server.CurrentConnections.Read(), WrittenBytesTotal: server.WrittenBytesTotal.Read(), ReadBytesTotal: server.ReadBytesTotal.Read(), CommandsTotal: server.CommandsTotal.Read(), }, DMaps: stats.DMaps{ EntriesTotal: dmap.EntriesTotal.Read(), DeleteHits: dmap.DeleteHits.Read(), DeleteMisses: dmap.DeleteMisses.Read(), GetMisses: dmap.GetMisses.Read(), GetHits: dmap.GetHits.Read(), EvictedTotal: dmap.EvictedTotal.Read(), }, PubSub: stats.PubSub{ PublishedTotal: pubsub.PublishedTotal.Read(), CurrentSubscribers: pubsub.CurrentSubscribers.Read(), SubscribersTotal: pubsub.SubscribersTotal.Read(), CurrentPSubscribers: pubsub.CurrentPSubscribers.Read(), PSubscribersTotal: pubsub.PSubscribersTotal.Read(), }, } if cfg.CollectRuntime { s.Runtime = &stats.Runtime{ GOOS: runtime.GOOS, GOARCH: runtime.GOARCH, Version: runtime.Version(), NumCPU: runtime.NumCPU(), NumGoroutine: runtime.NumGoroutine(), } runtime.ReadMemStats(&s.Runtime.MemStats) } db.rt.RLock() defer db.rt.RUnlock() db.rt.Members().Range(func(id uint64, member discovery.Member) bool { s.ClusterMembers[stats.MemberID(id)] = toMember(member) return true }) for partID := uint64(0); partID < db.config.PartitionCount; partID++ { primary := db.primary.PartitionByID(partID) if db.checkPartitionOwnership(primary) { s.Partitions[stats.PartitionID(partID)] = db.collectPartitionMetrics(partID, primary) } backup := db.backup.PartitionByID(partID) if db.checkPartitionOwnership(backup) { s.Backups[stats.PartitionID(partID)] = db.collectPartitionMetrics(partID, backup) } } return s } func (db *Olric) statsCommandHandler(conn redcon.Conn, cmd redcon.Command) { statsCmd, err := protocol.ParseStatsCommand(cmd) if err != nil { protocol.WriteError(conn, err) return } sc := statsConfig{} if statsCmd.CollectRuntime { sc.CollectRuntime = true } memberStats := db.stats(sc) data, err := json.Marshal(memberStats) if err != nil { protocol.WriteError(conn, err) return } conn.WriteBulk(data) } ================================================ FILE: stats_test.go ================================================ // Copyright 2018-2025 The Olric Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package olric import ( "context" "fmt" "testing" "time" "github.com/olric-data/olric/internal/dmap" "github.com/olric-data/olric/internal/protocol" "github.com/olric-data/olric/internal/pubsub" "github.com/olric-data/olric/internal/testutil" "github.com/olric-data/olric/stats" "github.com/redis/go-redis/v9" "github.com/stretchr/testify/require" ) func resetPubSubStats() { pubsub.SubscribersTotal.Reset() pubsub.CurrentPSubscribers.Reset() pubsub.CurrentSubscribers.Reset() pubsub.PSubscribersTotal.Reset() pubsub.PublishedTotal.Reset() } func TestOlric_Stats(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) c := db.NewEmbeddedClient() dm, err := c.NewDMap("mymap") require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) defer cancel() for i := 0; i < 100; i++ { err = dm.Put(ctx, testutil.ToKey(i), testutil.ToVal(i)) require.NoError(t, err) } s, err := c.Stats(ctx, db.rt.This().String()) require.NoError(t, err) if s.ClusterCoordinator.ID != db.rt.This().ID { t.Fatalf("Expected cluster coordinator: %v. Got: %v", db.rt.This(), s.ClusterCoordinator) } require.Equal(t, s.Member.Name, db.rt.This().Name) require.Equal(t, s.Member.ID, db.rt.This().ID) require.Equal(t, s.Member.Birthdate, db.rt.This().Birthdate) if s.Runtime != nil { t.Error("Runtime stats must not be collected by default:", s.Runtime) } var total int for partID, part := range s.Partitions { total += part.Length if _, ok := part.DMaps["mymap"]; !ok { t.Fatalf("Expected dmap check result is true. Got false") } if len(part.PreviousOwners) != 0 { t.Fatalf("Expected PreviosOwners list is empty. "+ "Got: %v for PartID: %d", part.PreviousOwners, partID) } if part.Length <= 0 { t.Fatalf("Unexpected Length: %d", part.Length) } } if total != 100 { t.Fatalf("Expected total length of partition in stats is 100. Got: %d", total) } _, ok := s.ClusterMembers[stats.MemberID(db.rt.This().ID)] if !ok { t.Fatalf("Expected member ID: %d could not be found in ClusterMembers", db.rt.This().ID) } } func TestOlric_Stats_CollectRuntime(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) e := db.NewEmbeddedClient() s, err := e.Stats(context.Background(), db.rt.This().String(), CollectRuntime()) require.NoError(t, err) if s.Runtime == nil { t.Fatal("Runtime stats must be collected by default:", s.Runtime) } } func TestOlric_Stats_Cluster(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) db2 := cluster.addMember(t) e := db.NewEmbeddedClient() s, err := e.Stats(context.Background(), db2.rt.This().String()) require.NoError(t, err) require.Nil(t, s.Runtime) require.Equal(t, s.Member.String(), db2.rt.This().String()) } func TestStats_PubSub(t *testing.T) { resetPubSubStats() cluster := newTestOlricCluster(t) db := cluster.addMember(t) rc := redis.NewClient(&redis.Options{Addr: db.rt.This().String()}) ctx := context.Background() t.Run("Subscribe", func(t *testing.T) { defer func() { resetPubSubStats() }() var subscribers []*redis.PubSub for i := 0; i < 5; i++ { ps := rc.Subscribe(ctx, "my-channel") // Wait for confirmation that subscription is created before publishing anything. _, err := ps.Receive(ctx) require.NoError(t, err) subscribers = append(subscribers, ps) } for i := 0; i < 10; i++ { cmd := rc.Publish(ctx, "my-channel", fmt.Sprintf("message-%d", i)) res, err := cmd.Result() require.Equal(t, int64(5), res) require.NoError(t, err) } require.Equal(t, int64(50), pubsub.PublishedTotal.Read()) require.Equal(t, int64(5), pubsub.SubscribersTotal.Read()) require.Equal(t, int64(5), pubsub.CurrentSubscribers.Read()) require.Equal(t, int64(0), pubsub.PSubscribersTotal.Read()) require.Equal(t, int64(0), pubsub.CurrentPSubscribers.Read()) // Unsubscribe for _, s := range subscribers { err := s.Unsubscribe(ctx, "my-channel") require.NoError(t, err) <-time.After(100 * time.Millisecond) } require.Equal(t, int64(5), pubsub.SubscribersTotal.Read()) require.Equal(t, int64(0), pubsub.CurrentSubscribers.Read()) require.Equal(t, int64(0), pubsub.PSubscribersTotal.Read()) require.Equal(t, int64(0), pubsub.CurrentPSubscribers.Read()) }) t.Run("PSubscribe", func(t *testing.T) { defer func() { resetPubSubStats() }() ps := rc.PSubscribe(ctx, "h?llo") // Wait for confirmation that subscription is created before publishing anything. _, err := ps.Receive(ctx) require.NoError(t, err) cmd := rc.Publish(ctx, "hxllo", "message") res, err := cmd.Result() require.Equal(t, int64(1), res) require.NoError(t, err) require.Equal(t, int64(1), pubsub.PublishedTotal.Read()) require.Equal(t, int64(1), pubsub.PSubscribersTotal.Read()) require.Equal(t, int64(1), pubsub.CurrentPSubscribers.Read()) require.Equal(t, int64(0), pubsub.SubscribersTotal.Read()) require.Equal(t, int64(0), pubsub.CurrentSubscribers.Read()) err = ps.PUnsubscribe(ctx, "h?llo") require.NoError(t, err) <-time.After(100 * time.Millisecond) require.Equal(t, int64(1), pubsub.PSubscribersTotal.Read()) require.Equal(t, int64(0), pubsub.CurrentPSubscribers.Read()) require.Equal(t, int64(0), pubsub.SubscribersTotal.Read()) require.Equal(t, int64(0), pubsub.CurrentSubscribers.Read()) }) } func TestStats_DMap(t *testing.T) { cluster := newTestOlricCluster(t) db := cluster.addMember(t) rc := redis.NewClient(&redis.Options{Addr: db.rt.This().String()}) ctx := context.Background() t.Run("DMap stats without eviction", func(t *testing.T) { // EntriesTotal for i := 0; i < 10; i++ { cmd := protocol.NewPut("mydmap", fmt.Sprintf("mykey-%d", i), []byte("myvalue")).Command(ctx) err := rc.Process(ctx, cmd) require.NoError(t, err) require.NoError(t, cmd.Err()) } // GetHits for i := 0; i < 10; i++ { cmd := protocol.NewGet("mydmap", fmt.Sprintf("mykey-%d", i)).Command(ctx) err := rc.Process(ctx, cmd) require.NoError(t, err) require.NoError(t, cmd.Err()) } // DeleteHits for i := 0; i < 10; i++ { cmd := protocol.NewDel("mydmap", fmt.Sprintf("mykey-%d", i)).Command(ctx) err := rc.Process(ctx, cmd) require.NoError(t, err) require.NoError(t, cmd.Err()) } // GetMisses for i := 0; i < 10; i++ { cmd := protocol.NewGet("mydmap", fmt.Sprintf("mykey-%d", i)).Command(ctx) err := rc.Process(ctx, cmd) err = protocol.ConvertError(err) require.ErrorIs(t, err, dmap.ErrKeyNotFound) } // DeleteMisses for i := 0; i < 10; i++ { cmd := protocol.NewDel("mydmap", fmt.Sprintf("mykey-%d", i)).Command(ctx) err := rc.Process(ctx, cmd) require.NoError(t, err) require.NoError(t, cmd.Err()) } require.GreaterOrEqual(t, dmap.EntriesTotal.Read(), int64(10)) require.GreaterOrEqual(t, dmap.GetMisses.Read(), int64(10)) require.GreaterOrEqual(t, dmap.GetHits.Read(), int64(10)) require.GreaterOrEqual(t, dmap.DeleteHits.Read(), int64(10)) require.GreaterOrEqual(t, dmap.DeleteMisses.Read(), int64(10)) }) t.Run("DMap eviction stats", func(t *testing.T) { // EntriesTotal, EvictedTotal for i := 0; i < 10; i++ { cmd := protocol. NewPut("mydmap", fmt.Sprintf("mykey-%d", i), []byte("myvalue")). SetPX(time.Millisecond.Milliseconds()). Command(ctx) err := rc.Process(ctx, cmd) require.NoError(t, err) require.NoError(t, cmd.Err()) } <-time.After(100 * time.Millisecond) // GetMisses for i := 0; i < 10; i++ { cmd := protocol.NewGet("mydmap", "mykey").Command(ctx) err := rc.Process(ctx, cmd) err = protocol.ConvertError(err) require.ErrorIs(t, err, dmap.ErrKeyNotFound) } require.Greater(t, dmap.DeleteHits.Read(), int64(0)) require.Greater(t, dmap.EvictedTotal.Read(), int64(0)) require.GreaterOrEqual(t, dmap.EntriesTotal.Read(), int64(10)) }) }