Showing preview only (343K chars total). Download the full file or copy to clipboard to get everything.
Repository: nomic-io/merk
Branch: develop
Commit: 898cc4e95df1
Files: 40
Total size: 328.6 KB
Directory structure:
gitextract_1164yhgf/
├── .github/
│ └── workflows/
│ └── ci.yml
├── .gitignore
├── CHANGELOG.md
├── Cargo.toml
├── LICENSE
├── README.md
├── benches/
│ ├── merk.rs
│ └── ops.rs
├── docs/
│ └── algorithms.md
├── rustfmt.toml
├── scripts/
│ └── pgo.sh
└── src/
├── error.rs
├── lib.rs
├── merk/
│ ├── chunks.rs
│ ├── mod.rs
│ ├── restore.rs
│ └── snapshot.rs
├── owner.rs
├── proofs/
│ ├── chunk.rs
│ ├── encoding.rs
│ ├── mod.rs
│ ├── query/
│ │ ├── map.rs
│ │ └── mod.rs
│ └── tree.rs
├── test_utils/
│ ├── crash_merk.rs
│ ├── mod.rs
│ └── temp_merk.rs
└── tree/
├── commit.rs
├── debug.rs
├── encoding.rs
├── fuzz_tests.rs
├── hash.rs
├── iter.rs
├── kv.rs
├── link.rs
├── mod.rs
├── ops.rs
└── walk/
├── fetch.rs
├── mod.rs
└── ref_walker.rs
================================================
FILE CONTENTS
================================================
================================================
FILE: .github/workflows/ci.yml
================================================
name: CI
on:
push:
branches: [master, develop]
pull_request:
branches: [master, develop]
env:
CARGO_TERM_COLOR: always
jobs:
test-base:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Use Nightly
uses: actions-rs/toolchain@v1
with:
toolchain: nightly-2024-04-25
override: true
- name: Cache
uses: actions/cache@v3
with:
path: |
~/.cargo/bin/
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
~/.cargo/registry/src/**/librocksdb-sys-*
target/
key: ${{ runner.os }}-test-base-${{ hashFiles('Cargo.toml') }}
- name: Test
uses: actions-rs/cargo@v1
with:
command: test
args: --verbose
test-all-features:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Use Nightly
uses: actions-rs/toolchain@v1
with:
toolchain: nightly-2024-04-25
override: true
- name: Cache
uses: actions/cache@v3
with:
path: |
~/.cargo/bin/
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
~/.cargo/registry/src/**/librocksdb-sys-*
target/
key: ${{ runner.os }}-test-all-features-${{ hashFiles('Cargo.toml') }}
- name: Test
uses: actions-rs/cargo@v1
with:
command: test
args: --verbose --all-features
coverage:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Use Nightly
uses: actions-rs/toolchain@v1
with:
toolchain: nightly-2024-04-25
components: llvm-tools-preview
override: true
- name: Cache
uses: actions/cache@v3
with:
path: |
~/.cargo/bin/
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
~/.cargo/registry/src/**/librocksdb-sys-*
target/
key: ${{ runner.os }}-coverage-${{ hashFiles('Cargo.toml') }}
- name: Install Coverage Tooling
uses: actions-rs/cargo@v1
with:
command: install
args: cargo-llvm-cov --force
- name: Run Coverage
uses: actions-rs/cargo@v1
with:
command: llvm-cov
args: --all-features --workspace --lcov --output-path lcov.info
- name: Upload to codecov.io
uses: codecov/codecov-action@v1
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: lcov.info
fail_ci_if_error: true
format:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Use Nightly
uses: actions-rs/toolchain@v1
with:
toolchain: nightly-2024-04-25
components: rustfmt
override: true
- name: Check
uses: actions-rs/cargo@v1
with:
command: fmt
args: --all -- --check
clippy:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Use Nightly
uses: actions-rs/toolchain@v1
with:
toolchain: nightly-2024-04-25
components: clippy
override: true
- name: Cache
uses: actions/cache@v3
with:
path: |
~/.cargo/bin/
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
~/.cargo/registry/src/**/librocksdb-sys-*
target/
key: ${{ runner.os }}-clippy-${{ hashFiles('Cargo.toml') }}
- name: Check
uses: actions-rs/clippy-check@v1
with:
token: ${{ secrets.GITHUB_TOKEN }}
args: --all-features -- -D warnings
benches:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Use Nightly
uses: actions-rs/toolchain@v1
with:
toolchain: nightly-2024-04-25
override: true
- name: Cache
uses: actions/cache@v3
with:
path: |
~/.cargo/bin/
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
~/.cargo/registry/src/**/librocksdb-sys-*
target/
key: ${{ runner.os }}-benches-${{ hashFiles('Cargo.toml') }}
- name: Run Benches
uses: actions-rs/cargo@v1
with:
command: bench
================================================
FILE: .gitignore
================================================
target
temp.db
.DS_Store
Cargo.lock
================================================
FILE: CHANGELOG.md
================================================
# Changelog
## [Unreleased]
### Bug Fixes
- Fixed bug where column families would be non-atomically flushed when one memtable was filled, resulting in inconsistency after a crash.
[Unreleased]: https://github.com/nomic-io/merk/compare/v1.0.0-alpha.8...HEAD
================================================
FILE: Cargo.toml
================================================
[package]
name = "merk"
description = "High-performance Merkle key/value store"
version = "2.0.0"
authors = ["Turbofish <team@turbofish.org>"]
edition = "2018"
license = "Apache-2.0"
[dependencies]
thiserror= "1.0.58"
sha2 = "0.10.8"
log = "0.4.21"
[dependencies.colored]
version = "2.1.0"
optional = true
[dependencies.num_cpus]
version = "1.16.0"
optional = true
[dependencies.ed]
version = "0.3.0"
optional = true
[dependencies.rand]
version = "0.8.5"
features = ["small_rng"]
optional = true
[dependencies.rocksdb]
version = "0.22.0"
default-features = false
optional = true
[dependencies.jemallocator]
version = "0.5.4"
features = ["disable_initial_exec_tls"]
optional = true
[features]
default = ["full", "verify"]
full = [
"rand",
"rocksdb",
"colored",
"num_cpus",
"ed",
]
verify = ["ed"]
================================================
FILE: LICENSE
================================================
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: README.md
================================================
<h1 align="left">
<picture>
<source media="(prefers-color-scheme: dark)" srcset="./merk-dark.svg">
<source media="(prefers-color-scheme: light)" srcset="./merk.svg">
<img alt="merk" src="./merk.svg">
</picture>
</h1>
*High-performance Merkle key/value store*
[](https://codecov.io/gh/turbofish-org/merk)
[](https://crates.io/crates/merk)
[](https://docs.rs/merk)
Merk is a crypto key/value store - more specifically, it's a Merkle AVL tree built on top of RocksDB (Facebook's fork of LevelDB).
Its priorities are performance and reliability. While Merk was designed to be the state database for blockchains, it can also be used anywhere an auditable key/value store is needed.
### Features
- **Fast reads/writes** - Reads have no overhead compared to a normal RocksDB store, and writes are optimized for batch operations (e.g. blocks in a blockchain).
- **Fast proof generation** - Since Merk implements an AVL tree rather than a trie, it is very efficient to create and verify proofs for ranges of keys.
- **Concurrency** - Unlike most other Merkle stores, all operations utilize all available cores - giving huge performance gains and allowing nodes to scale along with Moore's Law.
- **Replication** - The tree is optimized to efficiently build proofs of large chunks, allowing for nodes to download the entire state (e.g. "state syncing").
- **Checkpointing** - Merk can create checkpoints on disk (an immutable view of the entire store at a certain point in time) without blocking, so there are no delays in availability or liveness.
- **Web-friendly** - Being written in Rust means it is easy to run the proof-verification code in browsers with WebAssembly, allowing for light-clients that can verify data for themselves.
- **Fits any Profile** - Performant on RAM-constrained Raspberry Pi's and beefy validator rigs alike.
The algorithms are based on AVL, but optimized for batches of operations and random fetches from the backing store.
## Usage
**Install:**
```
cargo add merk
```
**Example:**
```rust
extern crate merk;
use merk::*;
// load or create a Merk store at the given path
let mut merk = Merk::open("./merk.db").unwrap();
// apply some operations
let batch = [
(b"key", Op::Put(b"value")),
(b"key2", Op::Put(b"value2")),
(b"key3", Op::Put(b"value3")),
(b"key4", Op::Delete)
];
merk.apply(&batch).unwrap();
```
Merk is currently used by [Nomic](https://github.com/nomic-io/nomic), a blockchain powering decentralized custody of Bitcoin, built on [Orga](https://github.com/turbofish-org/orga).
## Benchmarks
Benchmarks are measured on a 1M node tree, each node having a key length of 16 bytes and value length of 40 bytes. All tests are single-threaded (not counting RocksDB background threads).
You can test these yourself by running `cargo bench`.
### 2017 Macbook Pro
*(Using 1 Merk thread and 4 RocksDB compaction threads)*
**Pruned (no state kept in memory)**
*RAM usage:* ~20MB average, ~26MB max
| Test | Ops per second |
| -------- | ------ |
| Random inserts | 23,000 |
| Random updates | 32,000 |
| Random deletes | 26,000 |
| Random reads | 210,000 |
| Random proof generation | 133,000 |
**Cached (all state kept in memory)**
*RAM usage:* ~400MB average, ~1.1GB max
| Test | Ops per second |
| -------- | ------ |
| Random inserts | 58,000 |
| Random updates | 81,000 |
| Random deletes | 72,000 |
| Random reads | 1,565,000 |
| Random proof generation | 311,000 |
### i9-9900K Desktop
*(Using 1 Merk thread and 16 RocksDB compaction threads)*
**Pruned (no state kept in memory)**
*RAM usage:* ~20MB average, ~26MB max
| Test | Ops per second |
| -------- | ------ |
| Random inserts | 40,000 |
| Random updates | 55,000 |
| Random deletes | 45,000 |
| Random reads | 383,000 |
| Random proof generation | 249,000 |
**Cached (all state kept in memory)**
*RAM usage:* ~400MB average, ~1.1GB max
| Test | Ops per second |
| -------- | ------ |
| Random inserts | 93,000 |
| Random updates | 123,000 |
| Random deletes | 111,000 |
| Random reads | 2,370,000 |
| Random proof generation | 497,000 |
## Contributing
Merk is an open-source project spearheaded by Turbofish. Anyone is able to contribute to Merk via GitHub.
[Contribute to Merk](https://github.com/turbofish-org/merk/contribute)
## Security
### Security Audits
| Date | Auditor | Scope | Report |
| ---: | :---: | :--- | :---: |
| October 2024 | Trail of Bits | `orga` `merk` `ed` `abci2` | [📄](https://github.com/trailofbits/publications/blob/master/reviews/2024-11-orgaandmerk-securityreview.pdf) |
Vulnerabilities should not be reported through public channels, including GitHub Issues. You can report a vulnerability via GitHub's Private Vulnerability Reporting or to Turbofish at `security@turbofish.org`.
[Report a Vulnerability](https://github.com/turbofish-org/merk/security/advisories/new)
## License
Licensed under the Apache License, Version 2.0 (the "License"); you may not use the files in this repository except in compliance with the License. You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
---
Copyright © 2024 Turbofish, Inc.
================================================
FILE: benches/merk.rs
================================================
#![feature(test)]
extern crate test;
use merk::proofs::encode_into as encode_proof_into;
use merk::restore::Restorer;
use merk::test_utils::*;
use merk::{Merk, Result};
use rand::prelude::*;
use std::thread;
use test::Bencher;
#[bench]
fn get_1m_rocksdb(b: &mut Bencher) {
let initial_size = 1_000_000;
let batch_size = 2_000;
let num_batches = initial_size / batch_size;
let path = thread::current().name().unwrap().to_owned();
let mut merk = TempMerk::open(path).expect("failed to open merk");
let mut batches = vec![];
for i in 0..num_batches {
let batch = make_batch_rand(batch_size, i);
unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
batches.push(batch);
}
let mut i = 0;
b.iter(|| {
let batch_index = (i % num_batches) as usize;
let key_index = (i / num_batches) as usize;
let key = &batches[batch_index][key_index].0;
merk.get(key).expect("get failed");
i = (i + 1) % initial_size;
});
}
#[bench]
fn insert_1m_2k_seq_rocksdb_noprune(b: &mut Bencher) {
let initial_size = 1_000_000;
let batch_size = 2_000;
let path = thread::current().name().unwrap().to_owned();
let mut merk = TempMerk::open(path).expect("failed to open merk");
for i in 0..(initial_size / batch_size) {
let batch = make_batch_seq((i * batch_size)..((i + 1) * batch_size));
unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
}
let mut i = initial_size / batch_size;
b.iter(|| {
let batch = make_batch_seq((i * batch_size)..((i + 1) * batch_size));
unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
i += 1;
});
}
#[bench]
fn insert_1m_2k_rand_rocksdb_noprune(b: &mut Bencher) {
let initial_size = 1_000_000;
let batch_size = 2_000;
let path = thread::current().name().unwrap().to_owned();
let mut merk = TempMerk::open(path).expect("failed to open merk");
for i in 0..(initial_size / batch_size) {
let batch = make_batch_rand(batch_size, i);
unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
}
let mut i = initial_size / batch_size;
b.iter(|| {
let batch = make_batch_rand(batch_size, i);
unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
i += 1;
});
}
#[bench]
fn update_1m_2k_seq_rocksdb_noprune(b: &mut Bencher) {
let initial_size = 1_000_000;
let batch_size = 2_000;
let path = thread::current().name().unwrap().to_owned();
let mut merk = TempMerk::open(path).expect("failed to open merk");
for i in 0..(initial_size / batch_size) {
let batch = make_batch_seq((i * batch_size)..((i + 1) * batch_size));
unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
}
let mut i = 0;
b.iter(|| {
let batch = make_batch_seq((i * batch_size)..((i + 1) * batch_size));
unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
i = (i + 1) % (initial_size / batch_size);
});
}
#[bench]
fn update_1m_2k_rand_rocksdb_noprune(b: &mut Bencher) {
let initial_size = 1_000_000;
let batch_size = 2_000;
let path = thread::current().name().unwrap().to_owned();
let mut merk = TempMerk::open(path).expect("failed to open merk");
for i in 0..(initial_size / batch_size) {
let batch = make_batch_rand(batch_size, i);
unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
}
let mut i = 0;
b.iter(|| {
let batch = make_batch_rand(batch_size, i);
unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
i = (i + 1) % (initial_size / batch_size);
});
}
#[bench]
fn delete_1m_2k_rand_rocksdb_noprune(b: &mut Bencher) {
let initial_size = 1_000_000;
let batch_size = 2_000;
let path = thread::current().name().unwrap().to_owned();
let mut merk = TempMerk::open(path).expect("failed to open merk");
for i in 0..(initial_size / batch_size) {
let batch = make_batch_rand(batch_size, i);
unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
}
let mut i = 0;
b.iter(|| {
if i >= (initial_size / batch_size) {
println!("WARNING: too many bench iterations, whole tree deleted");
return;
}
let batch = make_del_batch_rand(batch_size, i);
unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
i = (i + 1) % (initial_size / batch_size);
});
}
#[bench]
fn prove_1m_1_rand_rocksdb_noprune(b: &mut Bencher) {
let initial_size = 1_000_000;
let batch_size = 1_000;
let proof_size = 1;
let path = thread::current().name().unwrap().to_owned();
let mut merk = TempMerk::open(path).expect("failed to open merk");
for i in 0..(initial_size / batch_size) {
let batch = make_batch_rand(batch_size, i);
unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
}
let mut i = 0;
b.iter(|| {
let batch = make_batch_rand(proof_size, i);
let mut keys = Vec::with_capacity(batch.len());
for (key, _) in batch {
keys.push(merk::proofs::query::QueryItem::Key(key));
}
merk.prove(keys).expect("prove failed");
i = (i + 1) % (initial_size / batch_size);
merk.commit(std::collections::LinkedList::new(), &[])
.unwrap();
});
}
#[bench]
fn build_trunk_chunk_1m_1_rand_rocksdb_noprune(b: &mut Bencher) {
let initial_size = 1_000_000;
let batch_size = 1_000;
let path = thread::current().name().unwrap().to_owned();
let mut merk = TempMerk::open(path).expect("failed to open merk");
for i in 0..(initial_size / batch_size) {
let batch = make_batch_rand(batch_size, i);
unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
}
let mut bytes = vec![];
b.iter(|| {
bytes.clear();
let (ops, _) = merk.walk(|walker| walker.unwrap().create_trunk_proof().unwrap());
encode_proof_into(ops.iter(), &mut bytes);
merk.commit(std::collections::LinkedList::new(), &[])
.unwrap();
});
b.bytes = bytes.len() as u64;
}
#[bench]
fn chunkproducer_rand_1m_1_rand_rocksdb_noprune(b: &mut Bencher) {
let mut rng = rand::thread_rng();
let initial_size = 1_000_000;
let batch_size = 1_000;
let path = thread::current().name().unwrap().to_owned();
let mut merk = TempMerk::open(path).expect("failed to open merk");
for i in 0..(initial_size / batch_size) {
let batch = make_batch_rand(batch_size, i);
unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
}
let mut chunks = merk.chunks().unwrap();
let mut total_bytes = 0;
let mut i = 0;
let mut next = || {
let index = rng.gen::<usize>() % chunks.len();
chunks.chunk(index).unwrap()
};
b.iter(|| {
let chunk = next();
total_bytes += chunk.len();
i += 1;
});
b.bytes = (total_bytes / i) as u64;
}
#[bench]
fn chunk_iter_1m_1_rand_rocksdb_noprune(b: &mut Bencher) {
let initial_size = 1_000_000;
let batch_size = 1_000;
let path = thread::current().name().unwrap().to_owned();
let mut merk = TempMerk::open(path).expect("failed to open merk");
for i in 0..(initial_size / batch_size) {
let batch = make_batch_rand(batch_size, i);
unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
}
let mut chunks = merk.chunks().unwrap().into_iter();
let mut total_bytes = 0;
let mut i = 0;
let mut next = || match chunks.next() {
Some(chunk) => chunk,
None => {
chunks = merk.chunks().unwrap().into_iter();
chunks.next().unwrap()
}
};
b.iter(|| {
let chunk = next();
total_bytes += chunk.unwrap().len();
i += 1;
});
b.bytes = (total_bytes / i) as u64;
}
#[bench]
fn restore_1m_1_rand_rocksdb_noprune(b: &mut Bencher) {
let initial_size = 1_000_000;
let batch_size = 1_000;
let path = thread::current().name().unwrap().to_owned();
let mut merk = TempMerk::open(path).expect("failed to open merk");
for i in 0..(initial_size / batch_size) {
let batch = make_batch_rand(batch_size, i);
unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
}
let chunks = merk
.chunks()
.unwrap()
.into_iter()
.collect::<Result<Vec<_>>>()
.unwrap();
let path = thread::current().name().unwrap().to_owned() + "_restore";
let mut restorer: Option<Restorer> = None;
let mut total_bytes = 0;
let mut i = 0;
b.iter(|| {
if i % chunks.len() == 0 {
if i != 0 {
let restorer_merk = restorer.take().unwrap().finalize();
drop(restorer_merk);
std::fs::remove_dir_all(&path).unwrap();
}
restorer = Some(Merk::restore(&path, merk.root_hash(), chunks.len()).unwrap());
}
let restorer = restorer.as_mut().unwrap();
let chunk = chunks[i % chunks.len()].as_slice();
restorer.process_chunk(chunk).unwrap();
total_bytes += chunk.len();
i += 1;
});
std::fs::remove_dir_all(&path).unwrap();
b.bytes = (total_bytes / i) as u64;
}
#[bench]
fn checkpoint_create_destroy_1m_1_rand_rocksdb_noprune(b: &mut Bencher) {
let initial_size = 1_000_000;
let batch_size = 1_000;
let path = thread::current().name().unwrap().to_owned();
let mut merk = TempMerk::open(&path).expect("failed to open merk");
for i in 0..(initial_size / batch_size) {
let batch = make_batch_rand(batch_size, i);
unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
}
let path = path + ".checkpoint";
b.iter(|| {
let checkpoint = merk.checkpoint(&path).unwrap();
checkpoint.destroy().unwrap();
});
}
================================================
FILE: benches/ops.rs
================================================
#![feature(test)]
extern crate test;
use merk::owner::Owner;
use merk::test_utils::*;
use test::Bencher;
#[bench]
fn insert_1m_10k_seq_memonly(b: &mut Bencher) {
let initial_size = 1_000_000;
let batch_size = 10_000;
let mut tree = Owner::new(make_tree_seq(initial_size));
let mut i = initial_size / batch_size;
b.iter(|| {
let batch = make_batch_seq((i * batch_size)..((i + 1) * batch_size));
tree.own(|tree| apply_memonly_unchecked(tree, &batch));
i += 1;
});
}
#[bench]
fn insert_1m_10k_rand_memonly(b: &mut Bencher) {
let initial_size = 1_000_000;
let batch_size = 10_000;
let mut tree = Owner::new(make_tree_rand(initial_size, batch_size, 0));
let mut i = initial_size / batch_size;
b.iter(|| {
let batch = make_batch_rand(batch_size, i);
tree.own(|tree| apply_memonly_unchecked(tree, &batch));
i += 1;
});
}
#[bench]
fn update_1m_10k_seq_memonly(b: &mut Bencher) {
let initial_size = 1_000_000;
let batch_size = 10_000;
let mut tree = Owner::new(make_tree_seq(initial_size));
let mut i = 0;
b.iter(|| {
let batch = make_batch_seq((i * batch_size)..((i + 1) * batch_size));
tree.own(|tree| apply_memonly_unchecked(tree, &batch));
i = (i + 1) % (initial_size / batch_size);
});
}
#[bench]
fn update_1m_10k_rand_memonly(b: &mut Bencher) {
let initial_size = 1_010_000;
let batch_size = 10_000;
let mut tree = Owner::new(make_tree_rand(initial_size, batch_size, 0));
let mut i = 0;
b.iter(|| {
let batch = make_batch_rand(batch_size, i);
tree.own(|tree| apply_memonly_unchecked(tree, &batch));
i = (i + 1) % (initial_size / batch_size);
});
}
================================================
FILE: docs/algorithms.md
================================================
# Merk - A High-Performance Merkle AVL Tree
**Matt Bell ([@mappum](https://twitter.com/mappum))** • [Nomic Hodlings, Inc.](https://nomic.io)
v0.0.4 - _August 5, 2020_
## Introduction
Merk is a Merkle AVL tree designed for performance, running on top of a backing key/value store such as RocksDB. Notable features include concurrent operations for higher throughput, an optimized key/value layout for performant usage of the backing store, and efficient proof generation to enable bulk tree replication.
_Note that this document is meant to be a way to grok how Merk works, rather than an authoritative specification._
## Algorithm Overview
The Merk tree was inspired by [`tendermint/iavl`](https://github.com/tendermint/iavl) from the [Tendermint](https://tendermint.com) team but makes various fundamental design changes in the name of performance.
### Tree Structure
#### Nodes and Hashing
In many Merkle tree designs, only leaf nodes contain key/value pairs (inner nodes only contain child hashes). To contrast, every node in a Merk tree contains a key and a value, including inner nodes.
Each node contains a "kv hash", which is the hash of its key/value pair, in addition to its child hashes. The hash of the node is just the hash of the concatenation of these three hashes:
```
kv_hash = H(key, value)
node_hash = H(kv_hash, left_child_hash, right_child_hash)
```
Note that the `left_child_hash` and/or `right_child_hash` values may be null since it is possible for the node to have no children or only one child.
In our implementation, the hash function used is SHA512/256 (SHA512 with output truncated to 256 bits) but this choice is trivially swappable.
#### Database Representation
In the backing key/value store, nodes are stored using their key/value pair key as the database key, and a binary encoding that contains the fields in the above `Node` structure - minus the `key` field since that is already implied by the database entry.
Storing nodes by key rather than by hash is an important optimization, and is the reason why inner nodes each have a key/value pair. The implication is that reading a key does not require traversing through the tree structure but only requires a single read in the backing key/value store, meaning there is practically no overhead versus using the backing store without a tree structure. Additionally, we can efficiently iterate through nodes in the tree in their in-order traversal just by iterating by key in the backing store (which RocksDB and LevelDB are optimized for).
This means we lose the "I" compared to the IAVL library - immutability. Since now we operate on the tree nodes in-place in the backing store, we don't by default have views of past states of the tree. However, **in** our implementation we replicate this functionality with RocksDB's snapshot and checkpoint features which provide a consistent view of the store at a certain point in history - either ephemerally in memory or persistently on disk.
### Operations
Operating on a Merk tree is optimized for batches - in the real world we will only be updating the tree once per block, applying a batch of many changes from many transactions at the same time.
#### Concurrent Batch Operator
To mutate the tree, we apply batches of operations, each of which can either be `Put(key, value)` or `Delete(key)`.
Batches of operations are expected to be sorted by key, with every key appearing only once. Our implementation provides an `apply` method which sorts the batch and checks for duplicate keys, and an `apply_unchecked` method which skips the sorting/checking step for performance reasons when the caller has already ensured the batch is sorted.
The algorithm to apply these operations to the tree is called recursively on each relevant node.
_Simplified pseudocode for the operation algorithm:_
- Given a node and a batch of operations:
- Binary search for the current node's key in the batch:
- If this node's key is found in the batch at index `i`:
- Apply the operation to this node:
- If operation is `Put`, update its `value` and `kv_hash`
- If the operation is `Delete`, perform a traditional BST node removal
- Split the batch into left and right sub-batches (excluding the operation we just applied):
- Left batch from batch start to index `i`
- Right batch from index `i + 1` to the end of the batch
- If this node's key is not found in the batch, but could be inserted at index `i` maintaining sorted order:
- Split the batch into left and right sub-batches:
- Left batch from batch start to index `i`
- Right batch from index `i` to the end of the batch
- Recurse:
- Apply the left sub-batch to this node's left child
- Apply the right sub-batch to this node's right child
- Balance:
- If after recursing the left and right subtrees are unbalanced (their heights differ by more than 1), perform an AVL tree rotation (possibly more than one)
- Recompute node's hash based on hash of its updated children and `kv_hash`, then return
This batch application of operations can happen concurrently - recursing into the left and right subtrees of a node are two fully independent operations (operations on one subtree will never involve reading or writing to/from any of the nodes on the other subtree). This means we have an _implicit lock_ - we don't need to coordinate with mutexes but only need to wait for both the left side and right side to finish their operations.
### Proofs
Merk was designed with efficient proofs in mind, both for application queries (e.g. a user checking their account balance) and bulk tree replication (a.k.a. "state syncing") between untrusted nodes.
#### Structure
Merk proofs are a list of stack-based operators and node data, with 3 possible operators: `Push(node)`, `Parent`, and `Child`. A stream of these operators can be processed by a verifier in order to reconstruct a sparse representation of part of the tree, in a way where the data can be verified against a known root hash.
The value of `node` in a `Push` operation can be one of three types:
- `Hash(hash)` - The hash of a node
- `KVHash(hash)` - The key/value hash of a node
- `KV(key, value)` - The key and value of a node
This proof format can be encoded in a binary format and has negligible space overhead for efficient transport over the network.
#### Verification
A verifier can process a proof by maintaining a stack of connected tree nodes, and executing the operators in order:
- `Push(node)` - Push some node data onto the stack.
- `Child` - Pop a value from the stack, `child`. Pop another value from the stack, `parent`. Set `child` as the right child of `parent`, and push the combined result back on the stack.
- `Parent` - Pop a value from the stack, `parent`. Pop another value from the stack, `child`. Set `child` as the left child of `parent`, and push the combined result back on the stack.
Proof verification will fail if e.g. `Child` or `Parent` try to pop a value from the stack but the stack is empty, `Child` or `Parent` try to overwrite an existing child, or the proof does not result in exactly one stack item.
This proof language can be used to specify any possible set or subset of the tree's data in a way that can be reconstructed efficiently by the verifier. Proofs can contain either an arbitrary set of selected key/value pairs (e.g. in an application query), or contiguous tree chunks (when replicating the tree). After processing an entire proof, the verifier should have derived a root hash which can be compared to the root hash they expect (e.g. the one validators committed to in consensus), and have a set of proven key/value pairs.
Note that this can be computed in a streaming fashion, e.g. while downloading the proof, which makes the required memory for verification very low even for large proofs. However, the verifier cannot tell if the proof is valid until finishing the entire proof, so very large proofs should be broken up into multiple proofs of smaller size.
#### Generation
Efficient proof generation is important since nodes will likely receive a high volume of queries and constantly be serving proofs, essentially providing an API service to end-user application clients, as well as servicing demand for replication when new nodes come onto the network.
Nodes can generate proofs for a set of keys by traversing through the tree from the root and building up the required proof branches. Much like the batch operator aglorithm, this algorithm takes a batch of sorted, unique keys as input.
_Simplified pseudocode for proof generation (based on an in-order traversal):_
- Given a node and a batch of keys to include in the proof:
- If the batch is empty, append `Push(Hash(node_hash))` to the proof and return
- Binary search the for the current node's key in the batch:
- If this node's key is found in the batch at index `i`:
- Partition the batch into left and right sub-batches at index `i` (excluding index `i`)
- If this node's key is not found in the batch, but could be inserted at index `i` maintaining sorted order:
- Partition the batch into left and right sub-batches at index `i`
- **Recurse left:** If there is a left child:
- If the left sub-batch is not empty, query the left child (appending operators to the proof)
- If the left sub-batch is empty, append `Push(Hash(left_child_hash))` to the proof
- Append proof operator:
- If this node's key is in the batch, or if the left sub-batch was not empty and no left child exists, or if the right sub-batch is not empty and no right child exists,or if the left child's right edge queried a non-existent key, or if the right child's left edge queried a non-existent key, append `Push(KV(key, value))` to the proof
- Otherwise, append `Push(KVHash(kv_hash))` to the proof
- If the left child exists, append `Parent` to the proof
- **Recurse right:** If there is a right child:
- If the right sub-batch is not empty, query the right child (appending operators to the proof)
- If the right sub-batch is empty, append `Push(Hash(left_child_hash))` to the proof
- Append `Child` to the proof
Since RocksDB allows concurrent reading from a consistent snapshot/checkpoint, nodes can concurrently generate proofs on all cores to service a higher volume of queries, even if our algorithm isn't designed for concurrency.
#### Binary Format
We can efficiently encode these proofs by encoding each operator as follows:
```
Push(Hash(hash)) => 0x01 <20-byte hash>
Push(KVHash(hash)) => 0x02 <20-byte hash>
Push(KV(key, value)) => 0x03 <1-byte key length> <n-byte key> <2-byte value length> <n-byte value>
Parent => 0x10
Child => 0x11
```
This results in a compact binary representation, with a very small space overhead (roughly 2 bytes per node in the proof (1 byte for the `Push` operator type flag, and 1 byte for a `Parent` or `Child` operator), plus 3 bytes per key/value pair (1 byte for the key length, and 2 bytes for the value length)).
#### Efficient Chunk Proofs for Replication
An alternate, optimized proof generation can be used when generating proofs for large contiguous subtrees, e.g. chunks for tree replication. This works by iterating sequentially through keys in the backing store (which is much faster than random lookups).
Based on some early benchmarks, I estimate that typical server hardware should be able to generate this kind of range proof at a rate of hundreds of MB/s, which means the bottleneck for bulk replication will likely be bandwidth rather than CPU. To improve performance further, these proofs can be cached and trivially served by a CDN or a P2P swarm (each node of which can easily verify the chunks they pass around).
Due to the tree structure we already use, streaming the entries in key-order gives us all the nodes to construct complete contiguous subtrees. For instance, in the diagram below, streaming from keys `1` to `7` will give us a complete subtree. This subtree can be verified to be a part of the full tree as long as we know the hash of `4`.
```
8
/ \
/ ...
4
/ \
2 6
/ \ / \
1 3 5 7
```
Our algorithm builds verifiable chunks by first constructing a chunk of the upper levels of the tree, called the _trunk chunk_, plus each subtree below that (each of which is called a _leaf chunk_).
The number of levels to include in the trunk can be chosen to control the size of the leaf nodes. For example, a tree of height 10 should have approximately 1,023 nodes. If the trunk contains the top 5 levels, the trunk and the 32 resulting leaf nodes will each contain ~31 nodes. We can even prove to the verifier the trunk size was chosen correctly by also including an approximate tree height proof, by including the branch all the way to the leftmost node of the tree (node `1` in the figure) and using this height as our basis to select the number of trunk levels.
After the prover builds the trunk by traversing from the root node and making random lookups down to the chosen level, it can generate the leaf nodes extremely efficiently by reading the database keys sequentially as described a few paragraphs above. We can trivially detect when a chunk should end whenever a node at or above the trunk level is encountered (e.g. encountering node `8` signals we have read a complete subtree).
The generated proofs can be efficiently encoded into the same proof format described above. Verifiers only have the added constraint that none of the data should be abbridged (all nodes contain a key and value, rather than just a hash or kvhash). After first downloading and verifying the trunk, verifiers can also download leaf chunks in parallel and verify that each connects to the trunk by comparing each subtree's root hash.
Note that this algorithm produces proofs with very little memory requirements, plus little overhead added to the sequential read from disk. In a proof-of-concept benchmark, proof generation was measured to be ~750 MB/s on a modern solid-state drive and processor, meaning a 4GB state tree (the size of the Cosmos Hub state at the time of writing) could be fully proven in ~5 seconds (without considering parallelization). In conjunction with the RocksDB checkpoint feature, this process can happen in the background without blocking the node from executing later blocks.
_Pseudocode for the range proof generation algorithm:_
- Given a tree and a range of keys to prove:
- Create a stack of keys (initially empty)
- **Range iteration:** for every key/value entry within the query range in the backing store:
- Append `Push(KV(key, value))` to the proof
- If the current node has a left child, append `Parent` to the proof
- If the current node has a right child, push the right child's key onto the key stack
- If the current node does not have a right child:
- While the current node's key is greater than or equal to the key at the top of the key stack, append `Child` to the proof and pop from the key stack
Note that this algorithm produces the proof in a streaming fashion and has very little memory requirements (the only overhead is the key stack, which will be small even for extremely large trees since its length is a maximum of `log N`).
#### Example Proofs
Let's walk through a concrete proof example. Consider the following tree:
```
5
/ \
/ \
2 9
/ \ / \
1 4 7 11
/ / \ /
3 6 8 10
```
_Small proof:_
First, let's create a proof for a small part of the tree. Let's say the user makes a query for keys `1, 2, 3, 4`.
If we follow our proof generation algorithm, we should get a proof that looks like this:
```
Push(KV(1, <value of 1>)),
Push(KV(2, <value of 2>)),
Parent,
Push(KV(3, <value of 3>)),
Push(KV(4, <value of 4>)),
Parent,
Child,
Push(KVHash(<kv_hash of 5>)),
Parent,
Push(Hash(<hash of 9>)),
Child
```
Let's step through verification to show that this proof works. We'll create a verification stack, which starts out empty, and walk through each operator in the proof, in order:
```
Stack: (empty)
```
We will push a key/value pair on the stack, creating a node. However, note that for verification purposes this node will only need to contain the kv_hash which we will compute at this step.
```
Operator: Push(KV(1, <value of 1>))
Stack:
1
```
```
Operator: Push(KV(2, <value of 2>))
Stack:
1
2
```
Now we connect nodes 1 and 2, with 2 as the parent.
```
Operator: Parent
Stack:
2
/
1
```
```
Operator: Push(KV(3, <value of 3>))
Stack:
2
/
1
3
```
```
Operator: Push(KV(4, <value of 4>))
Stack:
2
/
1
3
4
```
```
Operator: Parent
Stack:
2
/
1
4
/
3
```
Now connect these two graphs with 4 as the child of 2.
```
Operator: Child
Stack:
2
/ \
1 4
/
3
```
Since the user isn't querying the data from node 5, we only need its kv_hash.
```
Operator: Push(KVHash(<kv_hash of 5>))
Stack:
2
/ \
1 4
/
3
5
```
```
Operator: Parent
Stack:
5
/
2
/ \
1 4
/
3
```
We only need the hash of node 9.
```
Operator: Push(Hash(<hash of 9>))
Stack:
5
/
2
/ \
1 4
/
3
9
```
```
Operator: Child
Stack:
5
/ \
2 9
/ \
1 4
/
3
```
Now after going through all these steps, we have sufficient knowlege of the tree's structure and data to compute node hashes in order to verify. At the end, we will have computed a hash for node 5 (the root), and we verify by comparing this hash to the one we expected.
================================================
FILE: rustfmt.toml
================================================
comment_width = 80
wrap_comments = true
================================================
FILE: scripts/pgo.sh
================================================
#!/bin/bash
default_host_triple=""
default_toolchain=""
IFS=" = "
while read -r name value
do
value="${value//\"/}"
if [ "${name}" == "default_host_triple" ]; then
default_host_triple="${value}"
elif [ "${name}" == "default_toolchain" ]; then
default_toolchain="${value}"
fi
done < ~/.rustup/settings.toml
echo "default_host_triple=${default_host_triple}"
echo "default_toolchain=${default_toolchain}"
rustup component add llvm-tools-preview
rm -rf /tmp/merk-pgo
RUSTFLAGS="-Cprofile-generate=/tmp/merk-pgo" cargo bench rand_rocks
~/.rustup/toolchains/${default_toolchain}/lib/rustlib/${default_host_triple}/bin/llvm-profdata merge -o /tmp/merk-pgo/merged.profdata /tmp/merk-pgo
RUSTFLAGS="-Cprofile-use=/tmp/merk-pgo/merged.profdata" cargo bench
================================================
FILE: src/error.rs
================================================
pub use thiserror::Error;
#[derive(Error, Debug)]
pub enum Error {
#[error("Attach Error: {0}")]
Attach(String),
#[error("Batch Key Error: {0}")]
BatchKey(String),
#[error("Bound Error: {0}")]
Bound(String),
#[error("Chunk Processing Error: {0}")]
ChunkProcessing(String),
#[error(transparent)]
Ed(#[from] ed::Error),
#[error("Fetch Error: {0}")]
Fetch(String),
#[error("Proof did not match expected hash\n\tExpected: {0:?}\n\tActual: {1:?}")]
HashMismatch([u8; 32], [u8; 32]),
#[error("Index OoB Error: {0}")]
IndexOutOfBounds(String),
#[error("Integer conversion error: {0}")]
IntegerConversionError(#[from] std::num::TryFromIntError),
#[error(transparent)]
IO(#[from] std::io::Error),
#[error("Tried to delete non-existent key {0:?}")]
KeyDelete(Vec<u8>),
#[error("Key Error: {0}")]
Key(String),
#[error("Key not found: {0}")]
KeyNotFound(String),
#[error("Proof is missing data for query")]
MissingData,
#[error("Path Error: {0}")]
Path(String),
#[error("Proof Error: {0}")]
Proof(String),
#[cfg(feature = "full")]
#[error(transparent)]
RocksDB(#[from] rocksdb::Error),
#[error("Stack Underflow")]
StackUnderflow,
#[error("Tree Error: {0}")]
Tree(String),
#[error("Unexpected Node Error: {0}")]
UnexpectedNode(String),
#[error("Unknown Error")]
Unknown,
#[error("Version Error: {0}")]
Version(String),
}
pub type Result<T> = std::result::Result<T, Error>;
================================================
FILE: src/lib.rs
================================================
//! A high-performance Merkle key/value store.
//!
//! Merk is a crypto key/value store - more specifically, it's a Merkle AVL tree
//! built on top of RocksDB (Facebook's fork of LevelDB).
//!
//! Its priorities are performance and reliability. While Merk was designed to
//! be the state database for blockchains, it can also be used anywhere an
//! auditable key/value store is needed.
#![feature(trivial_bounds)]
#[global_allocator]
#[cfg(feature = "jemallocator")]
static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
#[cfg(feature = "full")]
pub use rocksdb;
/// Error and Result types.
mod error;
/// The top-level store API.
#[cfg(feature = "full")]
mod merk;
/// Provides a container type that allows temporarily taking ownership of a
/// value.
// TODO: move this into its own crate
pub mod owner;
/// Algorithms for generating and verifying Merkle proofs.
pub mod proofs;
/// Various helpers useful for tests or benchmarks.
#[cfg(feature = "full")]
pub mod test_utils;
/// The core tree data structure.
pub mod tree;
#[cfg(feature = "full")]
pub use crate::merk::{chunks, restore, snapshot, Merk, MerkSource, Snapshot};
pub use error::{Error, Result};
pub use tree::{Batch, BatchEntry, Hash, Op, PanicSource, HASH_LENGTH};
#[allow(deprecated)]
pub use proofs::query::verify_query;
pub use proofs::query::verify;
================================================
FILE: src/merk/chunks.rs
================================================
//! Provides `ChunkProducer`, which creates chunk proofs for full replication of
//! a Merk.
use super::Merk;
use crate::proofs::{chunk::get_next_chunk, Node, Op};
use crate::{Error, Result};
use ed::Encode;
use rocksdb::DBRawIterator;
/// A `ChunkProducer` allows the creation of chunk proofs, used for trustlessly
/// replicating entire Merk trees.
///
/// Chunks can be generated on the fly in a random order, or iterated in order
/// for slightly better performance.
pub struct ChunkProducer<'a> {
trunk: Vec<Op>,
chunk_boundaries: Vec<Vec<u8>>,
raw_iter: DBRawIterator<'a>,
index: usize,
}
impl<'a> ChunkProducer<'a> {
/// Creates a new `ChunkProducer` for the given `Merk` instance. In the
/// constructor, the first chunk (the "trunk") will be created.
pub fn new(merk: &'a Merk) -> Result<Self> {
let (trunk, has_more) = merk.walk(|maybe_walker| match maybe_walker {
Some(mut walker) => walker.create_trunk_proof(),
None => Ok((vec![], false)),
})?;
let chunk_boundaries = if has_more {
trunk
.iter()
.filter_map(|op| match op {
Op::Push(Node::KV(key, _)) => Some(key.clone()),
_ => None,
})
.collect()
} else {
vec![]
};
let mut raw_iter = merk.raw_iter();
raw_iter.seek_to_first();
Ok(ChunkProducer {
trunk,
chunk_boundaries,
raw_iter,
index: 0,
})
}
/// Gets the chunk with the given index. Errors if the index is out of
/// bounds or the tree is empty - the number of chunks can be checked by
/// calling `producer.len()`.
pub fn chunk(&mut self, index: usize) -> Result<Vec<u8>> {
if index >= self.len() {
return Err(Error::IndexOutOfBounds("Chunk index out-of-bounds".into()));
}
self.index = index;
if index == 0 || index == 1 {
self.raw_iter.seek_to_first();
} else {
let preceding_key = self.chunk_boundaries.get(index - 2).unwrap();
self.raw_iter.seek(preceding_key);
self.raw_iter.next();
}
self.next_chunk()
}
/// Returns the total number of chunks for the underlying Merk tree.
#[allow(clippy::len_without_is_empty)]
pub fn len(&self) -> usize {
let boundaries_len = self.chunk_boundaries.len();
if boundaries_len == 0 {
1
} else {
boundaries_len + 2
}
}
/// Gets the next chunk based on the `ChunkProducer`'s internal index state.
/// This is mostly useful for letting `ChunkIter` yield the chunks in order,
/// optimizing throughput compared to random access.
fn next_chunk(&mut self) -> Result<Vec<u8>> {
if self.index == 0 {
if self.trunk.is_empty() {
return Err(Error::Fetch(
"Attempted to fetch chunk on empty tree".into(),
));
}
self.index += 1;
return Ok(self.trunk.encode()?);
}
assert!(self.index < self.len(), "Called next_chunk after end");
let end_key = self.chunk_boundaries.get(self.index - 1);
let end_key_slice = end_key.as_ref().map(|k| k.as_slice());
self.index += 1;
let chunk = get_next_chunk(&mut self.raw_iter, end_key_slice)?;
Ok(chunk.encode()?)
}
}
impl<'a> IntoIterator for ChunkProducer<'a> {
type IntoIter = ChunkIter<'a>;
type Item = <ChunkIter<'a> as Iterator>::Item;
fn into_iter(self) -> Self::IntoIter {
ChunkIter(self)
}
}
/// A `ChunkIter` iterates through all the chunks for the underlying `Merk`
/// instance in order (the first chunk is the "trunk" chunk). Yields `None`
/// after all chunks have been yielded.
pub struct ChunkIter<'a>(ChunkProducer<'a>);
impl<'a> Iterator for ChunkIter<'a> {
type Item = Result<Vec<u8>>;
fn size_hint(&self) -> (usize, Option<usize>) {
(self.0.len(), Some(self.0.len()))
}
fn next(&mut self) -> Option<Self::Item> {
if self.0.index >= self.0.len() {
None
} else {
Some(self.0.next_chunk())
}
}
}
impl Merk {
/// Creates a `ChunkProducer` which can return chunk proofs for replicating
/// the entire Merk tree.
pub fn chunks(&self) -> Result<ChunkProducer> {
ChunkProducer::new(self)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{
proofs::{
chunk::{verify_leaf, verify_trunk},
Decoder,
},
test_utils::*,
};
#[test]
fn len_small() {
let mut merk = TempMerk::new().unwrap();
let batch = make_batch_seq(1..256);
merk.apply(batch.as_slice(), &[]).unwrap();
let chunks = merk.chunks().unwrap();
assert_eq!(chunks.len(), 1);
assert_eq!(chunks.into_iter().size_hint().0, 1);
}
#[test]
fn len_big() {
let mut merk = TempMerk::new().unwrap();
let batch = make_batch_seq(1..10_000);
merk.apply(batch.as_slice(), &[]).unwrap();
let chunks = merk.chunks().unwrap();
assert_eq!(chunks.len(), 129);
assert_eq!(chunks.into_iter().size_hint().0, 129);
}
#[test]
fn generate_and_verify_chunks() -> Result<()> {
let mut merk = TempMerk::new().unwrap();
let batch = make_batch_seq(1..10_000);
merk.apply(batch.as_slice(), &[]).unwrap();
let mut chunks = merk.chunks().unwrap().into_iter().map(Result::unwrap);
let chunk = chunks.next().unwrap();
let ops = Decoder::new(chunk.as_slice());
let (trunk, height) = verify_trunk(ops).unwrap();
assert_eq!(height, 14);
assert_eq!(trunk.hash()?, merk.root_hash());
assert_eq!(trunk.layer(7).count(), 128);
for (chunk, node) in chunks.zip(trunk.layer(height / 2)) {
let ops = Decoder::new(chunk.as_slice());
verify_leaf(ops, node.hash()?).unwrap();
}
Ok(())
}
#[test]
fn chunks_from_reopen() {
let time = std::time::SystemTime::now()
.duration_since(std::time::SystemTime::UNIX_EPOCH)
.unwrap()
.as_nanos();
let path = format!("chunks_from_reopen_{time}.db");
let original_chunks = {
let mut merk = Merk::open(&path).unwrap();
let batch = make_batch_seq(1..10);
merk.apply(batch.as_slice(), &[]).unwrap();
merk.chunks()
.unwrap()
.into_iter()
.map(Result::unwrap)
.collect::<Vec<_>>()
.into_iter()
};
let merk = TempMerk::open(path).unwrap();
let reopen_chunks = merk.chunks().unwrap().into_iter().map(Result::unwrap);
for (original, checkpoint) in original_chunks.zip(reopen_chunks) {
assert_eq!(original.len(), checkpoint.len());
}
}
#[test]
fn chunks_from_checkpoint() {
let mut merk = TempMerk::new().unwrap();
let batch = make_batch_seq(1..10);
merk.apply(batch.as_slice(), &[]).unwrap();
let path: std::path::PathBuf = "generate_and_verify_chunks_from_checkpoint.db".into();
if path.exists() {
std::fs::remove_dir_all(&path).unwrap();
}
let checkpoint = merk.checkpoint(&path).unwrap();
let original_chunks = merk.chunks().unwrap().into_iter().map(Result::unwrap);
let checkpoint_chunks = checkpoint.chunks().unwrap().into_iter().map(Result::unwrap);
for (original, checkpoint) in original_chunks.zip(checkpoint_chunks) {
assert_eq!(original.len(), checkpoint.len());
}
std::fs::remove_dir_all(&path).unwrap();
}
#[test]
fn random_access_chunks() {
let mut merk = TempMerk::new().unwrap();
let batch = make_batch_seq(1..111);
merk.apply(batch.as_slice(), &[]).unwrap();
let chunks = merk
.chunks()
.unwrap()
.into_iter()
.map(Result::unwrap)
.collect::<Vec<_>>();
let mut producer = merk.chunks().unwrap();
for i in 0..chunks.len() * 2 {
let index = i % chunks.len();
assert_eq!(producer.chunk(index).unwrap(), chunks[index]);
}
}
#[test]
#[should_panic(expected = "Attempted to fetch chunk on empty tree")]
fn test_chunk_empty() {
let merk = TempMerk::new().unwrap();
let _chunks = merk
.chunks()
.unwrap()
.into_iter()
.map(Result::unwrap)
.collect::<Vec<_>>();
}
#[test]
#[should_panic(expected = "Chunk index out-of-bounds")]
fn test_chunk_index_oob() {
let mut merk = TempMerk::new().unwrap();
let batch = make_batch_seq(1..42);
merk.apply(batch.as_slice(), &[]).unwrap();
let mut producer = merk.chunks().unwrap();
let _chunk = producer.chunk(50000).unwrap();
}
#[test]
fn test_chunk_index_gt_1_access() {
let mut merk = TempMerk::new().unwrap();
let batch = make_batch_seq(1..513);
merk.apply(batch.as_slice(), &[]).unwrap();
let mut producer = merk.chunks().unwrap();
println!("length: {}", producer.len());
let chunk = producer.chunk(2).unwrap();
assert_eq!(
chunk,
vec![
3, 0, 8, 0, 0, 0, 0, 0, 0, 0, 18, 0, 60, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 3, 0, 8, 0, 0, 0, 0, 0, 0, 0, 19, 0, 60, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 16, 3, 0, 8, 0, 0, 0, 0, 0, 0, 0, 20, 0,
60, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 17, 3, 0, 8, 0, 0, 0,
0, 0, 0, 0, 21, 0, 60, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 16,
3, 0, 8, 0, 0, 0, 0, 0, 0, 0, 22, 0, 60, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 3, 0, 8, 0, 0, 0, 0, 0, 0, 0, 23, 0, 60, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 16, 3, 0, 8, 0, 0, 0, 0, 0, 0, 0, 24, 0,
60, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 17, 17, 3, 0, 8, 0, 0,
0, 0, 0, 0, 0, 25, 0, 60, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 16, 3, 0, 8, 0, 0, 0, 0, 0, 0, 0, 26, 0, 60, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 3, 0, 8, 0, 0, 0, 0, 0, 0, 0, 27, 0, 60, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 16, 3, 0, 8, 0, 0, 0, 0, 0, 0, 0,
28, 0, 60, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 17, 3, 0, 8,
0, 0, 0, 0, 0, 0, 0, 29, 0, 60, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 16, 3, 0, 8, 0, 0, 0, 0, 0, 0, 0, 30, 0, 60, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 3, 0, 8, 0, 0, 0, 0, 0, 0, 0, 31, 0, 60, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 16, 3, 0, 8, 0, 0, 0, 0, 0,
0, 0, 32, 0, 60, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 17, 17,
17
]
);
}
#[test]
#[should_panic(expected = "Called next_chunk after end")]
fn test_next_chunk_index_oob() {
let mut merk = TempMerk::new().unwrap();
let batch = make_batch_seq(1..42);
merk.apply(batch.as_slice(), &[]).unwrap();
let mut producer = merk.chunks().unwrap();
let _chunk1 = producer.next_chunk();
let _chunk2 = producer.next_chunk();
}
}
================================================
FILE: src/merk/mod.rs
================================================
pub mod chunks;
pub mod restore;
pub mod snapshot;
use std::cmp::Ordering;
use std::collections::LinkedList;
use std::path::{Path, PathBuf};
use std::sync::RwLock;
use rocksdb::DB;
use rocksdb::{checkpoint::Checkpoint, ColumnFamilyDescriptor, WriteBatch};
use crate::error::{Error, Result};
use crate::proofs::{encode_into, query::QueryItem};
use crate::tree::{Batch, Commit, Fetch, GetResult, Hash, Op, RefWalker, Tree, Walker, NULL_HASH};
pub use self::snapshot::Snapshot;
const ROOT_KEY_KEY: &[u8] = b"root";
const FORMAT_VERSION_KEY: &[u8] = b"format";
const AUX_CF_NAME: &str = "aux";
const INTERNAL_CF_NAME: &str = "internal";
const FORMAT_VERSION: u64 = 1;
fn column_families() -> Vec<ColumnFamilyDescriptor> {
vec![
// TODO: clone opts or take args
ColumnFamilyDescriptor::new(AUX_CF_NAME, Merk::default_db_opts()),
ColumnFamilyDescriptor::new(INTERNAL_CF_NAME, Merk::default_db_opts()),
]
}
/// A handle to a Merkle key/value store backed by RocksDB.
pub struct Merk {
pub(crate) tree: RwLock<Option<Tree>>,
pub(crate) db: rocksdb::DB,
pub(crate) path: PathBuf,
}
pub type UseTreeMutResult = Result<Vec<(Vec<u8>, Option<Vec<u8>>)>>;
impl Merk {
/// Opens a store with the specified file path. If no store exists at that
/// path, one will be created.
pub fn open<P: AsRef<Path>>(path: P) -> Result<Merk> {
let db_opts = Merk::default_db_opts();
Merk::open_opt(path, db_opts)
}
pub fn open_readonly<P: AsRef<Path>>(path: P) -> Result<Merk> {
let db_opts = Merk::default_db_opts();
let mut path_buf = PathBuf::new();
path_buf.push(path);
let db = rocksdb::DB::open_cf_descriptors_read_only(
&db_opts,
&path_buf,
column_families(),
false,
)?;
let format_version = load_format_version(&db)?;
if format_version != FORMAT_VERSION {
return Err(Error::Version(format!(
"Format version mismatch: expected {}, found {}",
FORMAT_VERSION, format_version,
)));
}
Ok(Merk {
tree: RwLock::new(load_root(&db)?),
db,
path: path_buf,
})
}
/// Opens a store with the specified file path and the given options. If no
/// store exists at that path, one will be created.
pub fn open_opt<P>(path: P, db_opts: rocksdb::Options) -> Result<Merk>
where
P: AsRef<Path>,
{
let mut path_buf = PathBuf::new();
path_buf.push(path);
let mut db = rocksdb::DB::open_cf_descriptors(&db_opts, &path_buf, column_families())?;
let format_version = load_format_version(&db)?;
if has_root(&db)? {
if format_version == 0 {
log::info!("Migrating store from version 0 to {}...", FORMAT_VERSION);
drop(db);
Merk::migrate_from_v0(&path_buf)?;
db = rocksdb::DB::open_cf_descriptors(&db_opts, &path_buf, column_families())?;
} else if format_version != FORMAT_VERSION {
return Err(Error::Version(format!(
"Unknown format version: expected <= {}, found {}",
FORMAT_VERSION, format_version,
)));
}
}
Ok(Merk {
tree: RwLock::new(load_root(&db)?),
db,
path: path_buf,
})
}
pub fn open_and_get_aux<P>(path: P, key: &[u8]) -> Result<Option<Vec<u8>>>
where
P: AsRef<Path>,
{
let db_opts = Merk::default_db_opts();
let db =
rocksdb::DB::open_cf_descriptors_read_only(&db_opts, path, column_families(), false)?;
let aux_cf = db.cf_handle(AUX_CF_NAME).unwrap();
Ok(db.get_cf(aux_cf, key)?)
}
pub fn default_db_opts() -> rocksdb::Options {
let mut opts = rocksdb::Options::default();
opts.create_if_missing(true);
opts.create_missing_column_families(true);
opts.set_atomic_flush(true);
// TODO: tune
opts.increase_parallelism(num_cpus::get() as i32);
// opts.set_advise_random_on_open(false);
opts.set_allow_mmap_writes(true);
opts.set_allow_mmap_reads(true);
opts.set_max_log_file_size(1_000_000);
opts.set_recycle_log_file_num(5);
opts.set_keep_log_file_num(5);
opts.set_log_level(rocksdb::LogLevel::Warn);
opts
}
/// Gets an auxiliary value.
pub fn get_aux(&self, key: &[u8]) -> Result<Option<Vec<u8>>> {
let aux_cf = self.db.cf_handle(AUX_CF_NAME);
Ok(self.db.get_cf(aux_cf.unwrap(), key)?)
}
/// Gets a value for the given key. If the key is not found, `None` is
/// returned.
///
/// Note that this is essentially the same as a normal RocksDB `get`, so
/// should be a fast operation and has almost no tree overhead.
pub fn get(&self, key: &[u8]) -> Result<Option<Vec<u8>>> {
self.use_tree(|maybe_tree| {
maybe_tree
.and_then(|tree| get(tree, self.source(), key).transpose())
.transpose()
})
}
/// Returns the root hash of the tree (a digest for the entire store which
/// proofs can be checked against). If the tree is empty, returns the null
/// hash (zero-filled).
pub fn root_hash(&self) -> Hash {
self.use_tree(root_hash)
}
/// Applies a batch of operations (puts and deletes) to the tree.
///
/// This will fail if the keys in `batch` are not sorted and unique. This
/// check creates some overhead, so if you are sure your batch is sorted and
/// unique you can use the unsafe `apply_unchecked` for a small performance
/// gain.
///
/// # Example
/// ```
/// # let mut store = merk::test_utils::TempMerk::new().unwrap();
/// # store.apply(&[(vec![4,5,6], Op::Put(vec![0]))], &[]).unwrap();
///
/// use merk::Op;
///
/// let batch = &[
/// (vec![1, 2, 3], Op::Put(vec![4, 5, 6])), // puts value [4,5,6] to key [1,2,3]
/// (vec![4, 5, 6], Op::Delete) // deletes key [4,5,6]
/// ];
/// store.apply(batch, &[]).unwrap();
/// ```
pub fn apply(&mut self, batch: &Batch, aux: &Batch) -> Result<()> {
// ensure keys in batch are sorted and unique
let mut maybe_prev_key: Option<Vec<u8>> = None;
for (key, _) in batch.iter() {
if let Some(prev_key) = maybe_prev_key {
match prev_key.cmp(key) {
Ordering::Greater => {
return Err(Error::BatchKey("Keys in batch must be sorted".into()));
}
Ordering::Equal => {
return Err(Error::BatchKey("Keys in batch must be unique".into()));
}
_ => (),
}
}
maybe_prev_key = Some(key.to_vec());
}
unsafe { self.apply_unchecked(batch, aux) }
}
/// Applies a batch of operations (puts and deletes) to the tree.
///
/// # Safety
/// This is unsafe because the keys in `batch` must be sorted and unique -
/// if they are not, there will be undefined behavior. For a safe version of
/// this method which checks to ensure the batch is sorted and unique, see
/// `apply`.
///
/// # Example
/// ```
/// # let mut store = merk::test_utils::TempMerk::new().unwrap();
/// # store.apply(&[(vec![4,5,6], Op::Put(vec![0]))], &[]).unwrap();
///
/// use merk::Op;
///
/// let batch = &[
/// (vec![1, 2, 3], Op::Put(vec![4, 5, 6])), // puts value [4,5,6] to key [1,2,3]
/// (vec![4, 5, 6], Op::Delete) // deletes key [4,5,6]
/// ];
/// unsafe { store.apply_unchecked(batch, &[]).unwrap() };
/// ```
pub unsafe fn apply_unchecked(&mut self, batch: &Batch, aux: &Batch) -> Result<()> {
let mut tree = self.tree.write().unwrap();
let maybe_walker = tree.take().map(|tree| Walker::new(tree, self.source()));
let (maybe_tree, deleted_keys) = Walker::apply_to(maybe_walker, batch, self.source())?;
*tree = maybe_tree;
drop(tree);
// commit changes to db
self.commit(deleted_keys, aux)
}
/// Closes the store and deletes all data from disk.
pub fn destroy(self) -> Result<()> {
let opts = Merk::default_db_opts();
let path = self.path.clone();
drop(self);
rocksdb::DB::destroy(&opts, path)?;
Ok(())
}
/// Completely rebuilds the tree, keeping all the same stored keys and
/// values.
pub fn repair(self) -> Result<Self> {
use rocksdb::IteratorMode;
let path = self.path.clone();
let create_path = |suffix| {
let mut tmp_path = path.clone();
let tmp_file_name =
format!("{}-{}", path.file_name().unwrap().to_str().unwrap(), suffix);
tmp_path.set_file_name(tmp_file_name);
tmp_path
};
let tmp_path = create_path("repair1");
let tmp = Merk::open(&tmp_path)?;
tmp.destroy()?;
// TODO: split up batch
let mut node = Tree::new(vec![], vec![])?;
let batch: Vec<_> = self
.db
.iterator(IteratorMode::Start)
.map(|entry| {
let (key, node_bytes) = entry.unwrap(); // TODO
node.decode_into(vec![], &node_bytes);
(key.to_vec(), Op::Put(node.value().to_vec()))
})
.collect();
let aux_cf = self.db.cf_handle(AUX_CF_NAME).unwrap();
let aux: Vec<_> = self
.db
.iterator_cf(aux_cf, IteratorMode::Start)
.map(|entry| {
let (key, value) = entry.unwrap(); // TODO
(key.to_vec(), Op::Put(value.to_vec()))
})
.collect();
drop(self);
let mut tmp = Self::open(&tmp_path)?;
tmp.apply(&batch, &aux)?;
drop(tmp);
let tmp_path2 = create_path("repair2");
std::fs::rename(&path, &tmp_path2)?;
std::fs::rename(&tmp_path, &path)?;
std::fs::remove_dir_all(&tmp_path2)?;
Self::open(path)
}
pub fn migrate_from_v0<P: AsRef<Path>>(path: P) -> Result<()> {
let path = path.as_ref().to_path_buf();
let db =
rocksdb::DB::open_cf_descriptors(&Merk::default_db_opts(), path, column_families())?;
let mut iter = db.raw_iterator();
iter.seek_to_first();
while iter.valid() {
let key = iter.key().unwrap();
let mut value = iter.value().unwrap();
let node = Tree::decode_v0(&mut value)?;
let new_value = node.encode();
db.put(key, new_value.as_slice())?;
iter.next();
}
db.put_cf(
db.cf_handle(INTERNAL_CF_NAME).unwrap(),
FORMAT_VERSION_KEY,
FORMAT_VERSION.to_be_bytes(),
)?;
Ok(())
}
/// Creates a Merkle proof for the list of queried keys. For each key in the
/// query, if the key is found in the store then the value will be proven to
/// be in the tree. For each key in the query that does not exist in the
/// tree, its absence will be proven by including boundary keys.
///
/// The proof returned is in an encoded format which can be verified with
/// `merk::verify`.
pub fn prove<Q, I>(&self, query: I) -> Result<Vec<u8>>
where
Q: Into<QueryItem>,
I: IntoIterator<Item = Q>,
{
self.use_tree_mut(move |maybe_tree| prove(maybe_tree, self.source(), query))
}
pub fn flush(&self) -> Result<()> {
Ok(self.db.flush()?)
}
pub fn commit(&mut self, deleted_keys: LinkedList<Vec<u8>>, aux: &Batch) -> Result<()> {
let internal_cf = self.db.cf_handle(INTERNAL_CF_NAME).unwrap();
let aux_cf = self.db.cf_handle(AUX_CF_NAME).unwrap();
let mut batch = rocksdb::WriteBatch::default();
let mut to_batch = self.use_tree_mut(|maybe_tree| -> UseTreeMutResult {
// TODO: concurrent commit
if let Some(tree) = maybe_tree {
// TODO: configurable committer
let mut committer = MerkCommitter::new(tree.height(), 21);
tree.commit(&mut committer)?;
// update pointer to root node
batch.put_cf(internal_cf, ROOT_KEY_KEY, tree.key());
Ok(committer.batch)
} else {
// empty tree, delete pointer to root
batch.delete_cf(internal_cf, ROOT_KEY_KEY);
Ok(vec![])
}
})?;
// TODO: move this to MerkCommitter impl?
for key in deleted_keys {
to_batch.push((key, None));
}
to_batch.sort_by(|a, b| a.0.cmp(&b.0));
for (key, maybe_value) in to_batch {
if let Some(value) = maybe_value {
batch.put(key, value);
} else {
batch.delete(key);
}
}
for (key, value) in aux {
match value {
Op::Put(value) => batch.put_cf(aux_cf, key, value),
Op::Delete => batch.delete_cf(aux_cf, key),
};
}
// update format version
// TODO: shouldn't need a write per commit
batch.put_cf(
internal_cf,
FORMAT_VERSION_KEY,
FORMAT_VERSION.to_be_bytes(),
);
// write to db
self.write(batch)?;
Ok(())
}
pub fn walk<T>(&self, f: impl FnOnce(Option<RefWalker<MerkSource>>) -> T) -> T {
let mut tree = self.tree.write().unwrap();
let maybe_walker = tree
.as_mut()
.map(|tree| RefWalker::new(tree, self.source()));
f(maybe_walker)
}
pub fn raw_iter(&self) -> rocksdb::DBRawIterator {
self.db.raw_iterator()
}
pub fn checkpoint<P: AsRef<Path>>(&self, path: P) -> Result<Merk> {
Checkpoint::new(&self.db)?.create_checkpoint(&path)?;
Merk::open(path)
}
pub fn snapshot(&self) -> Result<Snapshot> {
Ok(Snapshot::new(self.db.snapshot(), load_root(&self.db)?))
}
pub fn db(&self) -> &DB {
&self.db
}
fn source(&self) -> MerkSource {
MerkSource { db: &self.db }
}
fn use_tree<T>(&self, f: impl FnOnce(Option<&Tree>) -> T) -> T {
let tree = self.tree.read().unwrap();
f(tree.as_ref())
}
fn use_tree_mut<T>(&self, f: impl FnOnce(Option<&mut Tree>) -> T) -> T {
let mut tree_slot = self.tree.write().unwrap();
let mut tree = tree_slot.take();
let res = f(tree.as_mut());
*tree_slot = tree;
res
}
pub(crate) fn write(&mut self, batch: WriteBatch) -> Result<()> {
let mut opts = rocksdb::WriteOptions::default();
opts.set_sync(false);
// TODO: disable WAL once we can ensure consistency with transactions
self.db.write_opt(batch, &opts)?;
Ok(())
}
pub(crate) fn set_root_key(&mut self, key: Vec<u8>) -> Result<()> {
let internal_cf = self.db.cf_handle(INTERNAL_CF_NAME).unwrap();
let mut batch = WriteBatch::default();
batch.put_cf(internal_cf, ROOT_KEY_KEY, key);
self.write(batch)
}
pub(crate) fn fetch_node(&self, key: &[u8]) -> Result<Option<Tree>> {
self.source().fetch_by_key(key)
}
pub(crate) fn load_root(&mut self) -> Result<()> {
let root = load_root(&self.db)?;
*self.tree.write().unwrap() = root;
Ok(())
}
}
#[derive(Clone)]
pub struct MerkSource<'a> {
db: &'a rocksdb::DB,
}
impl<'a> Fetch for MerkSource<'a> {
fn fetch_by_key(&self, key: &[u8]) -> Result<Option<Tree>> {
Ok(self
.db
.get_pinned(key)?
.map(|bytes| Tree::decode(key.to_vec(), &bytes)))
}
}
struct MerkCommitter {
batch: Vec<(Vec<u8>, Option<Vec<u8>>)>,
height: u8,
levels: u8,
}
impl MerkCommitter {
fn new(height: u8, levels: u8) -> Self {
MerkCommitter {
batch: Vec::with_capacity(10000),
height,
levels,
}
}
}
impl Commit for MerkCommitter {
fn write(&mut self, tree: &Tree) -> Result<()> {
let mut buf = Vec::with_capacity(tree.encoding_length());
tree.encode_into(&mut buf);
self.batch.push((tree.key().to_vec(), Some(buf)));
Ok(())
}
fn prune(&self, tree: &Tree) -> (bool, bool) {
// keep N top levels of tree
let prune = (self.height - tree.height()) >= self.levels;
(prune, prune)
}
}
pub fn get<F: Fetch>(tree: &Tree, source: F, key: &[u8]) -> Result<Option<Vec<u8>>> {
Ok(match tree.get_value(key)? {
GetResult::Found(value) => Some(value),
GetResult::NotFound => None,
GetResult::Pruned => source.fetch_by_key(key)?.map(|node| node.value().to_vec()),
})
}
fn root_hash(maybe_tree: Option<&Tree>) -> Hash {
maybe_tree.map_or(NULL_HASH, |tree| tree.hash())
}
fn prove<Q, I, F>(maybe_tree: Option<&mut Tree>, source: F, query: I) -> Result<Vec<u8>>
where
Q: Into<QueryItem>,
I: IntoIterator<Item = Q>,
F: Fetch + Send + Clone,
{
let query_vec: Vec<QueryItem> = query.into_iter().map(Into::into).collect();
let tree =
maybe_tree.ok_or_else(|| Error::Proof("Cannot create proof for empty tree".into()))?;
let mut ref_walker = RefWalker::new(tree, source);
let (proof, _) = ref_walker.create_proof(query_vec.as_slice())?;
let mut bytes = Vec::with_capacity(128);
encode_into(proof.iter(), &mut bytes);
Ok(bytes)
}
fn has_root(db: &DB) -> Result<bool> {
let internal_cf = db.cf_handle(INTERNAL_CF_NAME).unwrap();
Ok(db.get_pinned_cf(internal_cf, ROOT_KEY_KEY)?.is_some())
}
fn load_root(db: &DB) -> Result<Option<Tree>> {
let internal_cf = db.cf_handle(INTERNAL_CF_NAME).unwrap();
db.get_pinned_cf(internal_cf, ROOT_KEY_KEY)?
.map(|key| MerkSource { db }.fetch_by_key_expect(key.to_vec().as_slice()))
.transpose()
}
fn load_format_version(db: &DB) -> Result<u64> {
let internal_cf = db.cf_handle(INTERNAL_CF_NAME).unwrap();
let maybe_version = db.get_pinned_cf(internal_cf, FORMAT_VERSION_KEY)?;
let Some(version) = maybe_version else {
return Ok(0);
};
let mut buf = [0; 8];
buf.copy_from_slice(&version);
Ok(u64::from_be_bytes(buf))
}
#[cfg(test)]
mod test {
use super::{Merk, MerkSource, RefWalker};
use crate::test_utils::*;
use crate::Op;
use std::thread;
// TODO: Close and then reopen test
fn assert_invariants(merk: &TempMerk) {
merk.use_tree(|maybe_tree| {
let tree = maybe_tree.expect("expected tree");
assert_tree_invariants(tree);
})
}
#[test]
fn simple_insert_apply() {
let batch_size = 20;
let path = thread::current().name().unwrap().to_owned();
let mut merk = TempMerk::open(path).expect("failed to open merk");
let batch = make_batch_seq(0..batch_size);
merk.apply(&batch, &[]).expect("apply failed");
assert_invariants(&merk);
assert_eq!(
merk.root_hash(),
[
29, 99, 91, 248, 54, 96, 47, 252, 39, 203, 208, 163, 199, 30, 34, 251, 247, 34,
241, 203, 17, 252, 127, 44, 155, 83, 22, 54, 117, 85, 252, 200
]
);
}
#[test]
fn insert_uncached() {
let batch_size = 20;
let path = thread::current().name().unwrap().to_owned();
let mut merk = TempMerk::open(path).expect("failed to open merk");
let batch = make_batch_seq(0..batch_size);
merk.apply(&batch, &[]).expect("apply failed");
assert_invariants(&merk);
let batch = make_batch_seq(batch_size..(batch_size * 2));
merk.apply(&batch, &[]).expect("apply failed");
assert_invariants(&merk);
}
#[test]
fn insert_rand() {
let tree_size = 40;
let batch_size = 4;
let path = thread::current().name().unwrap().to_owned();
let mut merk = TempMerk::open(path).expect("failed to open merk");
for i in 0..(tree_size / batch_size) {
println!("i:{i}");
let batch = make_batch_rand(batch_size, i);
merk.apply(&batch, &[]).expect("apply failed");
}
}
#[test]
fn actual_deletes() {
let path = thread::current().name().unwrap().to_owned();
let mut merk = TempMerk::open(path).expect("failed to open merk");
let batch = make_batch_rand(10, 1);
merk.apply(&batch, &[]).expect("apply failed");
let key = batch.first().unwrap().0.clone();
merk.apply(&[(key.clone(), Op::Delete)], &[]).unwrap();
let value = merk.db.get(key.as_slice()).unwrap();
assert!(value.is_none());
}
#[test]
fn aux_data() {
let path = thread::current().name().unwrap().to_owned();
let mut merk = TempMerk::open(path).expect("failed to open merk");
merk.apply(&[], &[(vec![1, 2, 3], Op::Put(vec![4, 5, 6]))])
.expect("apply failed");
let val = merk.get_aux(&[1, 2, 3]).unwrap();
assert_eq!(val, Some(vec![4, 5, 6]));
}
#[test]
fn simulated_crash() {
let path = thread::current().name().unwrap().to_owned();
let mut merk = CrashMerk::open(path).expect("failed to open merk");
merk.apply(
&[(vec![0], Op::Put(vec![1]))],
&[(vec![2], Op::Put(vec![3]))],
)
.expect("apply failed");
// make enough changes so that main column family gets auto-flushed
for i in 0..250 {
merk.apply(&make_batch_seq(i * 2_000..(i + 1) * 2_000), &[])
.expect("apply failed");
}
unsafe {
merk.crash().unwrap();
}
assert_eq!(merk.get_aux(&[2]).unwrap(), Some(vec![3]));
merk.destroy().unwrap();
}
#[test]
fn get_not_found() {
let path = thread::current().name().unwrap().to_owned();
let mut merk = TempMerk::open(path).expect("failed to open merk");
// no root
assert!(merk.get(&[1, 2, 3]).unwrap().is_none());
// cached
merk.apply(&[(vec![5, 5, 5], Op::Put(vec![]))], &[])
.unwrap();
assert!(merk.get(&[1, 2, 3]).unwrap().is_none());
// uncached
merk.apply(
&[
(vec![0, 0, 0], Op::Put(vec![])),
(vec![1, 1, 1], Op::Put(vec![])),
(vec![2, 2, 2], Op::Put(vec![])),
],
&[],
)
.unwrap();
assert!(merk.get(&[3, 3, 3]).unwrap().is_none());
}
#[test]
fn reopen() {
fn collect(mut node: RefWalker<MerkSource>, nodes: &mut Vec<Vec<u8>>) {
nodes.push(node.tree().encode());
node.walk(true)
.unwrap()
.into_iter()
.for_each(|c| collect(c, nodes));
node.walk(false)
.unwrap()
.into_iter()
.for_each(|c| collect(c, nodes));
}
let time = std::time::SystemTime::now()
.duration_since(std::time::SystemTime::UNIX_EPOCH)
.unwrap()
.as_nanos();
let path = format!("merk_reopen_{time}.db");
let original_nodes = {
let mut merk = Merk::open(&path).unwrap();
let batch = make_batch_seq(1..10_000);
merk.apply(batch.as_slice(), &[]).unwrap();
let mut tree = merk.tree.write().unwrap().take().unwrap();
let walker = RefWalker::new(&mut tree, merk.source());
let mut nodes = vec![];
collect(walker, &mut nodes);
nodes
};
let merk = TempMerk::open(&path).unwrap();
let mut tree = merk.tree.write().unwrap().take().unwrap();
let walker = RefWalker::new(&mut tree, merk.source());
let mut reopen_nodes = vec![];
collect(walker, &mut reopen_nodes);
assert_eq!(reopen_nodes, original_nodes);
}
#[test]
fn reopen_iter() {
fn collect(iter: &mut rocksdb::DBRawIterator, nodes: &mut Vec<(Vec<u8>, Vec<u8>)>) {
while iter.valid() {
nodes.push((iter.key().unwrap().to_vec(), iter.value().unwrap().to_vec()));
iter.next();
}
}
let time = std::time::SystemTime::now()
.duration_since(std::time::SystemTime::UNIX_EPOCH)
.unwrap()
.as_nanos();
let path = format!("merk_reopen_{time}.db");
let original_nodes = {
let mut merk = Merk::open(&path).unwrap();
let batch = make_batch_seq(1..10_000);
merk.apply(batch.as_slice(), &[]).unwrap();
let mut nodes = vec![];
collect(&mut merk.raw_iter(), &mut nodes);
nodes
};
let merk = TempMerk::open(&path).unwrap();
let mut reopen_nodes = vec![];
collect(&mut merk.raw_iter(), &mut reopen_nodes);
assert_eq!(reopen_nodes, original_nodes);
}
#[test]
fn checkpoint() {
let path = thread::current().name().unwrap().to_owned();
let mut merk = TempMerk::open(&path).expect("failed to open merk");
merk.apply(&[(vec![1], Op::Put(vec![0]))], &[])
.expect("apply failed");
let mut checkpoint = merk.checkpoint(path + ".checkpoint").unwrap();
assert_eq!(merk.get(&[1]).unwrap(), Some(vec![0]));
assert_eq!(checkpoint.get(&[1]).unwrap(), Some(vec![0]));
merk.apply(
&[(vec![1], Op::Put(vec![1])), (vec![2], Op::Put(vec![0]))],
&[],
)
.expect("apply failed");
assert_eq!(merk.get(&[1]).unwrap(), Some(vec![1]));
assert_eq!(merk.get(&[2]).unwrap(), Some(vec![0]));
assert_eq!(checkpoint.get(&[1]).unwrap(), Some(vec![0]));
assert_eq!(checkpoint.get(&[2]).unwrap(), None);
checkpoint
.apply(&[(vec![2], Op::Put(vec![123]))], &[])
.expect("apply failed");
assert_eq!(merk.get(&[1]).unwrap(), Some(vec![1]));
assert_eq!(merk.get(&[2]).unwrap(), Some(vec![0]));
assert_eq!(checkpoint.get(&[1]).unwrap(), Some(vec![0]));
assert_eq!(checkpoint.get(&[2]).unwrap(), Some(vec![123]));
checkpoint.destroy().unwrap();
assert_eq!(merk.get(&[1]).unwrap(), Some(vec![1]));
assert_eq!(merk.get(&[2]).unwrap(), Some(vec![0]));
}
#[test]
fn checkpoint_iterator() {
let path = thread::current().name().unwrap().to_owned();
let mut merk = TempMerk::open(&path).expect("failed to open merk");
merk.apply(&make_batch_seq(1..100), &[])
.expect("apply failed");
let path: std::path::PathBuf = (path + ".checkpoint").into();
if path.exists() {
std::fs::remove_dir_all(&path).unwrap();
}
let checkpoint = merk.checkpoint(&path).unwrap();
let mut merk_iter = merk.raw_iter();
let mut checkpoint_iter = checkpoint.raw_iter();
loop {
assert_eq!(merk_iter.valid(), checkpoint_iter.valid());
if !merk_iter.valid() {
break;
}
assert_eq!(merk_iter.key(), checkpoint_iter.key());
assert_eq!(merk_iter.value(), checkpoint_iter.value());
merk_iter.next();
checkpoint_iter.next();
}
std::fs::remove_dir_all(&path).unwrap();
}
#[test]
fn repair() {
let path = thread::current().name().unwrap().to_owned();
let mut merk = Merk::open(&path).expect("failed to open merk");
merk.apply(&make_batch_seq(0..100), &[])
.expect("apply failed");
let merk = merk.repair().unwrap();
merk.walk(|mut maybe_walker| {
fn recurse(maybe_walker: &mut Option<RefWalker<MerkSource>>) {
if let Some(walker) = maybe_walker {
recurse(&mut walker.walk(true).unwrap());
recurse(&mut walker.walk(false).unwrap());
}
}
recurse(&mut maybe_walker);
let walker = maybe_walker.unwrap();
let exp_value = put_entry_value();
for (i, (key, value)) in walker.tree().iter().enumerate() {
let exp_key = seq_key(i as u64);
assert_eq!(key, exp_key);
assert_eq!(value, exp_value);
}
});
std::fs::remove_dir_all(&path).unwrap();
}
}
================================================
FILE: src/merk/restore.rs
================================================
//! Provides `Restorer`, which can create a replica of a Merk instance by
//! receiving chunk proofs.
use super::Merk;
use crate::{
merk::MerkSource,
proofs::{
chunk::{verify_leaf, verify_trunk, MIN_TRUNK_HEIGHT},
tree::{Child, Tree as ProofTree},
Decoder, Node,
},
tree::{Link, RefWalker, Tree},
Error, Hash, Result,
};
use rocksdb::WriteBatch;
use std::iter::Peekable;
use std::path::Path;
/// A `Restorer` handles decoding, verifying, and storing chunk proofs to
/// replicate an entire Merk tree. It expects the chunks to be processed in
/// order, retrying the last chunk if verification fails.
pub struct Restorer {
leaf_hashes: Option<Peekable<std::vec::IntoIter<Hash>>>,
parent_keys: Option<Peekable<std::vec::IntoIter<Vec<u8>>>>,
trunk_height: Option<usize>,
merk: Merk,
expected_root_hash: Hash,
stated_length: usize,
}
impl Restorer {
/// Creates a new `Restorer`, which will initialize a new Merk at the given
/// file path. The first chunk (the "trunk") will be compared against
/// `expected_root_hash`, then each subsequent chunk will be compared
/// against the hashes stored in the trunk, so that the restore process will
/// never allow malicious peers to send more than a single invalid chunk.
///
/// The `stated_length` should be the number of chunks stated by the peer,
/// which will be verified after processing a valid first chunk to make it
/// easier to download chunks from peers without needing to trust this
/// length.
pub fn new<P: AsRef<Path>>(
db_path: P,
expected_root_hash: Hash,
stated_length: usize,
) -> Result<Self> {
if db_path.as_ref().exists() {
return Err(Error::Path("The given path already exists".into()));
}
Ok(Self {
expected_root_hash,
stated_length,
trunk_height: None,
merk: Merk::open(db_path)?,
leaf_hashes: None,
parent_keys: None,
})
}
/// Verifies a chunk and writes it to the working RocksDB instance. Expects
/// to be called for each chunk in order. Returns the number of remaining
/// chunks.
///
/// Once there are no remaining chunks to be processed, `finalize` should
/// be called.
pub fn process_chunk(&mut self, chunk_bytes: &[u8]) -> Result<usize> {
let ops = Decoder::new(chunk_bytes);
match self.leaf_hashes {
None => self.process_trunk(ops),
Some(_) => self.process_leaf(ops),
}
}
/// Consumes the `Restorer` and returns the newly-created, fully-populated
/// Merk instance. This method will return an error if called before
/// processing all chunks (e.g. `restorer.remaining_chunks()` is not equal
/// to 0).
pub fn finalize(mut self) -> Result<Merk> {
if self.remaining_chunks().is_none() || self.remaining_chunks().unwrap() != 0 {
return Err(Error::ChunkProcessing(
"Called finalize before all chunks were processed".into(),
));
}
if self.trunk_height.unwrap() >= MIN_TRUNK_HEIGHT {
self.rewrite_trunk_child_heights()?;
}
self.merk.flush()?;
self.merk.load_root()?;
Ok(self.merk)
}
/// Returns the number of remaining chunks to be processed. If called before
/// the first chunk is processed, this method will return `None` since we do
/// not yet have enough information to know about the number of chunks.
pub fn remaining_chunks(&self) -> Option<usize> {
self.leaf_hashes.as_ref().map(|lh| lh.len())
}
/// Writes the data contained in `tree` (extracted from a verified chunk
/// proof) to the RocksDB.
fn write_chunk(&mut self, tree: ProofTree) -> Result<()> {
let mut batch = WriteBatch::default();
tree.visit_refs(&mut |proof_node| {
let (key, mut node) = match &proof_node.node {
// TODO: encode tree node without cloning key/value
Node::KV(key, value) => match Tree::new(key.clone(), value.clone()) {
Ok(node) => (key, node),
Err(_) => return,
},
_ => return,
};
*node.slot_mut(true) = proof_node.left.as_ref().map(Child::as_link);
*node.slot_mut(false) = proof_node.right.as_ref().map(Child::as_link);
let bytes = node.encode();
batch.put(key, bytes);
});
self.merk.write(batch)
}
/// Verifies the trunk then writes its data to the RocksDB.
///
/// The trunk contains a height proof which lets us verify the total number
/// of expected chunks is the same as `stated_length` as passed into
/// `Restorer::new()`. We also verify the expected root hash at this step.
fn process_trunk(&mut self, ops: Decoder) -> Result<usize> {
let (trunk, height) = verify_trunk(ops)?;
if trunk.hash()? != self.expected_root_hash {
return Err(Error::HashMismatch(self.expected_root_hash, trunk.hash()?));
}
let root_key = trunk.key().to_vec();
let trunk_height = height / 2;
self.trunk_height = Some(trunk_height);
let chunks_remaining = if trunk_height >= MIN_TRUNK_HEIGHT {
let leaf_hashes = trunk
.layer(trunk_height)
.map(|node| node.hash())
.collect::<Result<Vec<_>>>()?
.into_iter()
.peekable();
self.leaf_hashes = Some(leaf_hashes);
let parent_keys = trunk
.layer(trunk_height - 1)
.map(|node| node.key().to_vec())
.collect::<Vec<Vec<u8>>>()
.into_iter()
.peekable();
self.parent_keys = Some(parent_keys);
assert_eq!(
self.parent_keys.as_ref().unwrap().len(),
self.leaf_hashes.as_ref().unwrap().len() / 2
);
let chunks_remaining = (2_usize).pow(trunk_height as u32);
assert_eq!(self.remaining_chunks_unchecked(), chunks_remaining);
chunks_remaining
} else {
self.leaf_hashes = Some(vec![].into_iter().peekable());
self.parent_keys = Some(vec![].into_iter().peekable());
0
};
if self.stated_length != chunks_remaining + 1 {
return Err(Error::ChunkProcessing(
"Stated length does not match calculated number of chunks".into(),
));
}
// note that these writes don't happen atomically, which is fine here
// because if anything fails during the restore process we will just
// scrap the whole restore and start over
self.write_chunk(trunk)?;
self.merk.set_root_key(root_key)?;
Ok(chunks_remaining)
}
/// Verifies a leaf chunk then writes it to the RocksDB. This needs to be
/// called in order, retrying the last chunk for any failed verifications.
fn process_leaf(&mut self, ops: Decoder) -> Result<usize> {
let leaf_hashes = self.leaf_hashes.as_mut().unwrap();
let leaf_hash = leaf_hashes
.peek()
.expect("Received more chunks than expected");
let leaf = verify_leaf(ops, *leaf_hash)?;
self.rewrite_parent_link(&leaf)?;
self.write_chunk(leaf)?;
let leaf_hashes = self.leaf_hashes.as_mut().unwrap();
leaf_hashes.next();
Ok(self.remaining_chunks_unchecked())
}
/// The parent of the root node of the leaf does not know the key of its
/// children when it is first written. Now that we have verified this leaf,
/// we can write the key into the parent node's entry. Note that this does
/// not need to recalcuate hashes since it already had the child hash.
fn rewrite_parent_link(&mut self, leaf: &ProofTree) -> Result<()> {
let parent_keys = self.parent_keys.as_mut().unwrap();
let parent_key = parent_keys.peek().unwrap().clone();
let mut parent = self
.merk
.fetch_node(parent_key.as_slice())?
.expect("Could not find parent of leaf chunk");
let is_left_child = self.remaining_chunks_unchecked() % 2 == 0;
if let Some(Link::Reference { ref mut key, .. }) = parent.link_mut(is_left_child) {
*key = leaf.key().to_vec();
} else {
panic!("Expected parent links to be type Link::Reference");
};
let parent_bytes = parent.encode();
self.merk.db.put(parent_key, parent_bytes)?;
if !is_left_child {
let parent_keys = self.parent_keys.as_mut().unwrap();
parent_keys.next();
}
Ok(())
}
fn rewrite_trunk_child_heights(&mut self) -> Result<()> {
fn recurse(
mut node: RefWalker<MerkSource>,
remaining_depth: usize,
batch: &mut WriteBatch,
) -> Result<(u8, u8)> {
if remaining_depth == 0 {
return Ok(node.tree().child_heights());
}
let mut cloned_node =
Tree::decode(node.tree().key().to_vec(), node.tree().encode().as_slice());
let left_child = node.walk(true)?.unwrap();
let left_child_heights = recurse(left_child, remaining_depth - 1, batch)?;
let left_height = left_child_heights.0.max(left_child_heights.1) + 1;
*cloned_node.link_mut(true).unwrap().child_heights_mut() = left_child_heights;
let right_child = node.walk(false)?.unwrap();
let right_child_heights = recurse(right_child, remaining_depth - 1, batch)?;
let right_height = right_child_heights.0.max(right_child_heights.1) + 1;
*cloned_node.link_mut(false).unwrap().child_heights_mut() = right_child_heights;
let bytes = cloned_node.encode();
batch.put(node.tree().key(), bytes);
Ok((left_height, right_height))
}
self.merk.flush()?;
self.merk.load_root()?;
let mut batch = WriteBatch::default();
let depth = self.trunk_height.unwrap();
self.merk.use_tree_mut(|maybe_tree| {
let tree = maybe_tree.unwrap();
let walker = RefWalker::new(tree, self.merk.source());
recurse(walker, depth, &mut batch)
})?;
self.merk.write(batch)?;
Ok(())
}
/// Returns the number of remaining chunks to be processed. This method will
/// panic if called before processing the first chunk (since that chunk
/// gives us the information to know how many chunks to expect).
pub fn remaining_chunks_unchecked(&self) -> usize {
self.leaf_hashes.as_ref().unwrap().len()
}
}
impl Merk {
/// Creates a new `Restorer`, which can be used to verify chunk proofs to
/// replicate an entire Merk tree. A new Merk instance will be initialized
/// by creating a RocksDB at `path`.
///
/// The restoration process will verify integrity by checking that the
/// incoming chunk proofs match `expected_root_hash`. The `stated_length`
/// should be the number of chunks as stated by peers, which will also be
/// verified during the restoration process.
pub fn restore<P: AsRef<Path>>(
path: P,
expected_root_hash: Hash,
stated_length: usize,
) -> Result<Restorer> {
Restorer::new(path, expected_root_hash, stated_length)
}
}
impl ProofTree {
fn child_heights(&self) -> (u8, u8) {
(
self.left.as_ref().map_or(0, |c| c.tree.height as u8),
self.right.as_ref().map_or(0, |c| c.tree.height as u8),
)
}
}
impl Child {
fn as_link(&self) -> Link {
let key = match &self.tree.node {
Node::KV(key, _) => key.as_slice(),
// for the connection between the trunk and leaf chunks, we don't
// have the child key so we must first write in an empty one. once
// the leaf gets verified, we can write in this key to its parent
_ => &[],
};
Link::Reference {
hash: self.hash,
child_heights: self.tree.child_heights(),
key: key.to_vec(),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::test_utils::*;
use crate::tree::{Batch, Op};
use std::path::PathBuf;
fn restore_test(batches: &[&Batch], expected_nodes: usize) {
let mut original = TempMerk::new().unwrap();
for batch in batches {
original.apply(batch, &[]).unwrap();
}
original.flush().unwrap();
let chunks = original.chunks().unwrap();
let path: PathBuf = std::thread::current().name().unwrap().into();
if path.exists() {
std::fs::remove_dir_all(&path).unwrap();
}
let mut restorer = Merk::restore(&path, original.root_hash(), chunks.len()).unwrap();
assert_eq!(restorer.remaining_chunks(), None);
let mut expected_remaining = chunks.len();
for chunk in chunks {
let chunk = chunk.unwrap();
let remaining = restorer.process_chunk(chunk.as_slice()).unwrap();
expected_remaining -= 1;
assert_eq!(remaining, expected_remaining);
assert_eq!(restorer.remaining_chunks().unwrap(), expected_remaining);
}
assert_eq!(expected_remaining, 0);
let restored = restorer.finalize().unwrap();
assert_eq!(restored.root_hash(), original.root_hash());
assert_raw_db_entries_eq(&restored, &original, expected_nodes);
std::fs::remove_dir_all(&path).unwrap();
}
#[test]
fn restore_10000() {
restore_test(&[&make_batch_seq(0..10_000)], 10_000);
}
#[test]
fn restore_3() {
restore_test(&[&make_batch_seq(0..3)], 3);
}
#[test]
fn restore_2_left_heavy() {
restore_test(
&[&[(vec![0], Op::Put(vec![]))], &[(vec![1], Op::Put(vec![]))]],
2,
);
}
#[test]
fn restore_2_right_heavy() {
restore_test(
&[&[(vec![1], Op::Put(vec![]))], &[(vec![0], Op::Put(vec![]))]],
2,
);
}
#[test]
fn restore_1() {
restore_test(&[&make_batch_seq(0..1)], 1);
}
fn assert_raw_db_entries_eq(restored: &Merk, original: &Merk, length: usize) {
let mut original_entries = original.raw_iter();
let mut restored_entries = restored.raw_iter();
original_entries.seek_to_first();
restored_entries.seek_to_first();
let mut i = 0;
loop {
assert_eq!(restored_entries.valid(), original_entries.valid());
if !restored_entries.valid() {
break;
}
assert_eq!(restored_entries.key(), original_entries.key());
assert_eq!(restored_entries.value(), original_entries.value());
restored_entries.next();
original_entries.next();
i += 1;
}
assert_eq!(i, length);
}
}
================================================
FILE: src/merk/snapshot.rs
================================================
//! In-memory snapshots of database state.
//!
//! Snapshots are read-only views of the database state at a particular point in
//! time. This can be useful for retaining recent versions of history which can
//! be queried against. Merk snapshots are backed by the similar RocksDB
//! snapshot, but with the added ability to create proofs.
use std::cell::Cell;
use crate::{
proofs::query::QueryItem,
tree::{Fetch, RefWalker, Tree, NULL_HASH},
Hash, Result,
};
/// A read-only view of the database state at a particular point in time.
///
/// `Snapshot`s are cheap to create since they are just a handle and don't copy
/// any data - they instead just prevent the underlying replaced data from being
/// compacted in RocksDB until they are dropped. They are only held in memory,
/// and will not be persisted after the process exits.
pub struct Snapshot<'a> {
/// The underlying RocksDB snapshot.
ss: Option<rocksdb::Snapshot<'a>>,
/// The Merk tree at the time the snapshot was created.
tree: Cell<Option<Tree>>,
/// Whether the underlying RocksDB snapshot should be dropped when the
/// `Snapshot` is dropped.
should_drop_ss: bool,
}
impl<'a> Snapshot<'a> {
/// Creates a new `Snapshot` from a RocksDB snapshot and a Merk tree.
///
/// The RocksDB snapshot will be dropped when the [Snapshot] is dropped.
pub fn new(db: rocksdb::Snapshot<'a>, tree: Option<Tree>) -> Self {
Snapshot {
ss: Some(db),
tree: Cell::new(tree),
should_drop_ss: true,
}
}
/// Converts the [Snapshot] into a [StaticSnapshot], an alternative which
/// has easier (but more dangerous) lifetime requirements.
pub fn staticize(mut self) -> StaticSnapshot {
let ss: RocksDBSnapshot = unsafe { std::mem::transmute(self.ss.take().unwrap()) };
StaticSnapshot {
tree: Cell::new(self.tree.take()),
inner: ss.inner,
should_drop: false,
}
}
/// Gets the value associated with the given key, from the time the snapshot
/// was created.
pub fn get(&self, key: &[u8]) -> Result<Option<Vec<u8>>> {
self.use_tree(|maybe_tree| {
maybe_tree
.and_then(|tree| super::get(tree, self.source(), key).transpose())
.transpose()
})
}
/// Gets the root hash of the tree at the time the snapshot was created.
pub fn root_hash(&self) -> Hash {
self.use_tree(|tree| tree.map_or(NULL_HASH, |tree| tree.hash()))
}
/// Proves the given query against the tree at the time the snapshot was
/// created.
pub fn prove<Q, I>(&self, query: I) -> Result<Vec<u8>>
where
Q: Into<QueryItem>,
I: IntoIterator<Item = Q>,
{
self.use_tree_mut(move |maybe_tree| super::prove(maybe_tree, self.source(), query))
}
/// Walks the tree at the time the snapshot was created, fetching the child
/// node from the backing store if necessary.
pub fn walk<T>(&self, f: impl FnOnce(Option<RefWalker<SnapshotSource>>) -> T) -> T {
let mut tree = self.tree.take();
let maybe_walker = tree
.as_mut()
.map(|tree| RefWalker::new(tree, self.source()));
let res = f(maybe_walker);
self.tree.set(tree);
res
}
/// Returns an iterator over the keys and values in the backing store from
/// the time the snapshot was created.
pub fn raw_iter(&self) -> rocksdb::DBRawIterator {
self.ss.as_ref().unwrap().raw_iterator()
}
/// A data source which can be used to fetch values from the backing store,
/// from the time the snapshot was created.
fn source(&self) -> SnapshotSource {
SnapshotSource(self.ss.as_ref().unwrap())
}
/// Uses the tree, and then puts it back.
fn use_tree<T>(&self, f: impl FnOnce(Option<&Tree>) -> T) -> T {
let tree = self.tree.take();
let res = f(tree.as_ref());
self.tree.set(tree);
res
}
/// Uses the tree mutably, and then puts it back.
fn use_tree_mut<T>(&self, f: impl FnOnce(Option<&mut Tree>) -> T) -> T {
let mut tree = self.tree.take();
let res = f(tree.as_mut());
self.tree.set(tree);
res
}
}
impl<'a> Drop for Snapshot<'a> {
fn drop(&mut self) {
if !self.should_drop_ss {
std::mem::forget(self.ss.take());
}
}
}
/// A data source which can be used to fetch values from the backing store, from
/// the time the snapshot was created.
///
/// This implements [Fetch] and should be used with a type such as [RefWalker].
#[derive(Clone)]
pub struct SnapshotSource<'a>(&'a rocksdb::Snapshot<'a>);
impl<'a> Fetch for SnapshotSource<'a> {
fn fetch_by_key(&self, key: &[u8]) -> Result<Option<Tree>> {
Ok(self
.0
.get(key)?
.map(|bytes| Tree::decode(key.to_vec(), &bytes)))
}
}
/// A read-only view of the database state at a particular point in time, but
/// with an internal raw pointer to allow for manual lifetime management.
///
/// This is useful when you would otherwise want a [Snapshot], but you want to
/// use the database while the snapshot is still alive. This is unsafe because
/// it is the caller's responsibility to ensure that the underlying RocksDB
/// snapshot outlives the [StaticSnapshot].
///
/// By default, the RocksDB snapshot will not be dropped when the
/// [StaticSnapshot] is dropped, resulting in a memory leak. For correct usage,
/// you must call [StaticSnapshot::drop] to ensure the RocksDB snapshot gets
/// dropped when the [StaticSnapshot] is dropped.
pub struct StaticSnapshot {
/// A Merk tree based on the database state at the time the snapshot was
/// created.
tree: Cell<Option<Tree>>,
/// A raw pointer to the RocksDB snapshot.
inner: *const (),
/// Used to detect whether the `StaticSnapshot` was set to manually drop
/// before its [Drop::drop] implementation was called.
pub should_drop: bool,
}
/// An equivalent struct to the [rocksdb::Snapshot] struct within the `rocksdb`
/// crate. This is used to access the private fields of the foreign crate's
/// struct by first transmuting.
///
/// To guarantee that breaking changes in the `rocksdb` crate do not affect the
/// transmutation into this struct, see the
/// [tests::rocksdb_snapshot_struct_format] test.
struct RocksDBSnapshot<'a> {
/// A reference to the associated RocksDB database.
_db: &'a rocksdb::DB,
/// A raw pointer to the snapshot handle.
inner: *const (),
}
// We need this because we have a raw pointer to a RocksDB snapshot, but we
// know that our usage of it is thread-safe:
// https://github.com/facebook/rocksdb/blob/main/include/rocksdb/snapshot.h#L15-L16
unsafe impl Send for StaticSnapshot {}
unsafe impl Sync for StaticSnapshot {}
impl StaticSnapshot {
/// Converts the [StaticSnapshot] to a [Snapshot] by re-associating with the
/// database it was originally created from.
///
/// # Safety
/// This will cause undefined behavior if a database other than the one
/// originally used to create the snapshot is passed as an argument.
///
/// This will also cause a memory leak if the underlying RocksDB snapshot is
/// not dropped by calling [StaticSnapshot::drop]. Unlike most uses of
/// [Snapshot], the RocksDB snapshot will not be dropped when the
/// [Snapshot] returned by this method is dropped.
pub unsafe fn with_db<'a>(&self, db: &'a rocksdb::DB) -> Snapshot<'a> {
let db_ss = RocksDBSnapshot {
_db: db,
inner: self.inner,
};
let db_ss: rocksdb::Snapshot<'a> = std::mem::transmute(db_ss);
Snapshot {
ss: Some(db_ss),
tree: self.clone_tree(),
should_drop_ss: false,
}
}
/// Drops the [StaticSnapshot] and the underlying RocksDB snapshot.
///
/// # Safety
/// This function is unsafe because it results in the RocksDB snapshot being
/// dropped, which could lead to use-after-free bugs if there are still
/// references to the snapshot in other [Snapshot] or [StaticSnapshot]
/// instances. The caller must be sure this is the last remaining reference
/// before calling this method.
pub unsafe fn drop(mut self, db: &rocksdb::DB) {
let mut ss = self.with_db(db);
ss.should_drop_ss = true;
self.should_drop = true;
// the snapshot drop implementation is now called, which includes
// dropping the RocksDB snapshot
}
/// Clones the root node of the Merk tree into a new [Tree].
fn clone_tree(&self) -> Cell<Option<Tree>> {
let tree = self.tree.take().unwrap();
let tree_clone = Cell::new(Some(Tree::decode(
tree.key().to_vec(),
tree.encode().as_slice(),
)));
self.tree.set(Some(tree));
tree_clone
}
}
impl Drop for StaticSnapshot {
fn drop(&mut self) {
if !self.should_drop {
log::debug!("StaticSnapshot must be manually dropped");
}
}
}
impl Clone for StaticSnapshot {
fn clone(&self) -> Self {
Self {
tree: self.clone_tree(),
inner: self.inner,
should_drop: self.should_drop,
}
}
}
#[cfg(test)]
mod tests {
use std::mem::transmute;
use super::RocksDBSnapshot;
use crate::test_utils::TempMerk;
#[test]
fn rocksdb_snapshot_struct_format() {
assert_eq!(std::mem::size_of::<rocksdb::Snapshot>(), 16);
let merk = TempMerk::new().unwrap();
let exptected_db_ptr = merk.db() as *const _;
let ss = merk.db().snapshot();
let ss: RocksDBSnapshot = unsafe { transmute(ss) };
let db_ptr = ss._db as *const _;
assert_eq!(exptected_db_ptr, db_ptr);
}
}
================================================
FILE: src/owner.rs
================================================
use std::ops::{Deref, DerefMut};
/// A container type which holds a value that may be temporarily owned by a
/// consumer.
pub struct Owner<T> {
inner: Option<T>,
}
impl<T> Owner<T> {
/// Creates a new `Owner` which holds the given value.
pub fn new(value: T) -> Owner<T> {
Owner { inner: Some(value) }
}
/// Takes temporary ownership of the contained value by passing it to `f`.
/// The function must return a value of the same type (the same value, or a
/// new value to take its place).
///
/// # Example
/// ```
/// # use merk::owner::Owner;
/// # struct SomeType();
/// # impl SomeType {
/// # fn method_which_requires_ownership(self) -> SomeType { self }
/// # }
/// #
/// let mut owner = Owner::new(SomeType());
/// owner.own(|value| {
/// value.method_which_requires_ownership();
/// SomeType() // now give back a value of the same type
/// });
/// ```
pub fn own<F: FnOnce(T) -> T>(&mut self, f: F) {
let old_value = unwrap(self.inner.take());
let new_value = f(old_value);
self.inner = Some(new_value);
}
/// Takes temporary ownership of the contained value by passing it to `f`.
/// The function must return a value of the same type (the same value, or a
/// new value to take its place).
///
/// Like `own`, but uses a tuple return type which allows specifying a value
/// to return from the call to `own_return` for convenience.
///
/// # Example
/// ```
/// # use merk::owner::Owner;
/// let mut owner = Owner::new(123);
/// let doubled = owner.own_return(|n| (n, n * 2));
/// ```
pub fn own_return<R, F>(&mut self, f: F) -> R
where
R: Sized,
F: FnOnce(T) -> (T, R),
{
let old_value = unwrap(self.inner.take());
let (new_value, return_value) = f(old_value);
self.inner = Some(new_value);
return_value
}
/// Takes temporary ownership of the contained value by passing it to `f`.
/// The function must return a value of the same type (the same value, or a
/// new value to take its place).
///
/// Like `own`, but with a fallible operation.
///
/// # Example
/// ```
/// # use merk::owner::Owner;
/// # use std::convert::TryFrom;
/// let mut owner = Owner::new(123);
/// let converted = owner.own_fallible(|n| u32::try_from(n));
/// ```
pub fn own_fallible<E, F: FnOnce(T) -> Result<T, E>>(&mut self, f: F) -> Result<(), E> {
let old_value = unwrap(self.inner.take());
let new_value = f(old_value)?;
self.inner = Some(new_value);
Ok(())
}
/// Sheds the `Owner` container and returns the value it contained.
pub fn into_inner(mut self) -> T {
unwrap(self.inner.take())
}
}
impl<T> Deref for Owner<T> {
type Target = T;
fn deref(&self) -> &T {
unwrap(self.inner.as_ref())
}
}
impl<T> DerefMut for Owner<T> {
fn deref_mut(&mut self) -> &mut T {
unwrap(self.inner.as_mut())
}
}
fn unwrap<T>(option: Option<T>) -> T {
match option {
Some(value) => value,
None => unreachable!("value should be Some"),
}
}
// TODO: unit tests
================================================
FILE: src/proofs/chunk.rs
================================================
#[cfg(feature = "full")]
use {
super::tree::{execute, Tree as ProofTree},
crate::tree::Hash,
crate::tree::Tree,
rocksdb::DBRawIterator,
};
use super::{Node, Op};
use crate::error::{Error, Result};
use crate::tree::{Fetch, RefWalker};
/// The minimum number of layers the trunk will be guaranteed to have before
/// splitting into multiple chunks.
///
/// If the tree's height is less than double this value, the trunk should be
/// verified as a leaf chunk.
pub const MIN_TRUNK_HEIGHT: usize = 5;
impl<'a, S> RefWalker<'a, S>
where
S: Fetch + Sized + Send + Clone,
{
/// Generates a trunk proof by traversing the tree.
///
/// Returns a tuple containing the produced proof, and a boolean indicating
/// whether or not there will be more chunks to follow. If the chunk
/// contains the entire tree, the boolean will be `false`, if the chunk
/// is abdriged and will be connected to leaf chunks, it will be `true`.
pub fn create_trunk_proof(&mut self) -> Result<(Vec<Op>, bool)> {
let approx_size = 2usize.pow((self.tree().height() / 2) as u32) * 3;
let mut proof = Vec::with_capacity(approx_size);
let trunk_height = self.traverse_for_height_proof(&mut proof, 1)?;
if trunk_height < MIN_TRUNK_HEIGHT {
proof.clear();
self.traverse_for_trunk(&mut proof, usize::MAX, true)?;
Ok((proof, false))
} else {
self.traverse_for_trunk(&mut proof, trunk_height, true)?;
Ok((proof, true))
}
}
/// Traverses down the left edge of the tree and pushes ops to the proof, to
/// act as a proof of the height of the tree. This is the first step in
/// generating a trunk proof.
fn traverse_for_height_proof(&mut self, proof: &mut Vec<Op>, depth: usize) -> Result<usize> {
let maybe_left = self.walk(true)?;
let has_left_child = maybe_left.is_some();
let trunk_height = if let Some(mut left) = maybe_left {
left.traverse_for_height_proof(proof, depth + 1)?
} else {
depth / 2
};
if depth > trunk_height {
proof.push(Op::Push(self.to_kvhash_node()));
if has_left_child {
proof.push(Op::Parent);
}
if let Some(right) = self.tree().link(false) {
proof.push(Op::Push(Node::Hash(*right.hash())));
proof.push(Op::Child);
}
}
Ok(trunk_height)
}
/// Traverses down the tree and adds KV push ops for all nodes up to a
/// certain depth. This expects the proof to contain a height proof as
/// generated by `traverse_for_height_proof`.
fn traverse_for_trunk(
&mut self,
proof: &mut Vec<Op>,
remaining_depth: usize,
is_leftmost: bool,
) -> Result<()> {
if remaining_depth == 0 {
// return early if we have reached bottom of trunk
// for leftmost node, we already have height proof
if is_leftmost {
return Ok(());
}
// add this node's hash
proof.push(Op::Push(self.to_hash_node()));
return Ok(());
}
// traverse left
let has_left_child = self.tree().link(true).is_some();
if has_left_child {
let mut left = self.walk(true)?.unwrap();
left.traverse_for_trunk(proof, remaining_depth - 1, is_leftmost)?;
}
// add this node's data
proof.push(Op::Push(self.to_kv_node()));
if has_left_child {
proof.push(Op::Parent);
}
// traverse right
if let Some(mut right) = self.walk(false)? {
right.traverse_for_trunk(proof, remaining_depth - 1, false)?;
proof.push(Op::Child);
}
Ok(())
}
}
/// Builds a chunk proof by iterating over values in a RocksDB, ending the chunk
/// when a node with key `end_key` is encountered.
///
/// Advances the iterator for all nodes in the chunk and the `end_key` (if any).
#[cfg(feature = "full")]
pub(crate) fn get_next_chunk(iter: &mut DBRawIterator, end_key: Option<&[u8]>) -> Result<Vec<Op>> {
let mut chunk = Vec::with_capacity(512);
let mut stack = Vec::with_capacity(32);
let mut node = Tree::new(vec![], vec![])?;
while iter.valid() {
let key = iter.key().unwrap();
if let Some(end_key) = end_key {
if key == end_key {
break;
}
}
let encoded_node = iter.value().unwrap();
Tree::decode_into(&mut node, vec![], encoded_node);
let kv = Node::KV(key.to_vec(), node.value().to_vec());
chunk.push(Op::Push(kv));
if node.link(true).is_some() {
chunk.push(Op::Parent);
}
if let Some(child) = node.link(false) {
stack.push(child.key().to_vec());
} else {
while let Some(top_key) = stack.last() {
if key < top_key.as_slice() {
break;
}
stack.pop();
chunk.push(Op::Child);
}
}
iter.next();
}
if iter.valid() {
iter.next();
}
Ok(chunk)
}
/// Verifies a leaf chunk proof by executing its operators. Checks that there
/// were no abridged nodes (Hash or KVHash) and the proof hashes to
/// `expected_hash`.
#[cfg(feature = "full")]
pub(crate) fn verify_leaf<I: Iterator<Item = Result<Op>>>(
ops: I,
expected_hash: Hash,
) -> Result<ProofTree> {
let tree = execute(ops, false, |node| match node {
Node::KV(_, _) => Ok(()),
_ => Err(Error::Tree("Leaf chunks must contain full subtree".into())),
})?;
if tree.hash()? != expected_hash {
return Err(Error::HashMismatch(expected_hash, tree.hash()?));
}
Ok(tree)
}
/// Verifies a trunk chunk proof by executing its operators. Ensures the
/// resulting tree contains a valid height proof, the trunk is the correct
/// height, and all of its inner nodes are not abridged. Returns the tree and
/// the height given by the height proof.
#[cfg(feature = "full")]
pub(crate) fn verify_trunk<I: Iterator<Item = Result<Op>>>(ops: I) -> Result<(ProofTree, usize)> {
fn verify_height_proof(tree: &ProofTree) -> Result<usize> {
let mut height = 1;
let mut cursor = tree;
while let Some(child) = cursor.child(true) {
if let Node::Hash(_) = child.tree.node {
return Err(Error::UnexpectedNode(
"Expected height proof to only contain KV and KVHash
nodes"
.into(),
));
}
height += 1;
cursor = &child.tree;
}
Ok(height)
}
fn verify_completeness(tree: &ProofTree, remaining_depth: usize, leftmost: bool) -> Result<()> {
let recurse = |left, leftmost| {
if let Some(child) = tree.child(left) {
verify_completeness(&child.tree, remaining_depth - 1, left && leftmost)?;
}
Ok(())
};
if remaining_depth > 0 {
match tree.node {
Node::KV(_, _) => {}
_ => {
return Err(Error::UnexpectedNode(
"Expected trunk inner nodes to contain keys and values".into(),
));
}
}
recurse(true, leftmost)?;
recurse(false, false)
} else if !leftmost {
match tree.node {
Node::Hash(_) => Ok(()),
_ => Err(Error::UnexpectedNode(
"Expected trunk leaves to contain Hash nodes".into(),
)),
}
} else {
match &tree.node {
Node::KVHash(_) => Ok(()),
_ => Err(Error::UnexpectedNode(
"Expected leftmost trunk leaf to contain KVHash node".into(),
)),
}
}
}
let mut kv_only = true;
let tree = execute(ops, false, |node| {
kv_only &= matches!(node, Node::KV(_, _));
Ok(())
})?;
let height = verify_height_proof(&tree)?;
if height > 64 {
// This is a sanity check to prevent stack overflows in `verify_completeness`,
// but any tree above 64 is probably an error (~3.7e19 nodes).
return Err(Error::Tree("Tree is too large".into()));
}
let trunk_height = height / 2;
if trunk_height < MIN_TRUNK_HEIGHT {
if !kv_only {
return Err(Error::Tree("Leaf chunks must contain full subtree".into()));
}
} else {
verify_completeness(&tree, trunk_height, true)?;
}
Ok((tree, height))
}
#[cfg(test)]
mod tests {
use super::super::tree::Tree;
use super::*;
use crate::test_utils::*;
use crate::tree::{NoopCommit, PanicSource, Tree as BaseTree};
use ed::Encode;
#[derive(Default)]
struct NodeCounts {
hash: usize,
kvhash: usize,
kv: usize,
}
fn count_node_types(tree: Tree) -> NodeCounts {
let mut counts = NodeCounts::default();
tree.visit_nodes(&mut |node| {
match node {
Node::Hash(_) => counts.hash += 1,
Node::KVHash(_) => counts.kvhash += 1,
Node::KV(_, _) => counts.kv += 1,
};
});
counts
}
#[test]
fn small_trunk_roundtrip() {
let mut tree = make_tree_seq(31);
let mut walker = RefWalker::new(&mut tree, PanicSource {});
let (proof, has_more) = walker.create_trunk_proof().unwrap();
assert!(!has_more);
println!("{:?}", &proof);
let (trunk, _) = verify_trunk(proof.into_iter().map(Ok)).unwrap();
let counts = count_node_types(trunk);
assert_eq!(counts.hash, 0);
assert_eq!(counts.kv, 32);
assert_eq!(counts.kvhash, 0);
}
#[test]
fn big_trunk_roundtrip() {
let mut tree = make_tree_seq(2u64.pow(MIN_TRUNK_HEIGHT as u32 * 2 + 1) - 1);
let mut walker = RefWalker::new(&mut tree, PanicSource {});
let (proof, has_more) = walker.create_trunk_proof().unwrap();
assert!(has_more);
let (trunk, _) = verify_trunk(proof.into_iter().map(Ok)).unwrap();
let counts = count_node_types(trunk);
// are these formulas correct for all values of `MIN_TRUNK_HEIGHT`? 🤔
assert_eq!(
counts.hash,
2usize.pow(MIN_TRUNK_HEIGHT as u32) + MIN_TRUNK_HEIGHT - 1
);
assert_eq!(counts.kv, 2usize.pow(MIN_TRUNK_HEIGHT as u32) - 1);
assert_eq!(counts.kvhash, MIN_TRUNK_HEIGHT + 1);
}
#[test]
fn one_node_tree_trunk_roundtrip() -> Result<()> {
let mut tree = BaseTree::new(vec![0], vec![])?;
tree.commit(&mut NoopCommit {}).unwrap();
let mut walker = RefWalker::new(&mut tree, PanicSource {});
let (proof, has_more) = walker.create_trunk_proof().unwrap();
assert!(!has_more);
let (trunk, _) = verify_trunk(proof.into_iter().map(Ok)).unwrap();
let counts = count_node_types(trunk);
assert_eq!(counts.hash, 0);
assert_eq!(counts.kv, 1);
assert_eq!(counts.kvhash, 0);
Ok(())
}
#[test]
fn two_node_right_heavy_tree_trunk_roundtrip() -> Result<()> {
// 0
// \
// 1
let mut tree =
BaseTree::new(vec![0], vec![])?.attach(false, Some(BaseTree::new(vec![1], vec![])?));
tree.commit(&mut NoopCommit {}).unwrap();
let mut walker = RefWalker::new(&mut tree, PanicSource {});
let (proof, has_more) = walker.create_trunk_proof().unwrap();
assert!(!has_more);
let (trunk, _) = verify_trunk(proof.into_iter().map(Ok)).unwrap();
let counts = count_node_types(trunk);
assert_eq!(counts.hash, 0);
assert_eq!(counts.kv, 2);
assert_eq!(counts.kvhash, 0);
Ok(())
}
#[test]
fn two_node_left_heavy_tree_trunk_roundtrip() -> Result<()> {
// 1
// /
// 0
let mut tree =
BaseTree::new(vec![1], vec![])?.attach(true, Some(BaseTree::new(vec![0], vec![])?));
tree.commit(&mut NoopCommit {}).unwrap();
let mut walker = RefWalker::new(&mut tree, PanicSource {});
let (proof, has_more) = walker.create_trunk_proof().unwrap();
assert!(!has_more);
let (trunk, _) = verify_trunk(proof.into_iter().map(Ok)).unwrap();
let counts = count_node_types(trunk);
assert_eq!(counts.hash, 0);
assert_eq!(counts.kv, 2);
assert_eq!(counts.kvhash, 0);
Ok(())
}
#[test]
fn three_node_tree_trunk_roundtrip() -> Result<()> {
// 1
// / \
// 0 2
let mut tree = BaseTree::new(vec![1], vec![])?
.attach(true, Some(BaseTree::new(vec![0], vec![])?))
.attach(false, Some(BaseTree::new(vec![2], vec![])?));
tree.commit(&mut NoopCommit {}).unwrap();
let mut walker = RefWalker::new(&mut tree, PanicSource {});
let (proof, has_more) = walker.create_trunk_proof().unwrap();
assert!(!has_more);
let (trunk, _) = verify_trunk(proof.into_iter().map(Ok)).unwrap();
let counts = count_node_types(trunk);
assert_eq!(counts.hash, 0);
assert_eq!(counts.kv, 3);
assert_eq!(counts.kvhash, 0);
Ok(())
}
#[test]
fn leaf_chunk_roundtrip() {
let mut merk = TempMerk::new().unwrap();
let batch = make_batch_seq(0..31);
merk.apply(batch.as_slice(), &[]).unwrap();
let root_node = merk.tree.read().unwrap();
let root_key = root_node.as_ref().unwrap().key().to_vec();
// whole tree as 1 leaf
let mut iter = merk.db.raw_iterator();
iter.seek_to_first();
let chunk = get_next_chunk(&mut iter, None).unwrap();
let ops = chunk.into_iter().map(Ok);
let chunk = verify_leaf(ops, merk.root_hash()).unwrap();
let counts = count_node_types(chunk);
assert_eq!(counts.kv, 31);
assert_eq!(counts.hash, 0);
assert_eq!(counts.kvhash, 0);
drop(iter);
let mut iter = merk.db.raw_iterator();
iter.seek_to_first();
// left leaf
let chunk = get_next_chunk(&mut iter, Some(root_key.as_slice())).unwrap();
let ops = chunk.into_iter().map(Ok);
let chunk = verify_leaf(
ops,
[
222, 93, 128, 149, 117, 136, 34, 175, 204, 82, 228, 113, 242, 144, 152, 190, 210,
27, 195, 34, 24, 196, 210, 99, 250, 119, 219, 114, 52, 167, 191, 249,
],
)
.unwrap();
let counts = count_node_types(chunk);
assert_eq!(counts.kv, 15);
assert_eq!(counts.hash, 0);
assert_eq!(counts.kvhash, 0);
// right leaf
let chunk = get_next_chunk(&mut iter, None).unwrap();
let ops = chunk.into_iter().map(Ok);
let chunk = verify_leaf(
ops,
[
128, 158, 92, 80, 118, 253, 48, 241, 74, 154, 213, 187, 92, 243, 154, 28, 164, 235,
156, 122, 174, 226, 84, 170, 233, 166, 27, 79, 100, 10, 88, 184,
],
)
.unwrap();
let counts = count_node_types(chunk);
assert_eq!(counts.kv, 15);
assert_eq!(counts.hash, 0);
assert_eq!(counts.kvhash, 0);
}
#[test]
#[should_panic(expected = "Tree is too large")]
fn test_verify_height_stack_overflow() {
let height = 5_000u32;
let push_op = |i: u32| Op::Push(Node::KV(i.to_be_bytes().to_vec(), vec![]));
let mut ops = Vec::with_capacity((height * 2) as usize);
ops.push(push_op(0));
for i in 1..height {
ops.push(push_op(i));
ops.push(Op::Parent)
}
assert!(ops.encoding_length().unwrap() < 50_000);
println!("Len: {}", ops.encoding_length().unwrap());
let (_, result_height) = verify_trunk(ops.into_iter().map(Ok)).unwrap();
assert_eq!(height, result_height as u32);
}
}
================================================
FILE: src/proofs/encoding.rs
================================================
use std::io::{Read, Write};
use ed::{Decode, Encode, Terminated};
use super::{Node, Op};
use crate::error::Result;
use crate::tree::HASH_LENGTH;
impl Encode for Op {
fn encode_into<W: Write>(&self, dest: &mut W) -> ed::Result<()> {
match self {
Op::Push(Node::Hash(hash)) => {
dest.write_all(&[0x01])?;
dest.write_all(hash)?;
}
Op::Push(Node::KVHash(kv_hash)) => {
dest.write_all(&[0x02])?;
dest.write_all(kv_hash)?;
}
Op::Push(Node::KV(key, value)) => {
debug_assert!(key.len() < 65536);
debug_assert!(value.len() < 65536);
dest.write_all(&[0x03])?;
(key.len() as u16).encode_into(dest)?;
dest.write_all(key)?;
(value.len() as u16).encode_into(dest)?;
dest.write_all(value)?;
}
Op::Parent => dest.write_all(&[0x10])?,
Op::Child => dest.write_all(&[0x11])?,
};
Ok(())
}
fn encoding_length(&self) -> ed::Result<usize> {
Ok(match self {
Op::Push(Node::Hash(_)) => 1 + HASH_LENGTH,
Op::Push(Node::KVHash(_)) => 1 + HASH_LENGTH,
Op::Push(Node::KV(key, value)) => 5 + key.len() + value.len(),
Op::Parent => 1,
Op::Child => 1,
})
}
}
impl Decode for Op {
fn decode<R: Read>(mut input: R) -> ed::Result<Self> {
let variant: u8 = Decode::decode(&mut input)?;
Ok(match variant {
0x01 => {
let mut hash = [0; HASH_LENGTH];
input.read_exact(&mut hash)?;
Op::Push(Node::Hash(hash))
}
0x02 => {
let mut hash = [0; HASH_LENGTH];
input.read_exact(&mut hash)?;
Op::Push(Node::KVHash(hash))
}
0x03 => {
let key_len: u16 = Decode::decode(&mut input)?;
let mut key = vec![0; key_len as usize];
input.read_exact(key.as_mut_slice())?;
let value_len: u16 = Decode::decode(&mut input)?;
let mut value = vec![0; value_len as usize];
input.read_exact(value.as_mut_slice())?;
Op::Push(Node::KV(key, value))
}
0x10 => Op::Parent,
0x11 => Op::Child,
byte => {
return Err(ed::Error::UnexpectedByte(byte));
}
})
}
}
impl Terminated for Op {}
impl Op {
fn encode_into<W: Write>(&self, dest: &mut W) -> Result<()> {
Ok(Encode::encode_into(self, dest)?)
}
fn encoding_length(&self) -> usize {
Encode::encoding_length(self).unwrap()
}
pub fn decode(bytes: &[u8]) -> Result<Self> {
Ok(Decode::decode(bytes)?)
}
}
pub fn encode_into<'a, T: Iterator<Item = &'a Op>>(ops: T, output: &mut Vec<u8>) {
for op in ops {
op.encode_into(output).unwrap();
}
}
pub struct Decoder<'a> {
offset: usize,
bytes: &'a [u8],
}
impl<'a> Decoder<'a> {
pub fn new(proof_bytes: &'a [u8]) -> Self {
Decoder {
offset: 0,
bytes: proof_bytes,
}
}
}
impl<'a> Iterator for Decoder<'a> {
type Item = Result<Op>;
fn next(&mut self) -> Option<Self::Item> {
if self.offset >= self.bytes.len() {
return None;
}
Some((|| {
let bytes = &self.bytes[self.offset..];
let op = Op::decode(bytes)?;
self.offset += op.encoding_length();
Ok(op)
})())
}
}
#[cfg(test)]
mod test {
use super::super::{Node, Op};
use crate::tree::HASH_LENGTH;
#[test]
fn encode_push_hash() {
let op = Op::Push(Node::Hash([123; HASH_LENGTH]));
assert_eq!(op.encoding_length(), 1 + HASH_LENGTH);
let mut bytes = vec![];
op.encode_into(&mut bytes).unwrap();
assert_eq!(
bytes,
vec![
0x01, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123
]
);
}
#[test]
fn encode_push_kvhash() {
let op = Op::Push(Node::KVHash([123; HASH_LENGTH]));
assert_eq!(op.encoding_length(), 1 + HASH_LENGTH);
let mut bytes = vec![];
op.encode_into(&mut bytes).unwrap();
assert_eq!(
bytes,
vec![
0x02, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123
]
);
}
#[test]
fn encode_push_kv() {
let op = Op::Push(Node::KV(vec![1, 2, 3], vec![4, 5, 6]));
assert_eq!(op.encoding_length(), 11);
let mut bytes = vec![];
op.encode_into(&mut bytes).unwrap();
assert_eq!(bytes, vec![0x03, 0, 3, 1, 2, 3, 0, 3, 4, 5, 6]);
}
#[test]
fn encode_parent() {
let op = Op::Parent;
assert_eq!(op.encoding_length(), 1);
let mut bytes = vec![];
op.encode_into(&mut bytes).unwrap();
assert_eq!(bytes, vec![0x10]);
}
#[test]
fn encode_child() {
let op = Op::Child;
assert_eq!(op.encoding_length(), 1);
let mut bytes = vec![];
op.encode_into(&mut bytes).unwrap();
assert_eq!(bytes, vec![0x11]);
}
#[test]
#[should_panic]
fn encode_push_kv_long_key() {
let op = Op::Push(Node::KV(vec![123; 70_000], vec![4, 5, 6]));
let mut bytes = vec![];
op.encode_into(&mut bytes).unwrap();
}
#[test]
fn decode_push_hash() {
let bytes = [
0x01, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
];
let op = Op::decode(&bytes[..]).expect("decode failed");
assert_eq!(op, Op::Push(Node::Hash([123; HASH_LENGTH])));
}
#[test]
fn decode_push_kvhash() {
let bytes = [
0x02, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
];
let op = Op::decode(&bytes[..]).expect("decode failed");
assert_eq!(op, Op::Push(Node::KVHash([123; HASH_LENGTH])));
}
#[test]
fn decode_push_kv() {
let bytes = [0x03, 0, 3, 1, 2, 3, 0, 3, 4, 5, 6];
let op = Op::decode(&bytes[..]).expect("decode failed");
assert_eq!(op, Op::Push(Node::KV(vec![1, 2, 3], vec![4, 5, 6])));
}
#[test]
fn decode_parent() {
let bytes = [0x10];
let op = Op::decode(&bytes[..]).expect("decode failed");
assert_eq!(op, Op::Parent);
}
#[test]
fn decode_child() {
let bytes = [0x11];
let op = Op::decode(&bytes[..]).expect("decode failed");
assert_eq!(op, Op::Child);
}
#[test]
fn decode_unknown() {
let bytes = [0x88];
assert!(Op::decode(&bytes[..]).is_err());
}
}
================================================
FILE: src/proofs/mod.rs
================================================
pub mod chunk;
pub mod encoding;
pub mod query;
pub mod tree;
use crate::tree::Hash;
pub use encoding::{encode_into, Decoder};
pub use query::Query;
pub use tree::Tree;
/// A proof operator, executed to verify the data in a Merkle proof.
#[derive(Debug, PartialEq)]
pub enum Op {
/// Pushes a node on the stack.
Push(Node),
/// Pops the top stack item as `parent`. Pops the next top stack item as
/// `child`. Attaches `child` as the left child of `parent`. Pushes the
/// updated `parent` back on the stack.
Parent,
/// Pops the top stack item as `child`. Pops the next top stack item as
/// `parent`. Attaches `child` as the right child of `parent`. Pushes the
/// updated `parent` back on the stack.
Child,
}
/// A selected piece of data about a single tree node, to be contained in a
/// `Push` operator in a proof.
#[derive(Clone, Debug, PartialEq)]
pub enum Node {
/// Represents the hash of a tree node.
Hash(Hash),
/// Represents the hash of the key/value pair of a tree node.
KVHash(Hash),
/// Represents the key and value of a tree node.
KV(Vec<u8>, Vec<u8>),
}
================================================
FILE: src/proofs/query/map.rs
================================================
use super::super::Node;
use crate::{Error, Result};
use std::collections::btree_map;
use std::collections::BTreeMap;
use std::iter::Peekable;
use std::ops::{Bound, RangeBounds};
/// `MapBuilder` allows a consumer to construct a `Map` by inserting the nodes
/// contained in a proof, in key-order.
pub(crate) struct MapBuilder(Map);
impl MapBuilder {
/// Creates a new `MapBuilder` with an empty internal `Map`.
pub fn new() -> Self {
MapBuilder(Map {
entries: Default::default(),
right_edge: true,
})
}
/// Adds the node's data to the uncerlying `Map` (if node is type `KV`), or
/// makes a note of non-contiguous data (if node is type `KVHash` or
/// `Hash`).
pub fn insert(&mut self, node: &Node) -> Result<()> {
match node {
Node::KV(key, value) => {
if let Some((prev_key, _)) = self.0.entries.last_key_value() {
if key <= prev_key {
return Err(Error::Key(
"Expected nodes to be in increasing key order".into(),
));
}
}
let value = (self.0.right_edge, value.clone());
self.0.entries.insert(key.clone(), value);
self.0.right_edge = true;
}
_ => self.0.right_edge = false,
}
Ok(())
}
/// Consumes the `MapBuilder` and returns its internal `Map`.
pub fn build(self) -> Map {
self.0
}
}
/// `Map` stores data extracted from a proof.
///
/// The data (which has already been verified against a known root hash) can be
/// accessed by a consumer by looking up individual keys using the `get` method,
/// or iterating over ranges using the `range` method.
#[derive(Clone, Debug)]
pub struct Map {
entries: BTreeMap<Vec<u8>, (bool, Vec<u8>)>,
right_edge: bool,
}
impl Map {
/// Gets the value for a single key, or `None` if the key was proven to not
/// exist in the tree. If the proof does not include the data and also does
/// not prove that the key is absent in the tree (meaning the proof is not
/// valid), an error will be returned.
pub fn get<'a>(&'a self, key: &'a [u8]) -> Result<Option<&'a [u8]>> {
// if key is in proof just get from entries
if let Some((_, value)) = self.entries.get(key) {
return Ok(Some(value.as_slice()));
}
// otherwise, use range which only includes exact key match to check
// absence proof
let entry = self
.range((Bound::Included(key), Bound::Included(key)))
.next()
.transpose()?
.map(|(_, value)| value);
Ok(entry)
}
/// Returns an iterator over all (key, value) entries in the requested range
/// of keys. If during iteration we encounter a gap in the data (e.g. the
/// proof did not include all nodes within the range), the iterator will
/// yield an error.
pub fn range<'a>(&self, bounds: impl RangeBounds<&'a [u8]>) -> Range {
let start_bound = bound_to_inner(bounds.start_bound());
let end_bound = bound_to_inner(bounds.end_bound());
let outer_bounds = (
start_bound.map_or(Bound::Unbounded, |k| {
self.entries
.range(..=k.to_vec())
.next_back()
.map_or(Bound::Unbounded, |prev| Bound::Included(prev.0.clone()))
}),
end_bound.map_or(Bound::Unbounded, |k| {
self.entries
.range(k.to_vec()..)
.next()
.map_or(Bound::Unbounded, |next| Bound::Included(next.0.clone()))
}),
);
Range {
map: self,
bounds: bounds_to_vec(bounds),
done: false,
iter: self.entries.range(outer_bounds).peekable(),
}
}
/// Joins two `Map`s together, combining the data in both.
///
/// If the maps contain contiguous iteration ranges, the contiguous ranges
/// will be joined. If the maps have differing values for the same key, this
/// will panic (this should never happen if the queries came from the same
/// root and the proofs were verified).
pub fn join(self, other: Map) -> Map {
// TODO: join at the partial tree level, joining with only Map data means
// data from different joins which happen to be contiguous (without explicitly
// querying based on next/prev) will be marked as non-contiguous
let mut entries = self.entries.clone();
entries.extend(other.entries);
for (key, (contiguous, val)) in entries.iter_mut() {
if let Some(shadowed) = self.entries.get(key) {
assert_eq!(val, &shadowed.1, "Maps have different values",);
*contiguous = *contiguous || shadowed.0;
}
}
Map {
entries,
right_edge: self.right_edge || other.right_edge,
}
}
/// Returns `true` if the [Map] can verify that there is no unproven data
/// between `key` and the node to its right (or the global tree edge).
///
/// For example, if the underlying tree contains the key `[a, b, c, d]` and
/// the map contains the keys `[a, b, d]`, then `contiguous_right(a)` will
/// return `true`, `contiguous_right(b)` and `contiguous_right(c)` will
/// return `false`, and `contiguous_right(d)` will return `true`.
fn contiguous_right(&self, key: &[u8]) -> bool {
self.entries
.range((Bound::Excluded(key.to_vec()), Bound::Unbounded))
.next()
.map_or(self.right_edge, |(_, (contiguous, _))| *contiguous)
}
}
/// Returns `None` for `Bound::Unbounded`, or the inner key value for
/// `Bound::Included` and `Bound::Excluded`.
fn bound_to_inner<T>(bound: Bound<T>) -> Option<T> {
match bound {
Bound::Unbounded => None,
Bound::Included(key) | Bound::Excluded(key) => Some(key),
}
}
/// Converts the inner key value of a `Bound` from a byte slice to a `Vec<u8>`.
fn bound_to_vec(bound: Bound<&&[u8]>) -> Bound<Vec<u8>> {
match bound {
Bound::Unbounded => Bound::Unbounded,
Bound::Excluded(k) => Bound::Excluded(k.to_vec()),
Bound::Included(k) => Bound::Included(k.to_vec()),
}
}
/// Converts the inner key values of a [RangeBounds] from byte slices to
/// `Vec<u8>`.
fn bounds_to_vec<'a, R: RangeBounds<&'a [u8]>>(bounds: R) -> (Bound<Vec<u8>>, Bound<Vec<u8>>) {
(
bound_to_vec(bounds.start_bound()),
bound_to_vec(bounds.end_bound()),
)
}
/// An iterator over (key, value) entries as extracted from a verified proof.
///
/// If during iteration we encounter a gap in the data (e.g. the proof did not
/// include all nodes within the range), the iterator will yield an error.
pub struct Range<'a> {
map: &'a Map,
bounds: (Bound<Vec<u8>>, Bound<Vec<u8>>),
done: bool,
iter: Peekable<InnerRange<'a>>,
}
type InnerRange<'a> = btree_map::Range<'a, Vec<u8>, (bool, Vec<u8>)>;
impl<'a> Range<'a> {
fn yield_entry_if_contiguous(
&mut self,
entry: (&'a Vec<u8>, &'a (bool, Vec<u8>)),
contiguous: bool,
forward: bool,
) -> Option<Result<(&'a [u8], &'a [u8])>> {
if !contiguous {
self.done = true;
return Some(Err(Error::MissingData));
}
self.yield_entry(entry, forward)
}
fn yield_entry(
&mut self,
entry: (&'a Vec<u8>, &'a (bool, Vec<u8>)),
forward: bool,
) -> Option<Result<(&'a [u8], &'a [u8])>> {
let (key, (_, value)) = entry;
if forward {
self.bounds.0 = Bound::Excluded(key.clone());
} else {
self.bounds.1 = Bound::Excluded(key.clone());
}
Some(Ok((key.as_slice(), value.as_slice())))
}
fn yield_none_if_contiguous(
&mut self,
contiguous: bool,
) -> Option<Result<(&'a [u8], &'a [u8])>> {
self.done = true;
if !contiguous {
return Some(Err(Error::MissingData));
}
None
}
fn yield_next_if_contiguous(&mut self) -> Option<Result<(&'a [u8], &'a [u8])>> {
if let Some((_, (contiguous, _))) = self.iter.peek() {
if !contiguous {
self.done = true;
return Some(Err(Error::MissingData));
}
}
self.next()
}
fn yield_next_back_if_contiguous(
&mut self,
contiguous: bool,
) -> Option<Result<(&'a [u8], &'a [u8])>> {
if !contiguous {
self.done = true;
return Some(Err(Error::MissingData));
}
self.next_back()
}
}
impl<'a> Iterator for Range<'a> {
type Item = Result<(&'a [u8], &'a [u8])>;
fn next(&mut self) -> Option<Self::Item> {
if self.done {
return None;
}
let entry = match self.iter.next() {
None => return self.yield_none_if_contiguous(self.map.right_edge),
Some(entry) => entry,
};
let (key, (contiguous, _)) = entry;
let past_start = match bound_to_inner(self.bounds.0.clone()) {
None => true,
Some(ref start_bound) => key > start_bound,
};
let at_start = match self.bounds.0 {
Bound::Unbounded => true,
Bound::Included(ref start_bound) => key == start_bound,
Bound::Excluded(_) => false,
};
let past_end = match self.bounds.1 {
Bound::Unbounded => false,
Bound::Included(ref end_bound) => key > end_bound,
Bound::Excluded(ref end_bound) => key >= end_bound,
};
if past_end {
self.yield_none_if_contiguous(*contiguous)
} else if past_start {
self.yield_entry_if_contiguous(entry, *contiguous, true)
} else if at_start {
self.yield_entry(entry, true)
} else {
self.yield_next_if_contiguous()
}
}
}
impl<'a> DoubleEndedIterator for Range<'a> {
fn next_back(&mut self) -> Option<Self::Item> {
if self.done {
return None;
}
let entry = match self.iter.next_back() {
None => return self.yield_none_if_contiguous(self.map.contiguous_right(&[])),
Some(entry) => entry,
};
let (key, (contiguous_l, _)) = entry;
let contiguous_r = self.map.contiguous_right(key);
let past_end = match bound_to_inner(self.bounds.1.clone()) {
None => true,
Some(ref end_bound) => key < end_bound,
};
let at_end = match self.bounds.1 {
Bound::Unbounded => true,
Bound::Included(ref end_bound) => key == end_bound,
Bound::Excluded(_) => false,
};
let past_start = match self.bounds.0 {
Bound::Unbounded => false,
Bound::Included(ref start_bound) => key < start_bound,
Bound::Excluded(ref start_bound) => key <= start_bound,
};
if past_start {
self.yield_none_if_contiguous(contiguous_r)
} else if past_end {
self.yield_entry_if_contiguous(entry, contiguous_r, false)
} else if at_end {
self.yield_entry(entry, false)
} else {
self.yield_next_back_if_contiguous(*contiguous_l)
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::HASH_LENGTH;
#[test]
#[should_panic(expected = "Expected nodes to be in increasing key order")]
fn mapbuilder_insert_out_of_order() {
let mut builder = MapBuilder::new();
builder.insert(&Node::KV(vec![1, 2, 3], vec![])).unwrap();
builder.insert(&Node::KV(vec![1, 2, 2], vec![])).unwrap();
}
#[test]
#[should_panic(expected = "Expected nodes to be in increasing key order")]
fn mapbuilder_insert_dupe() {
let mut builder = MapBuilder::new();
builder.insert(&Node::KV(vec![1, 2, 3], vec![])).unwrap();
builder.insert(&Node::KV(vec![1, 2, 3], vec![])).unwrap();
}
#[test]
fn mapbuilder_insert_including_edge() {
let mut builder = MapBuilder::new();
builder.insert(&Node::Hash([0; HASH_LENGTH])).unwrap();
builder.insert(&Node::KV(vec![1, 2, 4], vec![])).unwrap();
assert!(builder.0.right_edge);
}
#[test]
fn mapbuilder_insert_abridged_edge() {
let mut builder = MapBuilder::new();
builder.insert(&Node::KV(vec![1, 2, 3], vec![])).unwrap();
builder.insert(&Node::Hash([0; HASH_LENGTH])).unwrap();
assert!(!builder.0.right_edge);
}
#[test]
fn mapbuilder_build() {
let mut builder = MapBuilder::new();
builder.insert(&Node::KV(vec![1, 2, 3], vec![1])).unwrap();
builder.insert(&Node::Hash([0; HASH_LENGTH])).unwrap();
builder.insert(&Node::KV(vec![1, 2, 4], vec![2])).unwrap();
let map = builder.build();
let mut entries = map.entries.iter();
assert_eq!(entries.next(), Some((&vec![1, 2, 3], &(true, vec![1]))));
assert_eq!(entries.next(), Some((&vec![1, 2, 4], &(false, vec![2]))));
assert_eq!(entries.next(), None);
assert!(map.right_edge);
}
#[test]
fn map_get_included() {
let mut builder = MapBuilder::new();
builder.insert(&Node::KV(vec![1, 2, 3], vec![1])).unwrap();
builder.insert(&Node::Hash([0; HASH_LENGTH])).unwrap();
builder.insert(&Node::KV(vec![1, 2, 4], vec![2])).unwrap();
let map = builder.build();
assert_eq!(map.get(&[1, 2, 3]).unwrap().unwrap(), vec![1],);
assert_eq!(map.get(&[1, 2, 4]).unwrap().unwrap(), vec![2],);
}
#[test]
#[should_panic(expected = "MissingData")]
fn map_get_missing_absence_proof() {
let mut builder = MapBuilder::new();
builder.insert(&Node::KV(vec![1, 2, 3], vec![1])).unwrap();
builder.insert(&Node::Hash([0; HASH_LENGTH])).unwrap();
builder.insert(&Node::KV(vec![1, 2, 4], vec![2])).unwrap();
let map = builder.build();
map.get(&[1, 2, 3, 4]).unwrap();
}
#[test]
fn map_get_valid_absence_proof() {
let mut builder = MapBuilder::new();
builder.insert(&Node::KV(vec![1, 2, 3], vec![1])).unwrap();
builder.insert(&Node::KV(vec![1, 2, 4], vec![2])).unwrap();
let map = builder.build();
assert!(map.get(&[1, 2, 3, 4]).unwrap().is_none());
}
#[test]
#[should_panic(expected = "MissingData")]
fn range_abridged() {
let mut builder = MapBuilder::new();
builder.insert(&Node::KV(vec![1, 2, 3], vec![1])).unwrap();
builder.insert(&Node::Hash([0; HASH_LENGTH])).unwrap();
builder.insert(&Node::KV(vec![1, 2, 4], vec![2])).unwrap();
let map = builder.build();
let mut range = map.range(&[1u8, 2, 3][..]..&[1u8, 2, 4][..]);
assert_eq!(range.next().unwrap().unwrap(), (&[1, 2, 3][..], &[1][..]));
range.next().unwrap().unwrap();
}
#[test]
fn range_ok() {
let mut builder = MapBuilder::new();
builder.insert(&Node::KV(vec![1, 2, 3], vec![1])).unwrap();
builder.insert(&Node::KV(vec![1, 2, 4], vec![2])).unwrap();
builder.insert(&Node::KV(vec![1, 2, 5], vec![3])).unwrap();
let map = builder.build();
let mut range = map.range(&[1u8, 2, 3][..]..&[1u8, 2, 5][..]);
assert_eq!(range.next().unwrap().unwrap(), (&[1, 2, 3][..], &[1][..]));
assert_eq!(range.next().unwrap().unwrap(), (&[1, 2, 4][..], &[2][..]));
assert!(range.next().is_none());
assert!(range.next_back().is_none());
assert!(range.next().is_none());
}
#[test]
fn range_empty() {
let map = MapBuilder::new().build();
let mut range = map.range(..);
assert!(range.next().is_none());
assert!(range.next_back().is_none());
}
#[test]
#[should_panic(expected = "MissingData")]
fn range_lower_unbounded_map_non_contiguous() {
let mut builder = MapBuilder::new();
builder.insert(&Node::KV(vec![1, 2, 3], vec![1])).unwrap();
builder.insert(&Node::Hash([1; HASH_LENGTH])).unwrap();
builder.insert(&Node::KV(vec![1, 2, 4], vec![1])).unwrap();
let map = builder.build();
let mut range = map.range(..&[1u8, 2, 5][..]);
range.next().unwrap().unwrap();
range.next().unwrap().unwrap();
}
#[test]
fn range_reach_proof_end() {
let mut builder = MapBuilder::new();
builder.insert(&Node::KV(vec![1, 2, 3], vec![1])).unwrap();
builder.insert(&Node::KV(vec![1, 2, 4], vec![2])).unwrap();
let map = builder.build();
let mut range = map.range(&[1u8, 2, 3][..]..);
assert_eq!(range.next().unwrap().unwrap(), (&[1, 2, 3][..], &[1][..]));
assert_eq!(range.next().unwrap().unwrap(), (&[1, 2, 4][..], &[2][..]));
assert!(range.next().is_none());
}
#[test]
fn range_unbounded() {
let mut builder = MapBuilder::new();
builder.insert(&Node::KV(vec![1, 2, 3], vec![1])).unwrap();
builder.insert(&Node::KV(vec![1, 2, 4], vec![2])).unwrap();
let map = builder.build();
let mut range = map.range(..);
assert_eq!(range.next().unwrap().unwrap(), (&[1, 2, 3][..], &[1][..]));
assert_eq!(range.next().unwrap().unwrap(), (&[1, 2, 4][..], &[2][..]));
assert!(range.next().is_none());
}
#[test]
#[should_panic(expected = "MissingData")]
fn range_abridged_rev() {
let mut builder = MapBuilder::new();
builder.insert(&Node::KV(vec![1, 2, 3], vec![1])).unwrap();
builder.insert(&Node::Hash([0; HASH_LENGTH])).unwrap();
builder.insert(&Node::KV(vec![1, 2, 4], vec![2])).unwrap();
let map = builder.build();
let mut range = map.range(&[1u8, 2, 3][..]..=&[1u8, 2, 4][..]).rev();
assert_eq!(range.next().unwrap().unwrap(), (&[1, 2, 4][..], &[2][..]));
range.next().unwrap().unwrap();
}
#[test]
fn range_ok_rev() {
let mut builder = MapBuilder::new();
builder.insert(&Node::KV(vec![1, 2, 3], vec![1])).unwrap();
builder.insert(&Node::KV(vec![1, 2, 4], vec![2])).unwrap();
builder.insert(&Node::KV(vec![1, 2, 5], vec![3])).unwrap();
let map = builder.build();
let mut range = map.range(&[1u8, 2, 3][..]..&[1u8, 2, 5][..]).rev();
assert_eq!(range.next().unwrap().unwrap(), (&[1, 2, 4][..], &[2][..]));
assert_eq!(range.next().unwrap().unwrap(), (&[1, 2, 3][..], &[1][..]));
assert!(range.next().is_none());
assert!(range.next_back().is_none());
}
#[test]
#[should_panic(expected = "MissingData")]
fn range_upper_unbounded_map_non_contiguous() {
let mut builder = MapBuilder::new();
builder.insert(&Node::KV(vec![1, 2, 3], vec![1])).unwrap();
builder.insert(&Node::Hash([1; HASH_LENGTH])).unwrap();
builder.insert(&Node::KV(vec![1, 2, 4], vec![1])).unwrap();
let map = builder.build();
let mut range = map.range(&[1u8, 2, 3][..]..).rev();
range.next().unwrap().unwrap();
range.next().unwrap().unwrap();
}
#[test]
fn range_reach_proof_end_rev() {
let mut builder = MapBuilder::new();
builder.insert(&Node::KV(vec![1, 2, 3], vec![1])).unwrap();
builder.insert(&Node::KV(vec![1, 2, 4], vec![2])).unwrap();
let map = builder.build();
let mut range = map.range(..&[1u8, 2, 5][..]).rev();
assert_eq!(range.next().unwrap().unwrap(), (&[1, 2, 4][..], &[2][..]));
assert_eq!(range.next().unwrap().unwrap(), (&[1, 2, 3][..], &[1][..]));
assert!(range.next().is_none());
}
#[test]
fn range_unbounded_rev() {
let mut builder = MapBuilder::new();
builder.insert(&Node::KV(vec![1, 2, 3], vec![1])).unwrap();
builder.insert(&Node::KV(vec![1, 2, 4], vec![2])).unwrap();
let map = builder.build();
let mut range = map.range(..).rev();
assert_eq!(range.next().unwrap().unwrap(), (&[1, 2, 4][..], &[2][..]));
assert_eq!(range.next().unwrap().unwrap(), (&[1, 2, 3][..], &[1][..]));
assert!(range.next().is_none());
}
#[test]
fn map_join() {
let mut builder = MapBuilder::new();
builder.insert(&Node::KV(vec![1], vec![1])).unwrap();
builder.insert(&Node::KV(vec![2], vec![1])).unwrap();
builder.insert(&Node::KV(vec![3], vec![1])).unwrap();
builder.insert(&Node::Hash([0; HASH_LENGTH])).unwrap();
builder.insert(&Node::KV(vec![5], vec![1])).unwrap();
let a = builder.build();
let mut builder = MapBuilder::new();
builder.insert(&Node::KV(vec![1], vec![1])).unwrap();
builder.insert(&Node::Hash([0; HASH_LENGTH])).unwrap();
builder.insert(&Node::KV(vec![3], vec![1])).unwrap();
builder.insert(&Node::KV(vec![4], vec![1])).unwrap();
builder.insert(&Node::Hash([0; HASH_LENGTH])).unwrap();
let b = builder.build();
let joined = a.join(b);
let mut range = joined.range(..=&[4][..]);
assert_eq!(range.next().unwrap().unwrap(), (&[1][..], &[1][..]));
assert_eq!(range.next().unwrap().unwrap(), (&[2][..], &[1][..]));
assert_eq!(range.next().unwrap().unwrap(), (&[3][..], &[1][..]));
assert_eq!(range.next().unwrap().unwrap(), (&[4][..], &[1][..]));
assert!(range.next().is_none());
let mut range = joined.range(&[5][..]..);
assert_eq!(range.next().unwrap().unwrap(), (&[5][..], &[1][..]));
assert!(range.next().is_none());
}
}
================================================
FILE: src/proofs/query/mod.rs
================================================
mod map;
#[cfg(feature = "full")]
use {super::Op, std::collections::LinkedList};
use super::tree::execute;
use super::{Decoder, Node};
use crate::error::{Error, Result};
use crate::tree::{Fetch, Hash, Link, RefWalker};
use std::cmp::{max, min, Ordering};
use std::collections::BTreeSet;
use std::ops::RangeInclusive;
pub use map::*;
/// `Query` represents one or more keys or ranges of keys, which can be used to
/// resolve a proof which will include all of the requested values.
#[derive(Default)]
pub struct Query {
items: BTreeSet<QueryItem>,
}
impl Query {
/// Creates a new query which contains no items.
pub fn new() -> Self {
Default::default()
}
pub(crate) fn len(&self) -> usize {
self.items.len()
}
pub(crate) fn iter(&self) -> impl Iterator<Item = &QueryItem> {
self.items.iter()
}
/// Adds an individual key to the query, so that its value (or its absence)
/// in the tree will be included in the resulting proof.
///
/// If the key or a range including the key already exists in the query,
/// this will have no effect. If the query already includes a range that has
/// a non-inclusive bound equal to the key, the bound will be changed to be
/// inclusive.
pub fn insert_key(&mut self, key: Vec<u8>) {
let key = QueryItem::Key(key);
self.items.insert(key);
}
/// Adds a range to the query, so that all the entries in the tree with keys
/// in the range will be included in the resulting proof.
///
/// If a range including the range already exists in the query, this will
/// have no effect. If the query already includes a range that overlaps with
/// the range, the ranges will be joined together.
pub fn insert_range(&mut self, range: std::ops::Range<Vec<u8>>) {
let range = QueryItem::Range(range);
self.insert_item(range);
}
/// Adds an inclusive range to the query, so that all the entries in the
/// tree with keys in the range will be included in the resulting proof.
///
/// If a range including the range already exists in the query, this will
/// have no effect. If the query already includes a range that overlaps with
/// the range, the ranges will be merged together.
pub fn insert_range_inclusive(&mut self, range: RangeInclusive<Vec<u8>>) {
let range = QueryItem::RangeInclusive(range);
self.insert_item(range);
}
/// Adds the `QueryItem` to the query, first checking to see if it collides
/// with any existing ranges or keys. All colliding items will be removed
/// then merged together so that the query includes the minimum number of
/// items (with no items covering any duplicate parts of keyspace) while
/// still including every key or range that has been added to the query.
pub fn insert_item(&mut self, mut item: QueryItem) {
// since `QueryItem::eq` considers items equal if they collide at all
// (including keys within ranges or ranges which partially overlap),
// `items.take` will remove the first item which collides
while let Some(existing) = self.items.take(&item) {
item = item.merge(existing);
}
self.items.insert(item);
}
}
impl<Q: Into<QueryItem>> From<Vec<Q>> for Query {
fn from(other: Vec<Q>) -> Self {
let items = other.into_iter().map(Into::into).collect();
Query { items }
}
}
impl From<Query> for Vec<QueryItem> {
fn from(q: Query) -> Vec<QueryItem> {
q.into_iter().collect()
}
}
impl IntoIterator for Query {
type Item = QueryItem;
type IntoIter = <BTreeSet<QueryItem> as IntoIterator>::IntoIter;
fn into_iter(self) -> Self::IntoIter {
self.items.into_iter()
}
}
/// A `QueryItem` represents a key or range of keys to be included in a proof.
#[derive(Clone, Debug)]
pub enum QueryItem {
Key(Vec<u8>),
Range(std::ops::Range<Vec<u8>>),
RangeInclusive(RangeInclusive<Vec<u8>>),
}
impl QueryItem {
pub fn lower_bound(&self) -> &[u8] {
match self {
QueryItem::Key(key) => key.as_slice(),
QueryItem::Range(range) => range.start.as_ref(),
QueryItem::RangeInclusive(range) => range.start().as_ref(),
}
}
pub fn upper_bound(&self) -> (&[u8], bool) {
match self {
QueryItem::Key(key) => (key.as_slice(), true),
QueryItem::Range(range) => (range.end.as_ref(), false),
QueryItem::RangeInclusive(range) => (range.end().as_ref(), true),
}
}
pub fn contains(&self, key: &[u8]) -> bool {
let (bound, inclusive) = self.upper_bound();
return key >= self.lower_bound() && (key < bound || (key == bound && inclusive));
}
fn merge(self, other: QueryItem) -> QueryItem {
// TODO: don't copy into new vecs
let start = min(self.lower_bound(), other.lower_bound()).to_vec();
let end = max(self.upper_bound(), other.upper_bound());
if end.1 {
QueryItem::RangeInclusive(RangeInclusive::new(start, end.0.to_vec()))
} else {
QueryItem::Range(std::ops::Range {
start,
end: end.0.to_vec(),
})
}
}
}
impl PartialEq for QueryItem {
fn eq(&self, other: &QueryItem) -> bool {
self.cmp(other) == Ordering::Equal
}
}
impl PartialEq<&[u8]> for QueryItem {
fn eq(&self, other: &&[u8]) -> bool {
matches!(self.partial_cmp(other), Some(Ordering::Equal))
}
}
impl Eq for QueryItem {}
impl Ord for QueryItem {
fn cmp(&self, other: &QueryItem) -> Ordering {
let cmp_lu = self.lower_bound().cmp(other.upper_bound().0);
let cmp_ul = self.upper_bound().0.cmp(other.lower_bound());
let self_inclusive = self.upper_bound().1;
let other_inclusive = other.upper_bound().1;
match (cmp_lu, cmp_ul) {
(Ordering::Less, Ordering::Less) => Ordering::Less,
(Ordering::Less, Ordering::Equal) => match self_inclusive {
true => Ordering::Equal,
false => Ordering::Less,
},
(Ordering::Less, Ordering::Greater) => Ordering::Equal,
(Ordering::Equal, _) => match other_inclusive {
true => Ordering::Equal,
false => Ordering::Greater,
},
(Ordering::Greater, _) => Ordering::Greater,
}
}
}
impl PartialOrd for QueryItem {
fn partial_cmp(&self, other: &QueryItem) -> Option<Ordering> {
Some(self.cmp(other))
}
}
impl PartialOrd<&[u8]> for QueryItem {
fn partial_cmp(&self, other: &&[u8]) -> Option<Ordering> {
let other = QueryItem::Key(other.to_vec());
Some(self.cmp(&other))
}
}
impl From<Vec<u8>> for QueryItem {
fn from(key: Vec<u8>) -> Self {
QueryItem::Key(key)
}
}
impl Link {
/// Creates a `Node::Hash` from this link. Panics if the link is of variant
/// `Link::Modified` since its hash has not yet been computed.
#[cfg(feature = "full")]
fn to_hash_node(&self) -> Node {
let hash = match self {
Link::Reference { hash, .. } => hash,
Link::Modified { .. } => {
panic!("Cannot convert Link::Modified to proof hash node");
}
Link::Uncommitted { hash, .. } => hash,
Link::Loaded { hash, .. } => hash,
};
Node::Hash(*hash)
}
}
impl<'a, S> RefWalker<'a, S>
where
S: Fetch + Sized + Send + Clone,
{
/// Creates a `Node::KV` from the key/value pair of the root node.
pub(crate) fn to_kv_node(&self) -> Node {
Node::KV(self.tree().key().to_vec(), self.tree().value().to_vec())
}
/// Creates a `Node::KVHash` from the hash of the key/value pair of the root
/// node.
pub(crate) fn to_kvhash_node(&self) -> Node {
Node::KVHash(*self.tree().kv_hash())
}
/// Creates a `Node::Hash` from the hash of the node.
pub(crate) fn to_hash_node(&self) -> Node {
Node::Hash(self.tree().hash())
}
/// Generates a proof for the list of queried keys. Returns a tuple
/// containing the generated proof operators, and a tuple representing if
/// any keys were queried were less than the left edge or greater than the
/// right edge, respectively.
#[cfg(feature = "full")]
pub(crate) fn create_proof(
&mut self,
query: &[QueryItem],
) -> Result<(LinkedList<Op>, (bool, bool))> {
// TODO: don't copy into vec, support comparing QI to byte slice
let node_key = QueryItem::Key(self.tree().key().to_vec());
let search = query.binary_search_by(|key| key.cmp(&node_key));
let (left_items, right_items) = match search {
Ok(index) => {
let item = &query[index];
let left_bound = item.lower_bound();
let right_bound = item.upper_bound().0;
// if range starts before this node's key, include it in left
// child's query
let left_query = if left_bound < self.tree().key() {
&query[..=index]
} else {
&query[..index]
};
// if range ends after this node's key, include it in right
// child's query
let right_query = if right_bound > self.tree().key() {
&query[index..]
} else {
&query[index + 1..]
};
(left_query, right_query)
}
Err(index) => (&query[..index], &query[index..]),
};
let (mut proof, left_absence) = self.create_child_proof(true, left_items)?;
let (mut right_proof, right_absence) = self.create_child_proof(false, right_items)?;
let (has_left, has_right) = (!proof.is_empty(), !right_proof.is_empty());
proof.push_back(match search {
Ok(_) => Op::Push(self.to_kv_node()),
Err(_) => {
if left_absence.1 || right_absence.0 {
Op::Push(self.to_kv_node())
} else {
Op::Push(self.to_kvhash_node())
}
}
});
if has_left {
proof.push_back(Op::Parent);
}
if has_right {
proof.append(&mut right_proof);
proof.push_back(Op::Child);
}
Ok((proof, (left_absence.0, right_absence.1)))
}
/// Similar to `create_proof`. Recurses into the child on the given side and
/// generates a proof for the queried keys.
#[cfg(feature = "full")]
fn create_child_proof(
&mut self,
left: bool,
query: &[QueryItem],
) -> Result<(LinkedList<Op>, (bool, bool))> {
Ok(if !query.is_empty() {
if let Some(mut child) = self.walk(left)? {
child.create_proof(query)?
} else {
(LinkedList::new(), (true, true))
}
} else if let Some(link) = self.tree().link(left) {
let mut proof = LinkedList::new();
proof.push_back(Op::Push(link.to_hash_node()));
(proof, (false, false))
} else {
(LinkedList::new(), (false, false))
})
}
}
pub fn verify(bytes: &[u8], expected_hash: Hash) -> Result<Map> {
let ops = Decoder::new(bytes);
let mut map_builder = MapBuilder::new();
let root = execute(ops, true, |node| map_builder.insert(node))?;
if root.hash()? != expected_hash {
return Err(Error::HashMismatch(expected_hash, root.hash()?));
}
Ok(map_builder.build())
}
/// Verifies the encoded proof with the given query and expected hash.
///
/// Every key in `keys` is checked to either have a key/value pair in the proof,
/// or to have its absence in the tree proven.
///
/// Returns `Err` if the proof is invalid, or a list of proven values associated
/// with `keys`. For example, if `keys` contains keys `A` and `B`, the returned
/// list will contain 2 elements, the value of `A` and the value of `B`. Keys
/// proven to be absent in the tree will have an entry of `None`, keys that have
/// a proven value will have an entry of `Some(value)`.
#[deprecated]
pub fn verify_query(
bytes: &[u8],
query: &Query,
expected_hash: Hash,
) -> Result<Vec<(Vec<u8>, Vec<u8>)>> {
let mut output = Vec::with_capacity(query.len());
let mut last_push = None;
let mut query = query.iter().peekable();
let mut in_range = false;
let ops = Decoder::new(bytes);
let root = execute(ops, true, |node| {
if let Node::KV(key, value) = node {
while let Some(item) = query.peek() {
// get next item in query
let query_item = *item;
// we have not reached next queried part of tree
if *query_item > key.as_slice() {
// continue to next push
break;
}
if !in_range {
// this is the first data we have encountered for this query
// item. ensure lower bound of query item is proven
match last_push {
// lower bound is proven - we have an exact match
_ if key == query_item.lower_bound() => {}
// lower bound is proven - this is the leftmost node
// in the tree
None => {}
// lower bound is proven - the preceding tree node
// is lower than the bound
Some(Node::KV(_, _)) => {}
// cannot verify lower bound - we have an abridged
// tree so we cannot tell what the preceding key was
Some(_) => {
return Err(Error::Bound(
"Cannot verify lower bound of queried range".into(),
));
}
}
}
if key.as_slice() >= query_item.upper_bound().0 {
// at or past upper bound of range (or this was an exact
// match on a single-key queryitem), advance to next query
// item
query.next();
in_range = false;
} else {
// have not reached upper bound, we expect more values
// to be proven in the range (and all pushes should be
// unabridged until we reach end of range)
in_range = true;
}
// this push matches the queried item
if query_item.contains(key) {
// add data to output
output.push((key.clone(), value.clone()));
// continue to next push
break;
}
// continue to next queried item
}
} else if in_range {
// we encountered a queried range but the proof was abridged (saw a
// non-KV push), we are missing some part of the range
return Err(Error::MissingData);
}
last_push = Some(node.clone());
Ok(())
})?;
// we have remaining query items, check absence proof against right edge of
// tree
if query.peek().is_some() {
match last_push {
// last node in tree was less than queried item
Some(Node::KV(_, _)) => {}
// proof contains abridged data so we cannot verify absence of
// remaining query items
_ => {
return Err(Error::MissingData);
}
}
}
if root.hash()? != expected_hash {
return Err(Error::HashMismatch(expected_hash, root.hash()?));
}
Ok(output)
}
#[allow(deprecated)]
#[cfg(test)]
mod test {
use super::super::encoding::encode_into;
use super::super::*;
use super::*;
use crate::test_utils::make_tree_seq;
use crate::tree::{NoopCommit, PanicSource, RefWalker, Tree};
use ed::Encode;
fn make_3_node_tree() -> Result<Tree> {
let mut tree = Tree::new(vec![5], vec![5])?
.attach(true, Some(Tree::new(vec![3], vec![3])?))
.attach(false, Some(Tree::new(vec![7], vec![7])?));
tree.commit(&mut NoopCommit {}).expect("commit failed");
Ok(tree)
}
fn verify_keys_test(keys: Vec<Vec<u8>>, expected_result: Vec<Option<Vec<u8>>>) -> Result<()> {
let mut tree = make_3_node_tree()?;
let mut walker = RefWalker::new(&mut tree, PanicSource {});
let (proof, _) = walker
.create_proof(
keys.clone()
.into_iter()
.map(QueryItem::Key)
.collect::<Vec<_>>()
.as_slice(),
)
.expect("failed to create proof");
let mut bytes = vec![];
encode_into(proof.iter(), &mut bytes);
let expected_hash = [
185, 181, 28, 21, 108, 13, 202, 48, 129, 184, 3, 8, 157, 78, 213, 241, 94, 200, 205,
95, 179, 177, 195, 177, 216, 233, 164, 73, 102, 32, 141, 37,
];
let mut query = Query::new();
for key in keys.iter() {
query.insert_key(key.clone());
}
let result = verify_query(bytes.as_slice(), &query, expected_hash).expect("verify failed");
let mut values = std::collections::HashMap::new();
for (key, value) in result {
assert!(values.insert(key, value).is_none());
}
for (key, expected_value) in keys.iter().zip(expected_result.iter()) {
assert_eq!(values.get(key), expected_value.as_ref());
}
Ok(())
}
#[test]
fn root_verify() -> Result<()> {
verify_keys_test(vec![vec![5]], vec![Some(vec![5])])
}
#[test]
fn single_verify() -> Result<()> {
verify_keys_test(vec![vec![3]], vec![Some(vec![3])])
}
#[test]
fn double_verify() -> Result<()> {
verify_keys_test(vec![vec![3], vec![5]], vec![Some(vec![3]), Some(vec![5])])
}
#[test]
fn double_verify_2() -> Result<()> {
verify_keys_test(vec![vec![3], vec![7]], vec![Some(vec![3]), Some(vec![7])])
}
#[test]
fn triple_verify() -> Result<()> {
verify_keys_test(
vec![vec![3], vec![5], vec![7]],
vec![Some(vec![3]), Some(vec![5]), Some(vec![7])],
)
}
#[test]
fn left_edge_absence_verify() -> Result<()> {
verify_keys_test(vec![vec![2]], vec![None])
}
#[test]
fn right_edge_absence_verify() -> Result<()> {
verify_keys_test(vec![vec![8]], vec![None])
}
#[test]
fn inner_absence_verify() -> Result<()> {
verify_keys_test(vec![vec![6]], vec![None])
}
#[test]
fn absent_and_present_verify() -> Result<()> {
verify_keys_test(vec![vec![5], vec![6]], vec![Some(vec![5]), None])
}
#[test]
fn empty_proof() -> Result<()> {
let mut tree = make_3_node_tree()?;
let mut walker = RefWalker::new(&mut tree, PanicSource {});
let (proof, absence) = walker
.create_proof(vec![].as_slice())
.expect("create_proof errored");
let mut iter = proof.iter();
assert_eq!(
iter.next(),
Some(&Op::Push(Node::Hash([
203, 210, 184, 52, 29, 56, 76, 7, 155, 239, 81, 16, 54, 13, 106, 27, 44, 218, 198,
245, 203, 189, 15, 203, 55, 184, 75, 146, 127, 38, 143, 214
])))
);
assert_eq!(
iter.next(),
Some(&Op::Push(Node::KVHash([
169, 4, 73, 65, 62, 49, 160, 159, 37, 166, 195, 249, 63, 31, 23, 11, 169, 0, 24,
104, 179, 211, 218, 38, 108, 129, 117, 232, 65, 101, 194, 157
])))
);
assert_eq!(iter.next(), Some(&Op::Parent));
assert_eq!(
iter.next(),
Some(&Op::Push(Node::Hash([
219, 24, 98, 131, 160, 47, 139, 94, 223, 118, 217, 187, 42, 215, 213, 101, 213,
225, 169, 57, 224, 210, 17, 135, 220, 63, 160, 42, 148, 0, 121, 115
])))
);
assert_eq!(iter.next(), Some(&Op::Child));
assert!(iter.next().is_none());
assert_eq!(absence, (false, false));
let mut bytes = vec![];
encode_into(proof.iter(), &mut bytes);
let res = verify_query(bytes.as_slice(), &Query::new(), tree.hash()).unwrap();
assert!(res.is_empty());
Ok(())
}
#[test]
fn root_proof() -> Result<()> {
let mut tree = make_3_node_tree()?;
let mut walker = RefWalk
gitextract_1164yhgf/
├── .github/
│ └── workflows/
│ └── ci.yml
├── .gitignore
├── CHANGELOG.md
├── Cargo.toml
├── LICENSE
├── README.md
├── benches/
│ ├── merk.rs
│ └── ops.rs
├── docs/
│ └── algorithms.md
├── rustfmt.toml
├── scripts/
│ └── pgo.sh
└── src/
├── error.rs
├── lib.rs
├── merk/
│ ├── chunks.rs
│ ├── mod.rs
│ ├── restore.rs
│ └── snapshot.rs
├── owner.rs
├── proofs/
│ ├── chunk.rs
│ ├── encoding.rs
│ ├── mod.rs
│ ├── query/
│ │ ├── map.rs
│ │ └── mod.rs
│ └── tree.rs
├── test_utils/
│ ├── crash_merk.rs
│ ├── mod.rs
│ └── temp_merk.rs
└── tree/
├── commit.rs
├── debug.rs
├── encoding.rs
├── fuzz_tests.rs
├── hash.rs
├── iter.rs
├── kv.rs
├── link.rs
├── mod.rs
├── ops.rs
└── walk/
├── fetch.rs
├── mod.rs
└── ref_walker.rs
SYMBOL INDEX (546 symbols across 30 files)
FILE: benches/merk.rs
function get_1m_rocksdb (line 14) | fn get_1m_rocksdb(b: &mut Bencher) {
function insert_1m_2k_seq_rocksdb_noprune (line 42) | fn insert_1m_2k_seq_rocksdb_noprune(b: &mut Bencher) {
function insert_1m_2k_rand_rocksdb_noprune (line 63) | fn insert_1m_2k_rand_rocksdb_noprune(b: &mut Bencher) {
function update_1m_2k_seq_rocksdb_noprune (line 84) | fn update_1m_2k_seq_rocksdb_noprune(b: &mut Bencher) {
function update_1m_2k_rand_rocksdb_noprune (line 105) | fn update_1m_2k_rand_rocksdb_noprune(b: &mut Bencher) {
function delete_1m_2k_rand_rocksdb_noprune (line 126) | fn delete_1m_2k_rand_rocksdb_noprune(b: &mut Bencher) {
function prove_1m_1_rand_rocksdb_noprune (line 151) | fn prove_1m_1_rand_rocksdb_noprune(b: &mut Bencher) {
function build_trunk_chunk_1m_1_rand_rocksdb_noprune (line 180) | fn build_trunk_chunk_1m_1_rand_rocksdb_noprune(b: &mut Bencher) {
function chunkproducer_rand_1m_1_rand_rocksdb_noprune (line 208) | fn chunkproducer_rand_1m_1_rand_rocksdb_noprune(b: &mut Bencher) {
function chunk_iter_1m_1_rand_rocksdb_noprune (line 241) | fn chunk_iter_1m_1_rand_rocksdb_noprune(b: &mut Bencher) {
function restore_1m_1_rand_rocksdb_noprune (line 275) | fn restore_1m_1_rand_rocksdb_noprune(b: &mut Bencher) {
function checkpoint_create_destroy_1m_1_rand_rocksdb_noprune (line 325) | fn checkpoint_create_destroy_1m_1_rand_rocksdb_noprune(b: &mut Bencher) {
FILE: benches/ops.rs
function insert_1m_10k_seq_memonly (line 10) | fn insert_1m_10k_seq_memonly(b: &mut Bencher) {
function insert_1m_10k_rand_memonly (line 25) | fn insert_1m_10k_rand_memonly(b: &mut Bencher) {
function update_1m_10k_seq_memonly (line 40) | fn update_1m_10k_seq_memonly(b: &mut Bencher) {
function update_1m_10k_rand_memonly (line 55) | fn update_1m_10k_rand_memonly(b: &mut Bencher) {
FILE: src/error.rs
type Error (line 4) | pub enum Error {
type Result (line 52) | pub type Result<T> = std::result::Result<T, Error>;
FILE: src/merk/chunks.rs
type ChunkProducer (line 16) | pub struct ChunkProducer<'a> {
function new (line 26) | pub fn new(merk: &'a Merk) -> Result<Self> {
function chunk (line 58) | pub fn chunk(&mut self, index: usize) -> Result<Vec<u8>> {
function len (line 78) | pub fn len(&self) -> usize {
function next_chunk (line 90) | fn next_chunk(&mut self) -> Result<Vec<u8>> {
type IntoIter (line 114) | type IntoIter = ChunkIter<'a>;
type Item (line 115) | type Item = <ChunkIter<'a> as Iterator>::Item;
method into_iter (line 117) | fn into_iter(self) -> Self::IntoIter {
type ChunkIter (line 125) | pub struct ChunkIter<'a>(ChunkProducer<'a>);
type Item (line 128) | type Item = Result<Vec<u8>>;
method size_hint (line 130) | fn size_hint(&self) -> (usize, Option<usize>) {
method next (line 134) | fn next(&mut self) -> Option<Self::Item> {
method chunks (line 146) | pub fn chunks(&self) -> Result<ChunkProducer> {
function len_small (line 163) | fn len_small() {
function len_big (line 174) | fn len_big() {
function generate_and_verify_chunks (line 185) | fn generate_and_verify_chunks() -> Result<()> {
function chunks_from_reopen (line 208) | fn chunks_from_reopen() {
function chunks_from_checkpoint (line 237) | fn chunks_from_checkpoint() {
function random_access_chunks (line 259) | fn random_access_chunks() {
function test_chunk_empty (line 280) | fn test_chunk_empty() {
function test_chunk_index_oob (line 293) | fn test_chunk_index_oob() {
function test_chunk_index_gt_1_access (line 303) | fn test_chunk_index_gt_1_access() {
function test_next_chunk_index_oob (line 385) | fn test_next_chunk_index_oob() {
FILE: src/merk/mod.rs
constant ROOT_KEY_KEY (line 19) | const ROOT_KEY_KEY: &[u8] = b"root";
constant FORMAT_VERSION_KEY (line 20) | const FORMAT_VERSION_KEY: &[u8] = b"format";
constant AUX_CF_NAME (line 21) | const AUX_CF_NAME: &str = "aux";
constant INTERNAL_CF_NAME (line 22) | const INTERNAL_CF_NAME: &str = "internal";
constant FORMAT_VERSION (line 24) | const FORMAT_VERSION: u64 = 1;
function column_families (line 26) | fn column_families() -> Vec<ColumnFamilyDescriptor> {
type Merk (line 35) | pub struct Merk {
method open (line 46) | pub fn open<P: AsRef<Path>>(path: P) -> Result<Merk> {
method open_readonly (line 51) | pub fn open_readonly<P: AsRef<Path>>(path: P) -> Result<Merk> {
method open_opt (line 80) | pub fn open_opt<P>(path: P, db_opts: rocksdb::Options) -> Result<Merk>
method open_and_get_aux (line 112) | pub fn open_and_get_aux<P>(path: P, key: &[u8]) -> Result<Option<Vec<u...
method default_db_opts (line 123) | pub fn default_db_opts() -> rocksdb::Options {
method get_aux (line 144) | pub fn get_aux(&self, key: &[u8]) -> Result<Option<Vec<u8>>> {
method get (line 154) | pub fn get(&self, key: &[u8]) -> Result<Option<Vec<u8>>> {
method root_hash (line 165) | pub fn root_hash(&self) -> Hash {
method apply (line 189) | pub fn apply(&mut self, batch: &Batch, aux: &Batch) -> Result<()> {
method apply_unchecked (line 231) | pub unsafe fn apply_unchecked(&mut self, batch: &Batch, aux: &Batch) -...
method destroy (line 244) | pub fn destroy(self) -> Result<()> {
method repair (line 254) | pub fn repair(self) -> Result<Self> {
method migrate_from_v0 (line 307) | pub fn migrate_from_v0<P: AsRef<Path>>(path: P) -> Result<()> {
method prove (line 342) | pub fn prove<Q, I>(&self, query: I) -> Result<Vec<u8>>
method flush (line 350) | pub fn flush(&self) -> Result<()> {
method commit (line 354) | pub fn commit(&mut self, deleted_keys: LinkedList<Vec<u8>>, aux: &Batc...
method walk (line 412) | pub fn walk<T>(&self, f: impl FnOnce(Option<RefWalker<MerkSource>>) ->...
method raw_iter (line 420) | pub fn raw_iter(&self) -> rocksdb::DBRawIterator {
method checkpoint (line 424) | pub fn checkpoint<P: AsRef<Path>>(&self, path: P) -> Result<Merk> {
method snapshot (line 429) | pub fn snapshot(&self) -> Result<Snapshot> {
method db (line 433) | pub fn db(&self) -> &DB {
method source (line 437) | fn source(&self) -> MerkSource {
method use_tree (line 441) | fn use_tree<T>(&self, f: impl FnOnce(Option<&Tree>) -> T) -> T {
method use_tree_mut (line 446) | fn use_tree_mut<T>(&self, f: impl FnOnce(Option<&mut Tree>) -> T) -> T {
method write (line 454) | pub(crate) fn write(&mut self, batch: WriteBatch) -> Result<()> {
method set_root_key (line 462) | pub(crate) fn set_root_key(&mut self, key: Vec<u8>) -> Result<()> {
method fetch_node (line 469) | pub(crate) fn fetch_node(&self, key: &[u8]) -> Result<Option<Tree>> {
method load_root (line 473) | pub(crate) fn load_root(&mut self) -> Result<()> {
type UseTreeMutResult (line 41) | pub type UseTreeMutResult = Result<Vec<(Vec<u8>, Option<Vec<u8>>)>>;
type MerkSource (line 481) | pub struct MerkSource<'a> {
method fetch_by_key (line 486) | fn fetch_by_key(&self, key: &[u8]) -> Result<Option<Tree>> {
type MerkCommitter (line 494) | struct MerkCommitter {
method new (line 501) | fn new(height: u8, levels: u8) -> Self {
method write (line 511) | fn write(&mut self, tree: &Tree) -> Result<()> {
method prune (line 518) | fn prune(&self, tree: &Tree) -> (bool, bool) {
function get (line 525) | pub fn get<F: Fetch>(tree: &Tree, source: F, key: &[u8]) -> Result<Optio...
function root_hash (line 533) | fn root_hash(maybe_tree: Option<&Tree>) -> Hash {
function prove (line 537) | fn prove<Q, I, F>(maybe_tree: Option<&mut Tree>, source: F, query: I) ->...
function has_root (line 556) | fn has_root(db: &DB) -> Result<bool> {
function load_root (line 561) | fn load_root(db: &DB) -> Result<Option<Tree>> {
function load_format_version (line 568) | fn load_format_version(db: &DB) -> Result<u64> {
function assert_invariants (line 589) | fn assert_invariants(merk: &TempMerk) {
function simple_insert_apply (line 597) | fn simple_insert_apply() {
function insert_uncached (line 617) | fn insert_uncached() {
function insert_rand (line 633) | fn insert_rand() {
function actual_deletes (line 648) | fn actual_deletes() {
function aux_data (line 663) | fn aux_data() {
function simulated_crash (line 673) | fn simulated_crash() {
function get_not_found (line 698) | fn get_not_found() {
function reopen (line 724) | fn reopen() {
function reopen_iter (line 766) | fn reopen_iter() {
function checkpoint (line 799) | fn checkpoint() {
function checkpoint_iterator (line 838) | fn checkpoint_iterator() {
function repair (line 871) | fn repair() {
FILE: src/merk/restore.rs
type Restorer (line 22) | pub struct Restorer {
method new (line 42) | pub fn new<P: AsRef<Path>>(
method process_chunk (line 67) | pub fn process_chunk(&mut self, chunk_bytes: &[u8]) -> Result<usize> {
method finalize (line 80) | pub fn finalize(mut self) -> Result<Merk> {
method remaining_chunks (line 100) | pub fn remaining_chunks(&self) -> Option<usize> {
method write_chunk (line 106) | fn write_chunk(&mut self, tree: ProofTree) -> Result<()> {
method process_trunk (line 134) | fn process_trunk(&mut self, ops: Decoder) -> Result<usize> {
method process_leaf (line 193) | fn process_leaf(&mut self, ops: Decoder) -> Result<usize> {
method rewrite_parent_link (line 213) | fn rewrite_parent_link(&mut self, leaf: &ProofTree) -> Result<()> {
method rewrite_trunk_child_heights (line 239) | fn rewrite_trunk_child_heights(&mut self) -> Result<()> {
method remaining_chunks_unchecked (line 288) | pub fn remaining_chunks_unchecked(&self) -> usize {
method restore (line 302) | pub fn restore<P: AsRef<Path>>(
method child_heights (line 312) | fn child_heights(&self) -> (u8, u8) {
method as_link (line 321) | fn as_link(&self) -> Link {
function restore_test (line 345) | fn restore_test(batches: &[&Batch], expected_nodes: usize) {
function restore_10000 (line 382) | fn restore_10000() {
function restore_3 (line 387) | fn restore_3() {
function restore_2_left_heavy (line 392) | fn restore_2_left_heavy() {
function restore_2_right_heavy (line 400) | fn restore_2_right_heavy() {
function restore_1 (line 408) | fn restore_1() {
function assert_raw_db_entries_eq (line 412) | fn assert_raw_db_entries_eq(restored: &Merk, original: &Merk, length: us...
FILE: src/merk/snapshot.rs
type Snapshot (line 22) | pub struct Snapshot<'a> {
function new (line 36) | pub fn new(db: rocksdb::Snapshot<'a>, tree: Option<Tree>) -> Self {
function staticize (line 46) | pub fn staticize(mut self) -> StaticSnapshot {
function get (line 57) | pub fn get(&self, key: &[u8]) -> Result<Option<Vec<u8>>> {
function root_hash (line 66) | pub fn root_hash(&self) -> Hash {
function prove (line 72) | pub fn prove<Q, I>(&self, query: I) -> Result<Vec<u8>>
function walk (line 82) | pub fn walk<T>(&self, f: impl FnOnce(Option<RefWalker<SnapshotSource>>) ...
function raw_iter (line 94) | pub fn raw_iter(&self) -> rocksdb::DBRawIterator {
function source (line 100) | fn source(&self) -> SnapshotSource {
function use_tree (line 105) | fn use_tree<T>(&self, f: impl FnOnce(Option<&Tree>) -> T) -> T {
function use_tree_mut (line 113) | fn use_tree_mut<T>(&self, f: impl FnOnce(Option<&mut Tree>) -> T) -> T {
method drop (line 122) | fn drop(&mut self) {
type SnapshotSource (line 134) | pub struct SnapshotSource<'a>(&'a rocksdb::Snapshot<'a>);
method fetch_by_key (line 137) | fn fetch_by_key(&self, key: &[u8]) -> Result<Option<Tree>> {
type StaticSnapshot (line 157) | pub struct StaticSnapshot {
method with_db (line 200) | pub unsafe fn with_db<'a>(&self, db: &'a rocksdb::DB) -> Snapshot<'a> {
method drop (line 222) | pub unsafe fn drop(mut self, db: &rocksdb::DB) {
method clone_tree (line 231) | fn clone_tree(&self) -> Cell<Option<Tree>> {
type RocksDBSnapshot (line 175) | struct RocksDBSnapshot<'a> {
method drop (line 243) | fn drop(&mut self) {
method clone (line 251) | fn clone(&self) -> Self {
function rocksdb_snapshot_struct_format (line 268) | fn rocksdb_snapshot_struct_format() {
FILE: src/owner.rs
type Owner (line 5) | pub struct Owner<T> {
function new (line 11) | pub fn new(value: T) -> Owner<T> {
function own (line 33) | pub fn own<F: FnOnce(T) -> T>(&mut self, f: F) {
function own_return (line 52) | pub fn own_return<R, F>(&mut self, f: F) -> R
function own_fallible (line 76) | pub fn own_fallible<E, F: FnOnce(T) -> Result<T, E>>(&mut self, f: F) ->...
function into_inner (line 84) | pub fn into_inner(mut self) -> T {
type Target (line 90) | type Target = T;
method deref (line 92) | fn deref(&self) -> &T {
method deref_mut (line 98) | fn deref_mut(&mut self) -> &mut T {
function unwrap (line 103) | fn unwrap<T>(option: Option<T>) -> T {
FILE: src/proofs/chunk.rs
constant MIN_TRUNK_HEIGHT (line 18) | pub const MIN_TRUNK_HEIGHT: usize = 5;
function create_trunk_proof (line 30) | pub fn create_trunk_proof(&mut self) -> Result<(Vec<Op>, bool)> {
function traverse_for_height_proof (line 49) | fn traverse_for_height_proof(&mut self, proof: &mut Vec<Op>, depth: usiz...
function traverse_for_trunk (line 78) | fn traverse_for_trunk(
function get_next_chunk (line 127) | pub(crate) fn get_next_chunk(iter: &mut DBRawIterator, end_key: Option<&...
function verify_leaf (line 177) | pub(crate) fn verify_leaf<I: Iterator<Item = Result<Op>>>(
function verify_trunk (line 198) | pub(crate) fn verify_trunk<I: Iterator<Item = Result<Op>>>(ops: I) -> Re...
type NodeCounts (line 286) | struct NodeCounts {
function count_node_types (line 292) | fn count_node_types(tree: Tree) -> NodeCounts {
function small_trunk_roundtrip (line 307) | fn small_trunk_roundtrip() {
function big_trunk_roundtrip (line 324) | fn big_trunk_roundtrip() {
function one_node_tree_trunk_roundtrip (line 343) | fn one_node_tree_trunk_roundtrip() -> Result<()> {
function two_node_right_heavy_tree_trunk_roundtrip (line 360) | fn two_node_right_heavy_tree_trunk_roundtrip() -> Result<()> {
function two_node_left_heavy_tree_trunk_roundtrip (line 380) | fn two_node_left_heavy_tree_trunk_roundtrip() -> Result<()> {
function three_node_tree_trunk_roundtrip (line 400) | fn three_node_tree_trunk_roundtrip() -> Result<()> {
function leaf_chunk_roundtrip (line 422) | fn leaf_chunk_roundtrip() {
function test_verify_height_stack_overflow (line 480) | fn test_verify_height_stack_overflow() {
FILE: src/proofs/encoding.rs
method encode_into (line 10) | fn encode_into<W: Write>(&self, dest: &mut W) -> ed::Result<()> {
method encoding_length (line 35) | fn encoding_length(&self) -> ed::Result<usize> {
method decode (line 47) | fn decode<R: Read>(mut input: R) -> ed::Result<Self> {
method encode_into (line 84) | fn encode_into<W: Write>(&self, dest: &mut W) -> Result<()> {
method encoding_length (line 88) | fn encoding_length(&self) -> usize {
method decode (line 92) | pub fn decode(bytes: &[u8]) -> Result<Self> {
function encode_into (line 97) | pub fn encode_into<'a, T: Iterator<Item = &'a Op>>(ops: T, output: &mut ...
type Decoder (line 103) | pub struct Decoder<'a> {
function new (line 109) | pub fn new(proof_bytes: &'a [u8]) -> Self {
type Item (line 118) | type Item = Result<Op>;
method next (line 120) | fn next(&mut self) -> Option<Self::Item> {
function encode_push_hash (line 140) | fn encode_push_hash() {
function encode_push_kvhash (line 157) | fn encode_push_kvhash() {
function encode_push_kv (line 174) | fn encode_push_kv() {
function encode_parent (line 184) | fn encode_parent() {
function encode_child (line 194) | fn encode_child() {
function encode_push_kv_long_key (line 205) | fn encode_push_kv_long_key() {
function decode_push_hash (line 212) | fn decode_push_hash() {
function decode_push_kvhash (line 222) | fn decode_push_kvhash() {
function decode_push_kv (line 232) | fn decode_push_kv() {
function decode_parent (line 239) | fn decode_parent() {
function decode_child (line 246) | fn decode_child() {
function decode_unknown (line 253) | fn decode_unknown() {
FILE: src/proofs/mod.rs
type Op (line 14) | pub enum Op {
type Node (line 32) | pub enum Node {
FILE: src/proofs/query/map.rs
type MapBuilder (line 10) | pub(crate) struct MapBuilder(Map);
method new (line 14) | pub fn new() -> Self {
method insert (line 24) | pub fn insert(&mut self, node: &Node) -> Result<()> {
method build (line 46) | pub fn build(self) -> Map {
type Map (line 57) | pub struct Map {
method get (line 67) | pub fn get<'a>(&'a self, key: &'a [u8]) -> Result<Option<&'a [u8]>> {
method range (line 87) | pub fn range<'a>(&self, bounds: impl RangeBounds<&'a [u8]>) -> Range {
method join (line 119) | pub fn join(self, other: Map) -> Map {
method contiguous_right (line 145) | fn contiguous_right(&self, key: &[u8]) -> bool {
function bound_to_inner (line 155) | fn bound_to_inner<T>(bound: Bound<T>) -> Option<T> {
function bound_to_vec (line 163) | fn bound_to_vec(bound: Bound<&&[u8]>) -> Bound<Vec<u8>> {
function bounds_to_vec (line 173) | fn bounds_to_vec<'a, R: RangeBounds<&'a [u8]>>(bounds: R) -> (Bound<Vec<...
type Range (line 184) | pub struct Range<'a> {
type InnerRange (line 191) | type InnerRange<'a> = btree_map::Range<'a, Vec<u8>, (bool, Vec<u8>)>;
function yield_entry_if_contiguous (line 194) | fn yield_entry_if_contiguous(
function yield_entry (line 208) | fn yield_entry(
function yield_none_if_contiguous (line 222) | fn yield_none_if_contiguous(
function yield_next_if_contiguous (line 235) | fn yield_next_if_contiguous(&mut self) -> Option<Result<(&'a [u8], &'a [...
function yield_next_back_if_contiguous (line 246) | fn yield_next_back_if_contiguous(
type Item (line 260) | type Item = Result<(&'a [u8], &'a [u8])>;
method next (line 262) | fn next(&mut self) -> Option<Self::Item> {
method next_back (line 301) | fn next_back(&mut self) -> Option<Self::Item> {
function mapbuilder_insert_out_of_order (line 347) | fn mapbuilder_insert_out_of_order() {
function mapbuilder_insert_dupe (line 355) | fn mapbuilder_insert_dupe() {
function mapbuilder_insert_including_edge (line 362) | fn mapbuilder_insert_including_edge() {
function mapbuilder_insert_abridged_edge (line 371) | fn mapbuilder_insert_abridged_edge() {
function mapbuilder_build (line 380) | fn mapbuilder_build() {
function map_get_included (line 395) | fn map_get_included() {
function map_get_missing_absence_proof (line 408) | fn map_get_missing_absence_proof() {
function map_get_valid_absence_proof (line 419) | fn map_get_valid_absence_proof() {
function range_abridged (line 430) | fn range_abridged() {
function range_ok (line 443) | fn range_ok() {
function range_empty (line 459) | fn range_empty() {
function range_lower_unbounded_map_non_contiguous (line 468) | fn range_lower_unbounded_map_non_contiguous() {
function range_reach_proof_end (line 482) | fn range_reach_proof_end() {
function range_unbounded (line 495) | fn range_unbounded() {
function range_abridged_rev (line 509) | fn range_abridged_rev() {
function range_ok_rev (line 522) | fn range_ok_rev() {
function range_upper_unbounded_map_non_contiguous (line 538) | fn range_upper_unbounded_map_non_contiguous() {
function range_reach_proof_end_rev (line 552) | fn range_reach_proof_end_rev() {
function range_unbounded_rev (line 565) | fn range_unbounded_rev() {
function map_join (line 578) | fn map_join() {
FILE: src/proofs/query/mod.rs
type Query (line 19) | pub struct Query {
method new (line 25) | pub fn new() -> Self {
method len (line 29) | pub(crate) fn len(&self) -> usize {
method iter (line 33) | pub(crate) fn iter(&self) -> impl Iterator<Item = &QueryItem> {
method insert_key (line 44) | pub fn insert_key(&mut self, key: Vec<u8>) {
method insert_range (line 55) | pub fn insert_range(&mut self, range: std::ops::Range<Vec<u8>>) {
method insert_range_inclusive (line 66) | pub fn insert_range_inclusive(&mut self, range: RangeInclusive<Vec<u8>...
method insert_item (line 76) | pub fn insert_item(&mut self, mut item: QueryItem) {
method from (line 89) | fn from(other: Vec<Q>) -> Self {
function from (line 96) | fn from(q: Query) -> Vec<QueryItem> {
type Item (line 102) | type Item = QueryItem;
type IntoIter (line 103) | type IntoIter = <BTreeSet<QueryItem> as IntoIterator>::IntoIter;
method into_iter (line 105) | fn into_iter(self) -> Self::IntoIter {
type QueryItem (line 112) | pub enum QueryItem {
method lower_bound (line 119) | pub fn lower_bound(&self) -> &[u8] {
method upper_bound (line 127) | pub fn upper_bound(&self) -> (&[u8], bool) {
method contains (line 135) | pub fn contains(&self, key: &[u8]) -> bool {
method merge (line 140) | fn merge(self, other: QueryItem) -> QueryItem {
method eq (line 162) | fn eq(&self, other: &&[u8]) -> bool {
method partial_cmp (line 199) | fn partial_cmp(&self, other: &&[u8]) -> Option<Ordering> {
method from (line 206) | fn from(key: Vec<u8>) -> Self {
method eq (line 156) | fn eq(&self, other: &QueryItem) -> bool {
method cmp (line 170) | fn cmp(&self, other: &QueryItem) -> Ordering {
method partial_cmp (line 193) | fn partial_cmp(&self, other: &QueryItem) -> Option<Ordering> {
method to_hash_node (line 215) | fn to_hash_node(&self) -> Node {
function to_kv_node (line 233) | pub(crate) fn to_kv_node(&self) -> Node {
function to_kvhash_node (line 239) | pub(crate) fn to_kvhash_node(&self) -> Node {
function to_hash_node (line 244) | pub(crate) fn to_hash_node(&self) -> Node {
function create_proof (line 253) | pub(crate) fn create_proof(
function create_child_proof (line 319) | fn create_child_proof(
function verify (line 340) | pub fn verify(bytes: &[u8], expected_hash: Hash) -> Result<Map> {
function verify_query (line 364) | pub fn verify_query(
function make_3_node_tree (line 479) | fn make_3_node_tree() -> Result<Tree> {
function verify_keys_test (line 487) | fn verify_keys_test(keys: Vec<Vec<u8>>, expected_result: Vec<Option<Vec<...
function root_verify (line 527) | fn root_verify() -> Result<()> {
function single_verify (line 532) | fn single_verify() -> Result<()> {
function double_verify (line 537) | fn double_verify() -> Result<()> {
function double_verify_2 (line 542) | fn double_verify_2() -> Result<()> {
function triple_verify (line 547) | fn triple_verify() -> Result<()> {
function left_edge_absence_verify (line 555) | fn left_edge_absence_verify() -> Result<()> {
function right_edge_absence_verify (line 560) | fn right_edge_absence_verify() -> Result<()> {
function inner_absence_verify (line 565) | fn inner_absence_verify() -> Result<()> {
function absent_and_present_verify (line 570) | fn absent_and_present_verify() -> Result<()> {
function empty_proof (line 575) | fn empty_proof() -> Result<()> {
function root_proof (line 618) | fn root_proof() -> Result<()> {
function leaf_proof (line 660) | fn leaf_proof() -> Result<()> {
function double_leaf_proof (line 702) | fn double_leaf_proof() -> Result<()> {
function all_nodes_proof (line 738) | fn all_nodes_proof() -> Result<()> {
function global_edge_absence_proof (line 775) | fn global_edge_absence_proof() -> Result<()> {
function absence_proof (line 817) | fn absence_proof() -> Result<()> {
function doc_proof (line 853) | fn doc_proof() -> Result<()> {
function query_item_cmp (line 964) | fn query_item_cmp() {
function query_item_merge (line 986) | fn query_item_merge() {
function query_insert (line 1011) | fn query_insert() {
function range_proof (line 1029) | fn range_proof() {
function range_proof_inclusive (line 1116) | fn range_proof_inclusive() {
function range_proof_missing_upper_bound (line 1204) | fn range_proof_missing_upper_bound() {
function range_proof_missing_lower_bound (line 1291) | fn range_proof_missing_lower_bound() {
function query_from_vec (line 1373) | fn query_from_vec() {
function query_into_vec (line 1387) | fn query_into_vec() {
function query_item_from_vec_u8 (line 1407) | fn query_item_from_vec_u8() {
function verify_ops (line 1416) | fn verify_ops() -> Result<()> {
function verify_ops_mismatched_hash (line 1440) | fn verify_ops_mismatched_hash() {
function verify_query_mismatched_hash (line 1458) | fn verify_query_mismatched_hash() {
function hash_attach (line 1484) | fn hash_attach() {
FILE: src/proofs/tree.rs
type Child (line 8) | pub struct Child {
type Tree (line 18) | pub struct Tree {
method from (line 31) | fn from(node: Node) -> Self {
method hash (line 52) | pub fn hash(&self) -> Result<Hash> {
method layer (line 68) | pub fn layer(&self, depth: usize) -> LayerIter {
method visit_nodes (line 74) | pub fn visit_nodes<F: FnMut(Node)>(mut self, visit_node: &mut F) {
method visit_refs (line 89) | pub fn visit_refs<F: FnMut(&Tree)>(&self, visit_node: &mut F) {
method child (line 102) | pub fn child(&self, left: bool) -> Option<&Child> {
method child_mut (line 111) | pub(crate) fn child_mut(&mut self, left: bool) -> &mut Option<Child> {
method attach (line 121) | pub(crate) fn attach(&mut self, left: bool, child: Tree) -> Result<()> {
method child_hash (line 145) | fn child_hash(&self, left: bool) -> Hash {
method try_into_hash (line 151) | fn try_into_hash(self) -> Result<Tree> {
method key (line 156) | pub(crate) fn key(&self) -> &[u8] {
method eq (line 43) | fn eq(&self, other: &Self) -> bool {
type LayerIter (line 166) | pub struct LayerIter<'a> {
function new (line 173) | fn new(tree: &'a Tree, depth: usize) -> Self {
function traverse_to_start (line 185) | fn traverse_to_start(&mut self, tree: &'a Tree, remaining_depth: usize) {
type Item (line 201) | type Item = &'a Tree;
method next (line 203) | fn next(&mut self) -> Option<Self::Item> {
function execute (line 245) | pub(crate) fn execute<I, F>(ops: I, collapse: bool, mut visit_node: F) -...
function make_7_node_prooftree (line 321) | fn make_7_node_prooftree() -> ProofTree {
function height_counting (line 338) | fn height_counting() {
function layer_iter (line 356) | fn layer_iter() {
function visit_nodes (line 382) | fn visit_nodes() {
FILE: src/test_utils/crash_merk.rs
type CrashMerk (line 9) | pub struct CrashMerk {
method open (line 17) | pub fn open<P: AsRef<Path>>(path: P) -> Result<CrashMerk> {
method crash (line 27) | pub unsafe fn crash(&mut self) -> Result<()> {
method into_inner (line 44) | pub fn into_inner(self) -> Merk {
method destroy (line 48) | pub fn destroy(self) -> Result<()> {
type Target (line 54) | type Target = Merk;
method deref (line 56) | fn deref(&self) -> &Merk {
method deref_mut (line 62) | fn deref_mut(&mut self) -> &mut Merk {
function crash (line 74) | fn crash() {
FILE: src/test_utils/mod.rs
function assert_tree_invariants (line 14) | pub fn assert_tree_invariants(tree: &Tree) {
function apply_memonly_unchecked (line 37) | pub fn apply_memonly_unchecked(tree: Tree, batch: &Batch) -> Tree {
function apply_memonly (line 47) | pub fn apply_memonly(tree: Tree, batch: &Batch) -> Tree {
function apply_to_memonly (line 53) | pub fn apply_to_memonly(maybe_tree: Option<Tree>, batch: &Batch) -> Opti...
function seq_key (line 66) | pub fn seq_key(n: u64) -> Vec<u8> {
function put_entry_value (line 70) | pub fn put_entry_value() -> Vec<u8> {
function put_entry (line 74) | pub fn put_entry(n: u64) -> BatchEntry {
function del_entry (line 78) | pub fn del_entry(n: u64) -> BatchEntry {
function make_batch_seq (line 82) | pub fn make_batch_seq(range: Range<u64>) -> Vec<BatchEntry> {
function make_del_batch_seq (line 90) | pub fn make_del_batch_seq(range: Range<u64>) -> Vec<BatchEntry> {
function make_batch_rand (line 98) | pub fn make_batch_rand(size: u64, seed: u64) -> Vec<BatchEntry> {
function make_del_batch_rand (line 109) | pub fn make_del_batch_rand(size: u64, seed: u64) -> Vec<BatchEntry> {
function make_tree_rand (line 120) | pub fn make_tree_rand(node_count: u64, batch_size: u64, initial_seed: u6...
function make_tree_seq (line 139) | pub fn make_tree_seq(node_count: u64) -> Tree {
FILE: src/test_utils/temp_merk.rs
type TempMerk (line 8) | pub struct TempMerk {
method open (line 15) | pub fn open<P: AsRef<Path>>(path: P) -> Result<TempMerk> {
method new (line 21) | pub fn new() -> Result<TempMerk> {
method create_path (line 25) | pub fn create_path() -> PathBuf {
method drop (line 37) | fn drop(&mut self) {
type Target (line 47) | type Target = Merk;
method deref (line 49) | fn deref(&self) -> &Merk {
method deref_mut (line 55) | fn deref_mut(&mut self) -> &mut Merk {
FILE: src/tree/commit.rs
type Commit (line 6) | pub trait Commit {
method write (line 9) | fn write(&mut self, tree: &Tree) -> Result<()>;
method prune (line 15) | fn prune(&self, _tree: &Tree) -> (bool, bool) {
method write (line 24) | fn write(&mut self, _tree: &Tree) -> Result<()> {
method prune (line 28) | fn prune(&self, _tree: &Tree) -> (bool, bool) {
type NoopCommit (line 22) | pub struct NoopCommit {}
FILE: src/tree/debug.rs
method fmt (line 7) | fn fmt(&self, f: &mut Formatter) -> Result {
FILE: src/tree/encoding.rs
method encode (line 10) | pub fn encode(&self) -> Vec<u8> {
method encode_into (line 16) | pub fn encode_into(&self, dest: &mut Vec<u8>) {
method encoding_length (line 22) | pub fn encoding_length(&self) -> usize {
method decode_into (line 28) | pub fn decode_into(&mut self, key: Vec<u8>, input: &[u8]) {
method decode (line 35) | pub fn decode(key: Vec<u8>, input: &[u8]) -> Tree {
method decode_v0 (line 42) | pub fn decode_v0<R: Read>(mut input: R) -> Result<Self> {
function encode_leaf_tree (line 74) | fn encode_leaf_tree() {
function encode_modified_tree (line 88) | fn encode_modified_tree() {
function encode_loaded_tree (line 104) | fn encode_loaded_tree() -> Result<()> {
function encode_uncommitted_tree (line 129) | fn encode_uncommitted_tree() -> Result<()> {
function encode_reference_tree (line 154) | fn encode_reference_tree() {
function decode_leaf_tree (line 179) | fn decode_leaf_tree() {
function decode_reference_tree (line 190) | fn decode_reference_tree() {
FILE: src/tree/fuzz_tests.rs
constant ITERATIONS (line 10) | const ITERATIONS: usize = 2_000;
type Map (line 11) | type Map = BTreeMap<Vec<u8>, Vec<u8>>;
function fuzz (line 14) | fn fuzz() {
function fuzz_17391518417409062786 (line 24) | fn fuzz_17391518417409062786() {
function fuzz_396148930387069749 (line 29) | fn fuzz_396148930387069749() {
function fuzz_case (line 33) | fn fuzz_case(seed: u64) {
function make_batch (line 59) | fn make_batch(maybe_tree: Option<&Tree>, size: u64, seed: u64) -> Vec<Ba...
function apply_to_map (line 119) | fn apply_to_map(map: &mut Map, batch: &Batch) {
function assert_map (line 132) | fn assert_map(maybe_tree: Option<&Tree>, map: &Map) {
FILE: src/tree/hash.rs
type Hasher (line 5) | pub type Hasher = Sha512_256;
constant HASH_LENGTH (line 8) | pub const HASH_LENGTH: usize = 32;
constant NULL_HASH (line 11) | pub const NULL_HASH: Hash = [0; HASH_LENGTH];
type Hash (line 14) | pub type Hash = [u8; HASH_LENGTH];
function kv_hash (line 17) | pub fn kv_hash<D: Digest>(key: &[u8], value: &[u8]) -> Result<Hash, TryF...
function node_hash (line 39) | pub fn node_hash<D: Digest>(kv: &Hash, left: &Hash, right: &Hash) -> Hash {
FILE: src/tree/iter.rs
type StackItem (line 8) | struct StackItem<'a> {
function new (line 17) | fn new(tree: &'a Tree) -> Self {
function to_entry (line 29) | fn to_entry(&self) -> (Vec<u8>, Vec<u8>) {
type Iter (line 36) | pub struct Iter<'a> {
function new (line 42) | pub fn new(tree: &'a Tree) -> Self {
method iter (line 51) | pub fn iter(&'a self) -> Iter<'a> {
type Item (line 57) | type Item = (Vec<u8>, Vec<u8>);
method next (line 60) | fn next(&mut self) -> Option<Self::Item> {
FILE: src/tree/kv.rs
type KV (line 15) | pub struct KV {
method new (line 24) | pub fn new(key: Vec<u8>, value: Vec<u8>) -> std::result::Result<Self, ...
method from_fields (line 31) | pub fn from_fields(key: Vec<u8>, value: Vec<u8>, hash: Hash) -> Self {
method with_value (line 38) | pub fn with_value(mut self, value: Vec<u8>) -> std::result::Result<Sel...
method key (line 46) | pub fn key(&self) -> &[u8] {
method value (line 52) | pub fn value(&self) -> &[u8] {
method hash (line 58) | pub fn hash(&self) -> &Hash {
method take_key (line 64) | pub fn take_key(self) -> Vec<u8> {
method encode_into (line 71) | fn encode_into<W: Write>(&self, out: &mut W) -> Result<()> {
method encoding_length (line 78) | fn encoding_length(&self) -> Result<usize> {
method decode (line 90) | fn decode<R: Read>(input: R) -> Result<Self> {
method decode_into (line 101) | fn decode_into<R: Read>(&mut self, mut input: R) -> Result<()> {
function new_kv (line 118) | fn new_kv() -> std::result::Result<(), TryFromIntError> {
function with_value (line 128) | fn with_value() -> std::result::Result<(), TryFromIntError> {
FILE: src/tree/link.rs
type Link (line 14) | pub enum Link {
method from_modified_tree (line 54) | pub fn from_modified_tree(tree: Tree) -> Self {
method maybe_from_modified_tree (line 66) | pub fn maybe_from_modified_tree(maybe_tree: Option<Tree>) -> Option<Se...
method is_reference (line 72) | pub fn is_reference(&self) -> bool {
method is_modified (line 78) | pub fn is_modified(&self) -> bool {
method is_uncommitted (line 84) | pub fn is_uncommitted(&self) -> bool {
method is_stored (line 90) | pub fn is_stored(&self) -> bool {
method key (line 96) | pub fn key(&self) -> &[u8] {
method tree (line 108) | pub fn tree(&self) -> Option<&Tree> {
method hash (line 122) | pub fn hash(&self) -> &Hash {
method height (line 135) | pub fn height(&self) -> u8 {
method balance_factor (line 147) | pub fn balance_factor(&self) -> i8 {
method into_reference (line 160) | pub fn into_reference(self) -> Self {
method child_heights_mut (line 179) | pub(crate) fn child_heights_mut(&mut self) -> &mut (u8, u8) {
method default_reference (line 257) | fn default_reference() -> Self {
method decode_v0 (line 265) | pub(crate) fn decode_v0<R: Read>(mut input: R) -> Result<Self> {
method encode_into (line 203) | fn encode_into<W: Write>(&self, out: &mut W) -> Result<()> {
method encoding_length (line 240) | fn encoding_length(&self) -> Result<usize> {
method decode (line 287) | fn decode<R: Read>(input: R) -> Result<Link> {
method decode_into (line 294) | fn decode_into<R: Read>(&mut self, mut input: R) -> Result<()> {
function read_u16 (line 327) | fn read_u16<R: Read>(mut input: R) -> Result<u16> {
function read_u8 (line 334) | fn read_u8<R: Read>(mut input: R) -> Result<u8> {
function from_modified_tree (line 347) | fn from_modified_tree() -> std::result::Result<(), &'static str> {
function maybe_from_modified_tree (line 362) | fn maybe_from_modified_tree() -> std::result::Result<(), crate::error::E...
function types (line 373) | fn types() -> std::result::Result<(), crate::error::Error> {
function modified_hash (line 438) | fn modified_hash() {
function modified_into_reference (line 452) | fn modified_into_reference() {
function uncommitted_into_reference (line 463) | fn uncommitted_into_reference() {
function encode_link (line 473) | fn encode_link() {
function encode_link_long_key_valid (line 493) | fn encode_link_long_key_valid() {
function encode_link_long_key_invalid (line 508) | fn encode_link_long_key_invalid() {
FILE: src/tree/mod.rs
type TreeInner (line 30) | pub struct TreeInner {
type Tree (line 42) | pub struct Tree {
method new (line 50) | pub fn new(key: Vec<u8>, value: Vec<u8>) -> Result<Self> {
method from_fields (line 62) | pub fn from_fields(
method key (line 80) | pub fn key(&self) -> &[u8] {
method take_key (line 87) | pub fn take_key(self) -> Vec<u8> {
method value (line 93) | pub fn value(&self) -> &[u8] {
method kv_hash (line 99) | pub fn kv_hash(&self) -> &Hash {
method link (line 106) | pub fn link(&self, left: bool) -> Option<&Link> {
method link_mut (line 117) | pub fn link_mut(&mut self, left: bool) -> Option<&mut Link> {
method child (line 128) | pub fn child(&self, left: bool) -> Option<&Self> {
method child_mut (line 138) | pub fn child_mut(&mut self, left: bool) -> Option<&mut Self> {
method child_hash (line 151) | pub fn child_hash(&self, left: bool) -> &Hash {
method hash (line 157) | pub fn hash(&self) -> Hash {
method child_pending_writes (line 168) | pub fn child_pending_writes(&self, left: bool) -> usize {
method child_height (line 178) | pub fn child_height(&self, left: bool) -> u8 {
method child_heights (line 183) | pub fn child_heights(&self) -> (u8, u8) {
method height (line 191) | pub fn height(&self) -> u8 {
method balance_factor (line 200) | pub fn balance_factor(&self) -> i8 {
method attach (line 211) | pub fn attach(mut self, left: bool, maybe_child: Option<Self>) -> Self {
method detach (line 237) | pub fn detach(mut self, left: bool) -> (Self, Option<Self>) {
method detach_expect (line 257) | pub fn detach_expect(self, left: bool) -> (Self, Self) {
method walk (line 279) | pub fn walk<F>(self, left: bool, f: F) -> Self
method walk_expect (line 289) | pub fn walk_expect<F>(self, left: bool, f: F) -> Self
method slot_mut (line 299) | pub(crate) fn slot_mut(&mut self, left: bool) -> &mut Option<Link> {
method with_value (line 310) | pub fn with_value(mut self, value: Vec<u8>) -> Result<Self> {
method commit (line 325) | pub fn commit<C: Commit>(&mut self, c: &mut C) -> Result<()> {
method load (line 382) | pub fn load<S: Fetch>(&mut self, left: bool, source: &S) -> Result<()> {
method get_value (line 405) | pub fn get_value(&self, key: &[u8]) -> Result<GetResult> {
type GetResult (line 429) | pub enum GetResult {
function side_to_str (line 435) | pub fn side_to_str(left: bool) -> &'static str {
function build_tree (line 451) | fn build_tree() -> Result<()> {
function attach_existing (line 476) | fn attach_existing() {
function modify (line 490) | fn modify() -> Result<()> {
function child_and_link (line 518) | fn child_and_link() -> Result<()> {
function child_hash (line 537) | fn child_hash() -> Result<()> {
function hash (line 553) | fn hash() -> Result<()> {
function child_pending_writes (line 566) | fn child_pending_writes() -> Result<()> {
function height_and_balance (line 578) | fn height_and_balance() -> Result<()> {
function commit (line 601) | fn commit() -> Result<()> {
FILE: src/tree/ops.rs
type Op (line 8) | pub enum Op {
method fmt (line 16) | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
type BatchEntry (line 29) | pub type BatchEntry = (Vec<u8>, Op);
type Batch (line 32) | pub type Batch = [BatchEntry];
type PanicSource (line 37) | pub struct PanicSource {}
method fetch_by_key (line 40) | fn fetch_by_key(&self, _: &[u8]) -> Result<Option<Tree>> {
function apply_to (line 54) | pub fn apply_to(
function build (line 75) | fn build(batch: &Batch, source: S) -> Result<Option<Tree>> {
function apply (line 112) | fn apply(self, batch: &Batch) -> Result<(Option<Self>, LinkedList<Vec<u8...
function recurse (line 164) | fn recurse(
function balance_factor (line 210) | fn balance_factor(&self) -> i8 {
function maybe_balance (line 217) | fn maybe_balance(self) -> Result<Self> {
function rotate (line 237) | fn rotate(self, left: bool) -> Result<Self> {
function remove (line 250) | pub fn remove(self) -> Result<Option<Self>> {
function promote_edge (line 276) | fn promote_edge(self, left: bool, attach: Self) -> Result<Self> {
function remove_edge (line 286) | fn remove_edge(self, left: bool) -> Result<(Self, Option<Self>)> {
function simple_insert (line 309) | fn simple_insert() -> Result<()> {
function simple_update (line 323) | fn simple_update() -> Result<()> {
function simple_delete (line 339) | fn simple_delete() -> Result<()> {
function delete_non_existent (line 366) | fn delete_non_existent() -> Result<()> {
function delete_only_node (line 374) | fn delete_only_node() -> Result<()> {
function delete_deep (line 387) | fn delete_deep() {
function delete_recursive (line 399) | fn delete_recursive() {
function delete_recursive_2 (line 412) | fn delete_recursive_2() {
function rebalanced_delete (line 425) | fn rebalanced_delete() {
function apply_empty_none (line 459) | fn apply_empty_none() {
function insert_empty_single (line 467) | fn insert_empty_single() {
function insert_root_single (line 479) | fn insert_root_single() -> Result<()> {
function insert_root_double (line 490) | fn insert_root_double() -> Result<()> {
function insert_rebalance (line 501) | fn insert_rebalance() -> Result<()> {
function insert_100_sequential (line 517) | fn insert_100_sequential() -> Result<()> {
function delete_recursive_large (line 532) | fn delete_recursive_large() {
FILE: src/tree/walk/fetch.rs
type Fetch (line 8) | pub trait Fetch {
method fetch_by_key (line 9) | fn fetch_by_key(&self, key: &[u8]) -> Result<Option<Tree>>;
method fetch (line 13) | fn fetch(&self, link: &Link) -> Result<Tree> {
method fetch_by_key_expect (line 17) | fn fetch_by_key_expect(&self, key: &[u8]) -> Result<Tree> {
FILE: src/tree/walk/mod.rs
type Walker (line 12) | pub struct Walker<S>
function new (line 25) | pub fn new(tree: Tree, source: S) -> Self {
function detach (line 35) | pub fn detach(mut self, left: bool) -> Result<(Self, Option<Self>)> {
function detach_expect (line 62) | pub fn detach_expect(self, left: bool) -> Result<(Self, Self)> {
function walk (line 76) | pub fn walk<F, T>(self, left: bool, f: F) -> Result<Self>
function walk_expect (line 89) | pub fn walk_expect<F, T>(self, left: bool, f: F) -> Result<Self>
function tree (line 101) | pub fn tree(&self) -> &Tree {
function into_inner (line 106) | pub fn into_inner(self) -> Tree {
function wrap (line 112) | fn wrap(&self, tree: Tree) -> Self {
function clone_source (line 117) | pub fn clone_source(&self) -> S {
function attach (line 123) | pub fn attach<T>(mut self, left: bool, maybe_child: Option<T>) -> Self
function with_value (line 133) | pub fn with_value(mut self, value: Vec<u8>) -> Result<Self> {
method from (line 143) | fn from(walker: Walker<S>) -> Tree {
type MockSource (line 155) | struct MockSource {}
method fetch_by_key (line 158) | fn fetch_by_key(&self, key: &[u8]) -> Result<Option<Tree>> {
function walk_modified (line 164) | fn walk_modified() -> Result<()> {
function walk_stored (line 182) | fn walk_stored() -> Result<()> {
function walk_pruned (line 201) | fn walk_pruned() {
function walk_none (line 227) | fn walk_none() -> Result<()> {
FILE: src/tree/walk/ref_walker.rs
type RefWalker (line 13) | pub struct RefWalker<'a, S>
function new (line 26) | pub fn new(tree: &'a mut Tree, source: S) -> Self {
function tree (line 32) | pub fn tree(&self) -> &Tree {
function walk (line 39) | pub fn walk(&mut self, left: bool) -> Result<Option<RefWalker<S>>> {
Condensed preview — 40 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (349K chars).
[
{
"path": ".github/workflows/ci.yml",
"chars": 4712,
"preview": "name: CI\n\non:\n push:\n branches: [master, develop]\n pull_request:\n branches: [master, develop]\n\nenv:\n CARGO_TERM"
},
{
"path": ".gitignore",
"chars": 36,
"preview": "target\ntemp.db\n.DS_Store\nCargo.lock\n"
},
{
"path": "CHANGELOG.md",
"chars": 261,
"preview": "# Changelog\n\n## [Unreleased]\n\n### Bug Fixes\n\n- Fixed bug where column families would be non-atomically flushed when one "
},
{
"path": "Cargo.toml",
"chars": 824,
"preview": "[package]\nname = \"merk\"\ndescription = \"High-performance Merkle key/value store\"\nversion = \"2.0.0\"\nauthors = [\"Turbofish "
},
{
"path": "LICENSE",
"chars": 11357,
"preview": " Apache License\n Version 2.0, January 2004\n "
},
{
"path": "README.md",
"chars": 5712,
"preview": "<h1 align=\"left\">\n<picture>\n <source media=\"(prefers-color-scheme: dark)\" srcset=\"./merk-dark.svg\">\n <source media=\"(p"
},
{
"path": "benches/merk.rs",
"chars": 10135,
"preview": "#![feature(test)]\n\nextern crate test;\n\nuse merk::proofs::encode_into as encode_proof_into;\nuse merk::restore::Restorer;\n"
},
{
"path": "benches/ops.rs",
"chars": 1751,
"preview": "#![feature(test)]\n\nextern crate test;\n\nuse merk::owner::Owner;\nuse merk::test_utils::*;\nuse test::Bencher;\n\n#[bench]\nfn "
},
{
"path": "docs/algorithms.md",
"chars": 17602,
"preview": "# Merk - A High-Performance Merkle AVL Tree\n\n**Matt Bell ([@mappum](https://twitter.com/mappum))** • [Nomic Hodlings, In"
},
{
"path": "rustfmt.toml",
"chars": 41,
"preview": "comment_width = 80\nwrap_comments = true\n\n"
},
{
"path": "scripts/pgo.sh",
"chars": 767,
"preview": "#!/bin/bash\n\ndefault_host_triple=\"\"\ndefault_toolchain=\"\"\nIFS=\" = \"\nwhile read -r name value\ndo\n value=\"${value//\\\"/}\"\n "
},
{
"path": "src/error.rs",
"chars": 1542,
"preview": "pub use thiserror::Error;\n\n#[derive(Error, Debug)]\npub enum Error {\n #[error(\"Attach Error: {0}\")]\n Attach(String)"
},
{
"path": "src/lib.rs",
"chars": 1342,
"preview": "//! A high-performance Merkle key/value store.\n//!\n//! Merk is a crypto key/value store - more specifically, it's a Merk"
},
{
"path": "src/merk/chunks.rs",
"chars": 16224,
"preview": "//! Provides `ChunkProducer`, which creates chunk proofs for full replication of\n//! a Merk.\n\nuse super::Merk;\nuse crate"
},
{
"path": "src/merk/mod.rs",
"chars": 28870,
"preview": "pub mod chunks;\npub mod restore;\npub mod snapshot;\n\nuse std::cmp::Ordering;\nuse std::collections::LinkedList;\nuse std::p"
},
{
"path": "src/merk/restore.rs",
"chars": 15229,
"preview": "//! Provides `Restorer`, which can create a replica of a Merk instance by\n//! receiving chunk proofs.\n\nuse super::Merk;\n"
},
{
"path": "src/merk/snapshot.rs",
"chars": 9899,
"preview": "//! In-memory snapshots of database state.\n//!\n//! Snapshots are read-only views of the database state at a particular p"
},
{
"path": "src/owner.rs",
"chars": 3271,
"preview": "use std::ops::{Deref, DerefMut};\n\n/// A container type which holds a value that may be temporarily owned by a\n/// consum"
},
{
"path": "src/proofs/chunk.rs",
"chars": 16257,
"preview": "#[cfg(feature = \"full\")]\nuse {\n super::tree::{execute, Tree as ProofTree},\n crate::tree::Hash,\n crate::tree::Tr"
},
{
"path": "src/proofs/encoding.rs",
"chars": 7367,
"preview": "use std::io::{Read, Write};\n\nuse ed::{Decode, Encode, Terminated};\n\nuse super::{Node, Op};\nuse crate::error::Result;\nuse"
},
{
"path": "src/proofs/mod.rs",
"chars": 1144,
"preview": "pub mod chunk;\npub mod encoding;\npub mod query;\npub mod tree;\n\nuse crate::tree::Hash;\n\npub use encoding::{encode_into, D"
},
{
"path": "src/proofs/query/map.rs",
"chars": 21859,
"preview": "use super::super::Node;\nuse crate::{Error, Result};\nuse std::collections::btree_map;\nuse std::collections::BTreeMap;\nuse"
},
{
"path": "src/proofs/query/mod.rs",
"chars": 53106,
"preview": "mod map;\n\n#[cfg(feature = \"full\")]\nuse {super::Op, std::collections::LinkedList};\n\nuse super::tree::execute;\nuse super::"
},
{
"path": "src/proofs/tree.rs",
"chars": 12238,
"preview": "use super::{Node, Op};\nuse crate::error::{Error, Result};\nuse crate::tree::{kv_hash, node_hash, Hash, Hasher, NULL_HASH}"
},
{
"path": "src/test_utils/crash_merk.rs",
"chars": 2306,
"preview": "use crate::{Merk, Result};\nuse std::fs;\nuse std::mem::ManuallyDrop;\nuse std::ops::{Deref, DerefMut};\nuse std::path::Path"
},
{
"path": "src/test_utils/mod.rs",
"chars": 4399,
"preview": "#![allow(missing_docs)]\n\nmod crash_merk;\nmod temp_merk;\n\nuse crate::tree::{Batch, BatchEntry, NoopCommit, Op, PanicSourc"
},
{
"path": "src/test_utils/temp_merk.rs",
"chars": 1407,
"preview": "use crate::{Merk, Result};\nuse std::env::temp_dir;\nuse std::ops::{Deref, DerefMut};\nuse std::path::{Path, PathBuf};\nuse "
},
{
"path": "src/tree/commit.rs",
"chars": 1050,
"preview": "use super::Tree;\nuse crate::error::Result;\n\n/// To be used when committing a tree (writing it to a store after applying "
},
{
"path": "src/tree/debug.rs",
"chars": 2994,
"preview": "use super::{Link, Tree};\nuse colored::Colorize;\nuse std::fmt::{Debug, Formatter, Result};\n\nimpl Debug for Tree {\n // "
},
{
"path": "src/tree/encoding.rs",
"chars": 6459,
"preview": "use std::io::Read;\n\nuse crate::Result;\n\nuse super::{kv::KV, Link, Tree, TreeInner};\nuse ed::{Decode, Encode};\n\nimpl Tree"
},
{
"path": "src/tree/fuzz_tests.rs",
"chars": 4019,
"preview": "#![cfg(test)]\n\nuse crate::test_utils::*;\nuse crate::tree::*;\nuse rand::prelude::*;\nuse std::cell::RefCell;\nuse std::coll"
},
{
"path": "src/tree/hash.rs",
"chars": 1521,
"preview": "use sha2::{Digest, Sha512_256};\nuse std::{convert::TryFrom, num::TryFromIntError};\n\n/// The hash algorithm used for both"
},
{
"path": "src/tree/iter.rs",
"chars": 2602,
"preview": "use super::Tree;\n\n/// An entry stored on an `Iter`'s stack, containing a reference to a `Tree`,\n/// and its traversal st"
},
{
"path": "src/tree/kv.rs",
"chars": 3713,
"preview": "use super::hash::{kv_hash, Hash, Hasher, HASH_LENGTH, NULL_HASH};\nuse ed::{Decode, Encode, Result};\nuse std::{\n io::{"
},
{
"path": "src/tree/link.rs",
"chars": 15874,
"preview": "use std::cmp::max;\nuse std::io::{Read, Write};\n\nuse ed::{Decode, Encode, Result, Terminated};\n\nuse super::hash::Hash;\nus"
},
{
"path": "src/tree/mod.rs",
"chars": 19539,
"preview": "mod commit;\n#[cfg(feature = \"full\")]\nmod debug;\nmod encoding;\nmod fuzz_tests;\nmod hash;\nmod iter;\nmod kv;\nmod link;\nmod "
},
{
"path": "src/tree/ops.rs",
"chars": 19335,
"preview": "use super::{Fetch, Tree, Walker};\nuse crate::error::Result;\nuse std::collections::LinkedList;\nuse std::fmt;\nuse Op::*;\n\n"
},
{
"path": "src/tree/walk/fetch.rs",
"chars": 803,
"preview": "use super::super::{Link, Tree};\nuse crate::error::{Error, Result};\n\n/// A source of data to be used by the tree when enc"
},
{
"path": "src/tree/walk/mod.rs",
"chars": 7143,
"preview": "mod fetch;\nmod ref_walker;\n\nuse super::{Link, Tree};\nuse crate::error::Result;\nuse crate::owner::Owner;\npub use fetch::F"
},
{
"path": "src/tree/walk/ref_walker.rs",
"chars": 1745,
"preview": "use super::super::{Link, Tree};\nuse super::Fetch;\nuse crate::error::Result;\n\n/// Allows read-only traversal of a `Tree`,"
}
]
About this extraction
This page contains the full source code of the nomic-io/merk GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 40 files (328.6 KB), approximately 88.8k tokens, and a symbol index with 546 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.