[ { "path": ".github/workflows/ci.yml", "content": "name: CI\non: [push, pull_request, workflow_dispatch]\npermissions:\n contents: read\n\njobs:\n test:\n name: Test\n runs-on: ubuntu-latest\n timeout-minutes: 10\n\n steps:\n - uses: actions/checkout@v3\n - uses: dtolnay/rust-toolchain@1.93.1\n id: toolchain\n with:\n components: clippy, rustfmt\n - uses: actions/cache@v3\n with:\n path: target\n key: ${{runner.os}}-target-${{steps.toolchain.outputs.cachekey}}-${{hashFiles('Cargo.lock')}}\n - run: cargo build --bins --tests\n - run: cargo test\n - run: cargo clippy --tests --no-deps -- -D warnings\n - run: cargo fmt --check\n - run: cargo doc --no-deps \n env:\n RUSTDOCFLAGS: -D warnings" }, { "path": ".gitignore", "content": "/cluster/toydb*/data\n/data\n/docs/crate/target\n/target\n.DS_Store\n.vscode/\n**/*.rs.bk\n" }, { "path": "Cargo.toml", "content": "[package]\nname = \"toydb\"\nversion = \"1.0.0\"\ndescription = \"A simple distributed SQL database, built for education\"\nauthors = [\"Erik Grinaker \"]\nlicense = \"Apache-2.0\"\nhomepage = \"https://github.com/erikgrinaker/toydb\"\nrepository = \"https://github.com/erikgrinaker/toydb\"\nedition = \"2024\"\ndefault-run = \"toydb\"\npublish = false\n\n[lib]\ndoctest = false\n\n[dependencies]\nbincode = { version = \"2.0\", features = [\"serde\"] }\nclap = { version = \"4.5\", features = [\"cargo\", \"derive\"] }\nconfig = \"0.15\"\ncrossbeam = { version = \"0.8\", features = [\"crossbeam-channel\"] }\ndyn-clone = \"1.0\"\nfs4 = \"0.13\"\nhdrhistogram = \"7.5\"\nitertools = \"0.14\"\nlog = \"0.4\"\npetname = \"2.0.2\"\nrand = \"0.10\"\nregex = \"1.12\"\nrustyline = \"17.0\"\nrustyline-derive = \"0.11\"\nserde = { version = \"1.0\", features = [\"derive\"] }\nserde_bytes = \"0.11\"\nsimplelog = \"0.12\"\nuuid = { version = \"1.21\", features = [\"serde\", \"v4\"] }\n\n[dev-dependencies]\nescargot = \"0.5\"\ngoldenscript = \"0.7\"\nhex = \"0.4\"\npaste = \"1.0\"\nserde_json = \"1.0\"\ntempfile = \"3.25\"\ntest-case = \"3.3\"\ntest_each_file = \"0.3\"\n" }, { "path": "LICENSE", "content": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License." }, { "path": "README.md", "content": "# toyDB\n\nDistributed SQL database in Rust, built from scratch as an educational project. Main features:\n\n* [Raft distributed consensus][raft] for linearizable state machine replication.\n\n* [ACID transactions][txn] with MVCC-based snapshot isolation.\n\n* [Pluggable storage engine][storage] with [BitCask][bitcask] and [in-memory][memory] backends.\n\n* [Iterator-based query engine][query] with [heuristic optimization][optimizer] and time-travel \n support.\n\n* [SQL interface][sql] including joins, aggregates, and transactions.\n\ntoyDB is intended to be simple and understandable, and also functional and correct. Other aspects\nlike performance, scalability, and availability are non-goals -- these are major sources of\ncomplexity in production-grade databases, and obscure the basic underlying concepts. Shortcuts have\nbeen taken where possible.\n\nI originally wrote toyDB in 2020 to learn more about database internals. Since then, I've spent\nseveral years building real distributed SQL databases at\n[CockroachDB](https://github.com/cockroachdb/cockroach) and\n[Neon](https://github.com/neondatabase/neon). Based on this experience, I've rewritten toyDB as a\nsimple illustration of the architecture and concepts behind distributed SQL databases.\n\n[raft]: https://github.com/erikgrinaker/toydb/blob/main/src/raft/mod.rs\n[txn]: https://github.com/erikgrinaker/toydb/blob/main/src/storage/mvcc.rs\n[storage]: https://github.com/erikgrinaker/toydb/blob/main/src/storage/engine.rs\n[bitcask]: https://github.com/erikgrinaker/toydb/blob/main/src/storage/bitcask.rs\n[memory]: https://github.com/erikgrinaker/toydb/blob/main/src/storage/memory.rs\n[query]: https://github.com/erikgrinaker/toydb/blob/main/src/sql/execution/executor.rs\n[optimizer]: https://github.com/erikgrinaker/toydb/blob/main/src/sql/planner/optimizer.rs\n[sql]: https://github.com/erikgrinaker/toydb/blob/main/src/sql/parser/parser.rs\n\n## Documentation\n\n* [Architecture guide](docs/architecture/index.md): a guided tour of toyDB's code and architecture.\n\n* [SQL examples](docs/examples.md): walkthrough of toyDB's SQL features.\n\n* [SQL reference](docs/sql.md): reference documentation for toyDB's SQL dialect.\n\n* [References](docs/references.md): research materials used while building toyDB.\n\n## Usage\n\nWith a [Rust compiler](https://www.rust-lang.org/tools/install) installed, a local five-node \ncluster can be built and started as:\n\n```\n$ ./cluster/run.sh\nStarting 5 nodes on ports 9601-9605 with data under cluster/*/data/.\nTo connect to node 1, run: cargo run --release --bin toysql\n\ntoydb4 21:03:55 [INFO] Listening on [::1]:9604 (SQL) and [::1]:9704 (Raft)\ntoydb1 21:03:55 [INFO] Listening on [::1]:9601 (SQL) and [::1]:9701 (Raft)\ntoydb2 21:03:55 [INFO] Listening on [::1]:9602 (SQL) and [::1]:9702 (Raft)\ntoydb3 21:03:55 [INFO] Listening on [::1]:9603 (SQL) and [::1]:9703 (Raft)\ntoydb5 21:03:55 [INFO] Listening on [::1]:9605 (SQL) and [::1]:9705 (Raft)\ntoydb2 21:03:56 [INFO] Starting new election for term 1\n[...]\ntoydb2 21:03:56 [INFO] Won election for term 1, becoming leader\n```\n\nA command-line client can be built and used with node 1 on `localhost:9601`:\n\n```\n$ cargo run --release --bin toysql\nConnected to toyDB node n1. Enter !help for instructions.\ntoydb> CREATE TABLE movies (id INTEGER PRIMARY KEY, title VARCHAR NOT NULL);\ntoydb> INSERT INTO movies VALUES (1, 'Sicario'), (2, 'Stalker'), (3, 'Her');\ntoydb> SELECT * FROM movies;\n1, 'Sicario'\n2, 'Stalker'\n3, 'Her'\n```\n\ntoyDB supports most common SQL features, including joins, aggregates, and transactions. Below is an\n`EXPLAIN` query plan of a more complex query (fetches all movies from studios that have released any\nmovie with an IMDb rating of 8 or more):\n\n```\ntoydb> EXPLAIN SELECT m.title, g.name AS genre, s.name AS studio, m.rating\n FROM movies m JOIN genres g ON m.genre_id = g.id,\n studios s JOIN movies good ON good.studio_id = s.id AND good.rating >= 8\n WHERE m.studio_id = s.id\n GROUP BY m.title, g.name, s.name, m.rating, m.released\n ORDER BY m.rating DESC, m.released ASC, m.title ASC;\n\nRemap: m.title, genre, studio, m.rating (dropped: m.released)\n└─ Order: m.rating desc, m.released asc, m.title asc\n └─ Projection: m.title, g.name as genre, s.name as studio, m.rating, m.released\n └─ Aggregate: m.title, g.name, s.name, m.rating, m.released\n └─ HashJoin: inner on m.studio_id = s.id\n ├─ HashJoin: inner on m.genre_id = g.id\n │ ├─ Scan: movies as m\n │ └─ Scan: genres as g\n └─ HashJoin: inner on s.id = good.studio_id\n ├─ Scan: studios as s\n └─ Scan: movies as good (good.rating > 8 OR good.rating = 8)\n```\n\n## Architecture\n\ntoyDB's architecture is fairly typical for a distributed SQL database: a transactional\nkey/value store managed by a Raft cluster with a SQL query engine on top. See the\n[architecture guide](./docs/architecture/index.md) for more details.\n\n[![toyDB architecture](./docs/architecture/images/architecture.svg)](./docs/architecture/index.md)\n\n## Tests\n\ntoyDB mainly uses [Goldenscripts](https://github.com/erikgrinaker/goldenscript) for tests. These \nscript various scenarios, capture events and output, and later assert that the behavior remains the \nsame. See e.g.:\n\n* [Raft cluster tests](https://github.com/erikgrinaker/toydb/tree/main/src/raft/testscripts/node)\n* [MVCC transaction tests](https://github.com/erikgrinaker/toydb/tree/main/src/storage/testscripts/mvcc)\n* [SQL execution tests](https://github.com/erikgrinaker/toydb/tree/main/src/sql/testscripts)\n* [End-to-end tests](https://github.com/erikgrinaker/toydb/tree/main/tests/scripts)\n\nRun tests with `cargo test`, or have a look at the latest \n[CI run](https://github.com/erikgrinaker/toydb/actions/workflows/ci.yml).\n\n## Benchmarks\n\ntoyDB is not optimized for performance, but comes with a `workload` benchmark tool that can run \nvarious workloads against a toyDB cluster. For example:\n\n```sh\n# Start a 5-node toyDB cluster.\n$ ./cluster/run.sh\n[...]\n\n# Run a read-only benchmark via all 5 nodes.\n$ cargo run --release --bin workload read\nPreparing initial dataset... done (0.179s)\nSpawning 16 workers... done (0.006s)\nRunning workload read (rows=1000 size=64 batch=1)...\n\nTime Progress Txns Rate p50 p90 p99 pMax\n1.0s 13.1% 13085 13020/s 1.3ms 1.5ms 1.9ms 8.4ms\n2.0s 27.2% 27183 13524/s 1.3ms 1.5ms 1.8ms 8.4ms\n3.0s 41.3% 41301 13702/s 1.2ms 1.5ms 1.8ms 8.4ms\n4.0s 55.3% 55340 13769/s 1.2ms 1.5ms 1.8ms 8.4ms\n5.0s 70.0% 70015 13936/s 1.2ms 1.5ms 1.8ms 8.4ms\n6.0s 84.7% 84663 14047/s 1.2ms 1.4ms 1.8ms 8.4ms\n7.0s 99.6% 99571 14166/s 1.2ms 1.4ms 1.7ms 8.4ms\n7.1s 100.0% 100000 14163/s 1.2ms 1.4ms 1.7ms 8.4ms\n\nVerifying dataset... done (0.002s)\n```\n\nThe available workloads are:\n\n* `read`: single-row primary key lookups.\n* `write`: single-row inserts to sequential primary keys.\n* `bank`: bank transfers between various customers and accounts. To make things interesting, this\n includes joins, secondary indexes, sorting, and conflicts.\n\nFor more information about workloads and parameters, run `cargo run --bin workload -- --help`.\n\nExample workload results are listed below. Write performance is atrocious, due to\n[fsync](https://en.wikipedia.org/wiki/Sync_(Unix)) and a lack of write batching in the Raft layer.\nDisabling fsync, or using the in-memory engine, significantly improves write performance (at the\nexpense of durability).\n\n| Workload | BitCask | BitCask w/o fsync | Memory |\n|----------|-------------|-------------------|-------------|\n| `read` | 14163 txn/s | 13941 txn/s | 13949 txn/s |\n| `write` | 35 txn/s | 4719 txn/s | 7781 txn/s |\n| `bank` | 21 txn/s | 1120 txn/s | 1346 txn/s |\n\n## Debugging\n\n[VSCode](https://code.visualstudio.com) and the [CodeLLDB](https://marketplace.visualstudio.com/items?itemName=vadimcn.vscode-lldb)\nextension can be used to debug toyDB, with the debug configuration under `.vscode/launch.json`.\n\nUnder the \"Run and Debug\" tab, select e.g. \"Debug executable 'toydb'\" or \"Debug unit tests in\nlibrary 'toydb'\".\n\n## Credits\n\nThe toyDB logo is courtesy of [@jonasmerlin](https://github.com/jonasmerlin)." }, { "path": "cluster/run.sh", "content": "#!/usr/bin/env bash\n#\n# This script builds and runs a 5-node toyDB cluster listening on ports\n# 9601-9605. Config and data is stored under the toydb* directories.\n# To connect a toysql client to node 1 on port 9601, run:\n#\n# cargo run --release --bin toysql\n\nset -euo pipefail\n\n# Change into the script directory.\ncd \"$(dirname $0)\"\n\n# Build toyDB using release optimizations.\ncargo build --release --bin toydb\n\n# Start nodes 1-5 in the background, prefixing their output with the node ID.\necho \"Starting 5 nodes on ports 9601-9605 with data under cluster/*/data/.\"\necho \"To connect to node 1, run: cargo run --release --bin toysql\"\necho \"\"\n\nfor ID in 1 2 3 4 5; do\n (cargo run -q --release -- -c toydb$ID/toydb.yaml 2>&1 | sed -e \"s/\\\\(.*\\\\)/toydb$ID \\\\1/g\") &\ndone\n\n# Wait for the background processes to exit. Kill all toyDB processes when the\n# script exits (e.g. via Ctrl-C).\ntrap 'kill -TERM -- -$$ 2>/dev/null' INT TERM EXIT\nwait" }, { "path": "cluster/toydb1/toydb.yaml", "content": "id: 1\ndata_dir: toydb1/data\nlisten_sql: localhost:9601\nlisten_raft: localhost:9701\npeers:\n '2': localhost:9702\n '3': localhost:9703\n '4': localhost:9704\n '5': localhost:9705" }, { "path": "cluster/toydb2/toydb.yaml", "content": "id: 2\ndata_dir: toydb2/data\nlisten_sql: localhost:9602\nlisten_raft: localhost:9702\npeers:\n '1': localhost:9701\n '3': localhost:9703\n '4': localhost:9704\n '5': localhost:9705" }, { "path": "cluster/toydb3/toydb.yaml", "content": "id: 3\ndata_dir: toydb3/data\nlisten_sql: localhost:9603\nlisten_raft: localhost:9703\npeers:\n '1': localhost:9701\n '2': localhost:9702\n '4': localhost:9704\n '5': localhost:9705" }, { "path": "cluster/toydb4/toydb.yaml", "content": "id: 4\ndata_dir: toydb4/data\nlisten_sql: localhost:9604\nlisten_raft: localhost:9704\npeers:\n '1': localhost:9701\n '2': localhost:9702\n '3': localhost:9703\n '5': localhost:9705" }, { "path": "cluster/toydb5/toydb.yaml", "content": "id: 5\ndata_dir: toydb5/data\nlisten_sql: localhost:9605\nlisten_raft: localhost:9705\npeers:\n '1': localhost:9701\n '2': localhost:9702\n '3': localhost:9703\n '4': localhost:9704" }, { "path": "config/toydb.yaml", "content": "# The node ID (must be unique in the cluster), and map of peer IDs and Raft\n# addresses (empty for single node).\nid: 1\npeers: {}\n\n# Addresses to listen for SQL and Raft connections on.\nlisten_sql: localhost:9601\nlisten_raft: localhost:9701\n\n# The log level. Valid values are DEBUG, INFO, WARN, and ERROR.\nlog_level: INFO\n\n# Node data directory. The Raft log is stored in the file \"raft\", and the SQL\n# database in \"sql\".\ndata_dir: data\n\n# Storage engine to use for the Raft log and SQL database.\n#\n# * bitcask (default): an append-only log-structured store.\n# * memory: an in-memory store using the Rust standard library's BTreeMap.\nstorage_raft: bitcask\nstorage_sql: bitcask\n\n# Whether to fsync writes to disk. Disabling this yields much better write\n# performance, but may lose data on host crashes and violate Raft guarantees. It\n# only affects Raft log writes (the SQL state machine is never fsynced since it\n# can be reconstructed from the Raft log).\nfsync: true\n\n# The minimum garbage fraction and bytes to trigger Bitcask log compaction on\n# node startup.\ncompact_threshold: 0.2\ncompact_min_bytes: 1000000" }, { "path": "docs/architecture/README.md", "content": "See [`index.md`](index.md)." }, { "path": "docs/architecture/client.md", "content": "# Client\n\nThe toyDB client is in the [`client`](https://github.com/erikgrinaker/toydb/blob/213e5c02b09f1a3cac6a8bbd0a81773462f367f5/src/client.rs)\nmodule. It uses the same Bincode-based protocol that we saw in the server section, sending\n`toydb::Request` and receiving `toydb::Response`.\n\n## Client Library\n\nThe main client library `toydb::Client` is used to communicate with a toyDB server:\n\nhttps://github.com/erikgrinaker/toydb/blob/213e5c02b09f1a3cac6a8bbd0a81773462f367f5/src/client.rs#L15-L24\n\nWhen initialized, it connects to a toyDB server over TCP, which establishes a SQL session for it:\n\nhttps://github.com/erikgrinaker/toydb/blob/213e5c02b09f1a3cac6a8bbd0a81773462f367f5/src/client.rs#L27-L33\n\nIt can then send Bincode-encoded `toydb::Request` to the server, and receive `toydb::Response`\nback.\n\nhttps://github.com/erikgrinaker/toydb/blob/213e5c02b09f1a3cac6a8bbd0a81773462f367f5/src/client.rs#L35-L40\n\n\nIn particular, `Client::execute` can be used to execute arbitrary SQL statements in the client's\ncurrent session:\n\nhttps://github.com/erikgrinaker/toydb/blob/213e5c02b09f1a3cac6a8bbd0a81773462f367f5/src/client.rs#L42-L56\n\n## `toysql` Binary\n\nHowever, `toydb::Client` is a programmatic API, and we want a more convenient user interface.\nThe `toysql` client in [`src/bin/toysql.rs`](https://github.com/erikgrinaker/toydb/blob/0839215770e31f1e693d5cccf20a68210deaaa3f/src/bin/toysql.rs)\nprovides a typical [REPL](https://en.wikipedia.org/wiki/Read–eval–print_loop) (read-evaluate-print loop) where users can enter SQL statements and view the results.\n\nLike `toydb`, `toysql` is a tiny [`clap`](https://docs.rs/clap/latest/clap/) command that takes a\ntoyDB server address to connect to and starts an interactive shell:\n\nhttps://github.com/erikgrinaker/toydb/blob/0839215770e31f1e693d5cccf20a68210deaaa3f/src/bin/toysql.rs#L29-L53\n\nIt first attempts to connect to the toyDB server using the `toydb::Client` client, and then starts\nan interactive shell using the [Rustyline](https://docs.rs/rustyline/latest/rustyline/) library.\n\nhttps://github.com/erikgrinaker/toydb/blob/0839215770e31f1e693d5cccf20a68210deaaa3f/src/bin/toysql.rs#L55-L81\n\nThe shell is simply a loop that prompts the user to input a SQL statement:\n\nhttps://github.com/erikgrinaker/toydb/blob/0839215770e31f1e693d5cccf20a68210deaaa3f/src/bin/toysql.rs#L216-L250\n\nEach statement is the executed against the server via `toydb::Client::execute`, and the response\nis formatted and printed as output:\n\nhttps://github.com/erikgrinaker/toydb/blob/0839215770e31f1e693d5cccf20a68210deaaa3f/src/bin/toysql.rs#L83-L92\n\nhttps://github.com/erikgrinaker/toydb/blob/0839215770e31f1e693d5cccf20a68210deaaa3f/src/bin/toysql.rs#L175-L204\n\nAnd with that, we have a fully functional SQL database system and can run queries to our heart's\ncontent. Have fun!\n\n---\n\n

\n← Server\n

" }, { "path": "docs/architecture/encoding.md", "content": "# Key/Value Encoding\n\nThe key/value store uses binary `Vec` keys and values, so we need an encoding scheme to \ntranslate between in-memory Rust data structures and the on-disk binary data. This is provided by\nthe [`encoding`](https://github.com/erikgrinaker/toydb/tree/213e5c02b09f1a3cac6a8bbd0a81773462f367f5/src/encoding)\nmodule, with separate schemes for key and value encoding.\n\n## `Bincode` Value Encoding\n\nValues are encoded using [Bincode](https://github.com/bincode-org/bincode), a third-party binary\nencoding scheme for Rust. Bincode is convenient because it can easily encode any arbitrary Rust\ndata type. But we could also have chosen e.g. [JSON](https://en.wikipedia.org/wiki/JSON),\n[Protobuf](https://protobuf.dev), [MessagePack](https://msgpack.org/), or any other encoding.\n\nWe won't dwell on the actual binary format here, see the [Bincode specification](https://git.sr.ht/~stygianentity/bincode/tree/trunk/item/docs/spec.md)\nfor details.\n\nTo use a consistent configuration for all encoding and decoding, we provide helper functions in\nthe [`encoding::bincode`](https://github.com/erikgrinaker/toydb/blob/213e5c02b09f1a3cac6a8bbd0a81773462f367f5/src/encoding/bincode.rs)\nmodule which use `bincode::config::standard()`.\n\nhttps://github.com/erikgrinaker/toydb/blob/0ce1fb34349fda043cb9905135f103bceb4395b4/src/encoding/bincode.rs#L15-L27\n\nBincode uses the very common [Serde](https://serde.rs) framework for its API. toyDB also provides an\n`encoding::Value` helper trait for value types which adds automatic `encode()` and `decode()`\nmethods:\n\nhttps://github.com/erikgrinaker/toydb/blob/b57ae6502e93ea06df00d94946a7304b7d60b977/src/encoding/mod.rs#L39-L68\n\nHere's an example of how this can be used to encode and decode an arbitrary `Dog` data type:\n\n```rust\n#[derive(serde::Serialize, serde::Deserialize)]\nstruct Dog {\n name: String,\n age: u8,\n good_boy: bool,\n}\n\nimpl encoding::Value for Dog {}\n\nlet pluto = Dog { name: \"Pluto\".into(), age: 4, good_boy: true };\nlet bytes = pluto.encode();\nprintln!(\"{bytes:02x?}\");\n\n// Outputs [05, 50, 6c, 75, 74, 6f, 04, 01]:\n//\n// * Length of string \"Pluto\": 05.\n// * String \"Pluto\": 50 6c 75 74 6f.\n// * Age 4: 04.\n// * Good boy: 01 (true).\n\nlet pluto = Dog::decode(&bytes)?; // gives us back Pluto\n```\n\n## `Keycode` Key Encoding\n\nUnlike values, keys can't just use any binary encoding like Bincode. As mentioned in the storage\nsection, the storage engine sorts data by key to enable range scans. The key encoding must therefore\npreserve the [lexicographical order](https://en.wikipedia.org/wiki/Lexicographic_order) of the\nencoded values: the binary byte slices must sort in the same order as the original values.\n\nAs an example of why we can't just use Bincode, consider the strings \"house\" and \"key\". These should\nbe sorted in alphabetical order: \"house\" before \"key\". However, Bincode encodes strings prefixed by\ntheir length, so \"key\" would be sorted before \"house\" in binary form:\n\n```\n03 6b 65 79 ← 3 bytes: key\n05 68 6f 75 73 65 ← 5 bytes: house\n```\n\nFor similar reasons, we can't just encode numbers in their native binary form: the\n[little-endian](https://en.wikipedia.org/wiki/Endianness) representation will order very large\nnumbers before small numbers, and the [sign bit](https://en.wikipedia.org/wiki/Sign_bit) will order\npositive numbers before negative numbers. This would violate the ordering of natural numbers.\n\nWe also have to be careful with value sequences, which should be ordered element-wise. For example,\nthe pair (\"a\", \"xyz\") should be ordered before (\"ab\", \"cd\"), so we can't just encode the strings\none after the other like \"axyz\" and \"abcd\" since that would sort (\"ab\", \"cd\") first.\n\ntoyDB provides an order-preserving encoding called \"Keycode\" in the [`encoding::keycode`](https://github.com/erikgrinaker/toydb/blob/213e5c02b09f1a3cac6a8bbd0a81773462f367f5/src/encoding/keycode.rs)\nmodule. Like Bincode, the Keycode encoding is not self-describing: the binary data does not say what\nthe data type is, the caller must provide a type to decode into. It only supports a handful of\nprimitive data types, and only needs to order values of the same type.\n\nKeycode is implemented as a [Serde](https://serde.rs) (de)serializer, which requires a lot of\nboilerplate code to satisfy the trait, but we'll just focus on the actual encoding. The encoding\nscheme is as follows:\n\n* `bool`: `00` for `false` and `01` for `true`.\n\n https://github.com/erikgrinaker/toydb/blob/2027641004989355c2162bbd9eeefcc991d6b29b/src/encoding/keycode.rs#L113-L117\n\n* `u64`: the [big-endian](https://en.wikipedia.org/wiki/Endianness) binary encoding.\n\n https://github.com/erikgrinaker/toydb/blob/2027641004989355c2162bbd9eeefcc991d6b29b/src/encoding/keycode.rs#L157-L161\n\n* `i64`: the [big-endian](https://en.wikipedia.org/wiki/Endianness) binary encoding, but with the\n sign bit flipped to order negative numbers before positive ones.\n\n https://github.com/erikgrinaker/toydb/blob/2027641004989355c2162bbd9eeefcc991d6b29b/src/encoding/keycode.rs#L131-L143\n\n* `f64`: the [big-endian IEEE 754](https://en.wikipedia.org/wiki/Double-precision_floating-point_format)\n binary encoding, but with the sign bit flipped, and all bits flipped for negative numbers, to\n order negative numbers correctly.\n\n https://github.com/erikgrinaker/toydb/blob/2027641004989355c2162bbd9eeefcc991d6b29b/src/encoding/keycode.rs#L167-L179\n\n* `Vec`: terminated by `00 00`, with `00` escaped as `00 ff` to disambiguate it.\n\n https://github.com/erikgrinaker/toydb/blob/2027641004989355c2162bbd9eeefcc991d6b29b/src/encoding/keycode.rs#L190-L205\n\n* `String`: like `Vec`.\n\n https://github.com/erikgrinaker/toydb/blob/2027641004989355c2162bbd9eeefcc991d6b29b/src/encoding/keycode.rs#L185-L188\n\n* `Vec`, `[T]`, `(T,)`: the concatenation of the inner values.\n\n https://github.com/erikgrinaker/toydb/blob/2027641004989355c2162bbd9eeefcc991d6b29b/src/encoding/keycode.rs#L295-L307\n\n* `enum`: the variant's numerical index as a `u8`, then the inner values (if any).\n\n https://github.com/erikgrinaker/toydb/blob/2027641004989355c2162bbd9eeefcc991d6b29b/src/encoding/keycode.rs#L223-L227\n\nLike `encoding::Value`, there is also an `encoding::Key` helper trait:\n\nhttps://github.com/erikgrinaker/toydb/blob/b57ae6502e93ea06df00d94946a7304b7d60b977/src/encoding/mod.rs#L20-L37\n\nDifferent kinds of keys are usually represented as enums. For example, if we wanted to store cars\nand video games, we could use:\n\n```rust\n#[derive(serde::Serialize, serde::Deserialize)]\nenum Key {\n Car(String, String, u64), // make, model, year\n Game(String, u64, Platform), // name, year, platform\n}\n\n#[derive(serde::Serialize, serde::Deserialize)]\nenum Platform {\n PC,\n PS5,\n Switch,\n Xbox,\n}\n\nimpl encoding::Key for Key {}\n\nlet returnal = Key::Game(\"Returnal\".into(), 2021, Platform::PS5);\nlet bytes = returnal.encode();\nprintln!(\"{bytes:02x?}\");\n\n// Outputs [01, 52, 65, 74, 75, 72, 6e, 61, 6c, 00, 00, 00, 00, 00, 00, 00, 00, 07, e5, 01].\n//\n// * Key::Game: 01\n// * Returnal: 52 65 74 75 72 6e 61 6c 00 00\n// * 2021: 00 00 00 00 00 00 07 e5\n// * Platform::PS5: 01\n\nlet returnal = Key::decode(&bytes)?;\n```\n\nBecause the keys are sorted in element-wise order, this would allow us to e.g. perform a prefix\nscan to fetch all platforms which Returnal (2021) was released on, or perform a range scan to fetch \nall models of Nissan Altima released between 2010 and 2015.\n\n---\n\n

\n← Storage Engine   |   MVCC Transactions →\n

" }, { "path": "docs/architecture/index.md", "content": "# toyDB Architecture\n\ntoyDB is a simple distributed SQL database, intended to illustrate how such systems are built. The\noverall structure is similar to real-world distributed databases, but the design and implementation\nhas been kept as simple as possible for understandability. Performance and scalability are explicit\nnon-goals, as these are major sources of complexity in real-world systems.\n\nThis guide will walk through toyDB's architecture and code from the bottom up, with plenty of links\nto the actual source code.\n\n> ℹ️ View on GitHub with a desktop browser for inline code listings.\n\n* [Overview](overview.md)\n * [Properties](overview.md#properties)\n * [Components](overview.md#components)\n* [Storage Engine](storage.md)\n * [`Memory` Storage Engine](storage.md#memory-storage-engine)\n * [`BitCask` Storage Engine](storage.md#bitcask-storage-engine)\n* [Key/Value Encoding](encoding.md)\n * [`Bincode` Value Encoding](encoding.md#bincode-value-encoding)\n * [`Keycode` Key Encoding](encoding.md#keycode-key-encoding)\n* [MVCC Transactions](mvcc.md)\n* [Raft Consensus](raft.md)\n * [Log Storage](raft.md#log-storage)\n * [State Machine Interface](raft.md#state-machine-interface)\n * [Node Roles](raft.md#node-roles)\n * [Node Interface and Communication](raft.md#node-interface-and-communication)\n * [Leader Election and Terms](raft.md#leader-election-and-terms)\n * [Client Requests and Forwarding](raft.md#client-requests-and-forwarding)\n * [Write Replication and Application](raft.md#write-replication-and-application)\n * [Read Processing](raft.md#read-processing)\n* [SQL Engine](sql.md)\n * [Data Model](sql-data.md)\n * [Data Types](sql-data.md#data-types)\n * [Schemas](sql-data.md#schemas)\n * [Expressions](sql-data.md#expressions)\n * [Storage](sql-storage.md)\n * [Key/Value Representation](sql-storage.md#keyvalue-representation)\n * [Schema Catalog](sql-storage.md#schema-catalog)\n * [Row Storage and Transactions](sql-storage.md#row-storage-and-transactions)\n * [Raft Replication](sql-raft.md)\n * [Parsing](sql-parser.md)\n * [Lexer](sql-parser.md#lexer)\n * [Abstract Syntax Tree](sql-parser.md#abstract-syntax-tree)\n * [Parser](sql-parser.md#parser)\n * [Planning](sql-planner.md)\n * [Execution Plan](sql-planner.md#execution-plan)\n * [Scope and Name Resolution](sql-planner.md#scope-and-name-resolution)\n * [Planner](sql-planner.md#planner)\n * [Optimization](sql-optimizer.md)\n * [Constant Folding](sql-optimizer.md#constant-folding)\n * [Filter Pushdown](sql-optimizer.md#filter-pushdown)\n * [Index Lookups](sql-optimizer.md#index-lookups)\n * [Hash Join](sql-optimizer.md#hash-join)\n * [Short Circuiting](sql-optimizer.md#short-circuiting)\n * [Execution](sql-execution.md)\n * [Plan Executor](sql-execution.md#plan-executor)\n * [Session Management](sql-execution.md#session-management)\n* [Server](server.md)\n * [Raft Routing](server.md#raft-routing)\n * [SQL Service](server.md#sql-service)\n * [`toydb` Binary](server.md#toydb-binary)\n* [Client](client.md)\n * [Client Library](client.md#client-library)\n * [`toysql` Binary](client.md#toysql-binary)\n\n---\n\n

\nOverview →\n

" }, { "path": "docs/architecture/mvcc.md", "content": "# MVCC Transactions\n\nTransactions are groups of reads and writes (e.g. to different keys) that are submitted together as\na single unit. For example, a bank transaction that transfers $100 from account A to account B might\nconsist of this group of reads and writes:\n\n```\na = get(A)\nb = get(B)\nif a < 100:\n error(\"insufficient balance\")\nset(A, a - 100)\nset(B, b + 100)\n```\n\ntoyDB provides [ACID](https://en.wikipedia.org/wiki/ACID) transactions, a set of very strong\nguarantees:\n\n* **Atomicity:** all of the writes take effect as an single, atomic unit, at the same instant, when\n they are _committed_. Other users will never see some of the writes without the others.\n\n* **Consistency:** database constraints are never violated (e.g. referential integrity or uniqueness\n contraints). We'll see how this is implemented later in the SQL execution layer.\n\n* **Isolation:** users should appear to have the entire database to themselves, unaffected by other\n simultaneous users. Two transactions may conflict, in which case one has to retry, but if a\n transaction succeeds then the user knows with certainty that the operations were executed without\n interference by anyone else. This eliminates the risk of [race conditions](https://en.wikipedia.org/wiki/Race_condition).\n \n* **Durability:** committed writes are never lost (even if the system crashes).\n\nTo illustrate how transactions work, here's an example MVCC test script where two concurrent users\nmodify a set of bank accounts (there's many [other test scripts](https://github.com/erikgrinaker/toydb/tree/aa14deb71f650249ce1cab8828ed7bcae2c9206e/src/storage/testscripts/mvcc)\nthere too):\n\nhttps://github.com/erikgrinaker/toydb/blob/a73e24b7e77671b9f466e0146323cd69c3e27bdf/src/storage/testscripts/mvcc/bank#L1-L69\n\nTo provide these guarantees, toyDB uses a common technique called\n[Multi-Version Concurrency Control](https://en.wikipedia.org/wiki/Multiversion_concurrency_control)\n(MVCC). It is implemented at the key/value storage level, in the [`storage::mvcc`](https://github.com/erikgrinaker/toydb/blob/8f8eae0dcf70b1a0df2e853b1f6600e0c7075340/src/storage/mvcc.rs)\nmodule. It uses a `storage::Engine` for actual data storage.\n\nhttps://github.com/erikgrinaker/toydb/blob/8f8eae0dcf70b1a0df2e853b1f6600e0c7075340/src/storage/mvcc.rs#L220-L231\n\nMVCC provides an [isolation level](https://en.wikipedia.org/wiki/Isolation_(database_systems)#Isolation_levels)\ncalled [snapshot isolation](https://en.wikipedia.org/wiki/Snapshot_isolation): a transaction sees a\nsnapshot of the database as it was when the transaction began. Any later changes are invisible to\nit.\n\nIt does this by storing historical versions of key/value pairs. The version number is simply a\nnumber that's incremented for every new transaction:\n\nhttps://github.com/erikgrinaker/toydb/blob/8f8eae0dcf70b1a0df2e853b1f6600e0c7075340/src/storage/mvcc.rs#L155-L158\n\nEach transaction has its own unique version number. When it writes a key/value pair it appends its\nversion number to the key as `Key::Version(&[u8], Version)` (using the Keycode encoding we've seen\npreviously). If an old version of the key already exists, it will have a different version number\nsuffix and therefore be stored as a separate key in the storage engine. Deleted keys are versions\nwith a special tombstone value.\n\nhttps://github.com/erikgrinaker/toydb/blob/8f8eae0dcf70b1a0df2e853b1f6600e0c7075340/src/storage/mvcc.rs#L183-L189\n\nHere's a simple diagram of what a history of versions 1 to 5 of keys `a` to `d` might look like:\n\nhttps://github.com/erikgrinaker/toydb/blob/8f8eae0dcf70b1a0df2e853b1f6600e0c7075340/src/storage/mvcc.rs#L11-L26\n\nAdditionally, we need to keep track of the currently ongoing (uncommitted) transaction versions,\nknown as the \"active set\".\n\nWith versioning and the active set, we can summarize the MVCC protocol with a few simple rules:\n\n1. When a new transaction begins, it:\n * Obtains the next available version number.\n * Takes a snapshot of the active set (other uncommitted transactions).\n * Adds its version number to the active set.\n\n2. When the transaction reads a key, it:\n * Returns the latest version of the key at or below its own version.\n * Ignores versions above its own version.\n * Ignores versions in its active set snapshot.\n\n3. When the transaction writes a key, it:\n * Looks for a key version above its own version; errors if found.\n * Looks for a key version in its active set snapshot; errors if found.\n * Writes a key/value pair with its own version.\n\n4. When the transaction commits, it:\n * Flushes all writes to disk.\n * Removes itself from the active set.\n\nThe magic happens when the transaction removes itself from the active set. This is a single, atomic\noperation, and when it completes all of its writes immediately become visible to _new_ transactions.\nHowever, ongoing transactions still won't see these writes, because the version is still in their\nactive set snapshot or at a later version (hence they are isolated from this transaction).\n\nFurthermore, the transaction could see its own uncommitted writes even though noone else could, and\nif any writes conflicted with another transaction it would error out and have to retry.\n\nNot only that, this also allows us to do time-travel queries, where we can query the database as it\nwas at any time in the past: we simply pick a version number to read at.\n\nThere are a few more details that we've left out here: transaction rollbacks need to keep track of\nthe writes and undo them, and read-only queries can avoid allocating new version numbers. We also\ndon't garbage collect old version, for simplicity. See the module documentation for more details:\n\nhttps://github.com/erikgrinaker/toydb/blob/8f8eae0dcf70b1a0df2e853b1f6600e0c7075340/src/storage/mvcc.rs#L1-L140\n\nLet's walk through a simple example with code pointers to get a feel for how this is implemented.\nNotice how we don't have to deal with any version numbers when we're using the MVCC API -- this is\nan internal MVCC implementation detail.\n\n```rust\n// Open a BitCask database in the file \"toy.db\" with MVCC support.\nlet path = PathBuf::from(\"toy.db\");\nlet db = MVCC::new(BitCask::new(path)?);\n\n// Begin a new transaction.\nlet txn = db.begin()?;\n\n// Read the key \"foo\", and decode the binary value as a u64 with bincode.\nlet bytes = txn.get(b\"foo\")?.expect(\"foo not found\");\nlet mut value: u64 = bincode::deserialize(&bytes)?;\n\n// Delete \"foo\".\ntxn.delete(b\"foo\")?;\n\n// Add 1 to the value, and write it back to the key \"bar\".\nvalue += 1;\nlet bytes = bincode::serialize(&value);\ntxn.set(b\"bar\", bytes)?;\n\n// Commit the transaction.\ntxn.commit()?;\n```\n\nFirst, we begin a new transaction with `MVCC::begin()`, which calls through to\n`Transaction::begin()`. This obtains a version number stored in `Key::NextVersion` and increments\nit, then takes a snapshot of the active set in `Key::ActiveSet` and adds itself to it:\n\nhttps://github.com/erikgrinaker/toydb/blob/8f8eae0dcf70b1a0df2e853b1f6600e0c7075340/src/storage/mvcc.rs#L368-L391\n\nThis returns a `Transaction` object which provides the main key/value API, with get/set/delete\nmethods. It keeps track of the main state of the transaction: it's version number and active set.\n\nhttps://github.com/erikgrinaker/toydb/blob/8f8eae0dcf70b1a0df2e853b1f6600e0c7075340/src/storage/mvcc.rs#L294-L327\n\nNext, we call `Transaction::get(b\"foo\")` to read the value of the key `foo`. This finds the latest\nversion that's visible to us (ignoring future versions and the active set). Recall that we store\nmultiple version of each key as `Key::Version(key, version)`. The Keycode encoding ensures that all\nversions are stored in sorted order, so we can do a reverse range scan from `Key::Version(b\"foo\",\nself.version)` to `Key::Version(b\"foo\", 0)` and return the latest version that's visible to us:\n\nhttps://github.com/erikgrinaker/toydb/blob/8f8eae0dcf70b1a0df2e853b1f6600e0c7075340/src/storage/mvcc.rs#L564-L581\n\nWe then call `Transaction::delete(b\"foo\")` and `Transaction::set(b\"bar\", value)`. Both of these just\ncall through to the same `Transaction::write_version()` method, but use `Some(value)` for a regular\nkey/value pair and `None` as a deletion tombstone:\n\nhttps://github.com/erikgrinaker/toydb/blob/8f8eae0dcf70b1a0df2e853b1f6600e0c7075340/src/storage/mvcc.rs#L514-L522\n\nTo write a new version of a key, we first have to check for conflicts by seeing if there's a\nversion of the key that's invisible to us -- if it is, we conflicted with a concurrent transaction.\nWe use a range scan for this, like we did in `Transaction::get()`.\n\nIf there are no conflicts, we go on to write `Key::Version(b\"foo\", self.version)` and encode the\nvalue as an `Option` to accomodate the `None` tombstone marker. We also write a\n`Key::TxnWrite(version, key)` to keep track of the keys we've written in case we have to roll back.\n\nhttps://github.com/erikgrinaker/toydb/blob/8f8eae0dcf70b1a0df2e853b1f6600e0c7075340/src/storage/mvcc.rs#L524-L562\n\nFinally, `Transaction::commit()` will make our transaction take effect and become visible. It does\nthis simply by removing itself from the active set in `Key::ActiveSet`, and also cleaning up its\n`Key::TxnWrite` write tracking. As the comment says, we don't actually have to flush to durable\nstorage here, because the Raft log will provide durability for us -- we'll get back to this later.\n\nhttps://github.com/erikgrinaker/toydb/blob/8f8eae0dcf70b1a0df2e853b1f6600e0c7075340/src/storage/mvcc.rs#L466-L485\n\n---\n\n

\n← Key/Value Encoding   |   Raft Consensus →\n

" }, { "path": "docs/architecture/overview.md", "content": "# Overview\n\ntoyDB consists of a cluster of nodes that execute [SQL](https://en.wikipedia.org/wiki/SQL)\ntransactions against a replicated state machine. Clients can connect to any node in the cluster and\nsubmit SQL statements. The cluster remains available if a minority of nodes crash or disconnect,\nbut halts if a majority of nodes fail.\n\n## Properties\n\n* **Distributed:** runs across a cluster of nodes.\n* **Highly available:** tolerates failure of a minority of nodes.\n* **SQL compliant:** correctly supports most common [SQL](https://en.wikipedia.org/wiki/SQL)\n features.\n* **Strongly consistent:** committed writes are immediately visible to all readers ([linearizability](https://en.wikipedia.org/wiki/Linearizability)).\n* **Transactional:** provides [ACID](https://en.wikipedia.org/wiki/ACID) transactions\n * **Atomic:** groups of writes are applied as a single, atomic unit.\n * **Consistent:** database constraints and referential integrity are always enforced.\n * **Isolated:** concurrent transactions don't affect each other ([snapshot isolation](https://en.wikipedia.org/wiki/Snapshot_isolation)).\n * **Durable:** committed writes are never lost.\n\nFor simplicity, toyDB is:\n\n* **Not scalable:** every node stores the full dataset, and reads/writes execute on one node.\n* **Not reliable:** only handles crash failures, not e.g. partial network partitions or node stalls.\n* **Not performant:** data processing is slow, and not optimized at all.\n* **Not efficient:** loads entire tables into memory, no compression or garbage collection, etc.\n* **Not full-featured:** only basic SQL functionality is implemented.\n* **Not backwards compatible:** changes to data formats and protocols will break databases.\n* **Not flexible:** nodes can't be added or removed while running, and take a long time to join.\n* **Not secure:** there is no authentication, authorization, nor encryption.\n\n## Components\n\nInternally, toyDB is made up of a few main components:\n\n* **Storage engine:** stores data on disk and manages transactions.\n* **Raft consensus engine:** replicates data and coordinates cluster nodes.\n* **SQL engine:** organizes SQL data, manages SQL sessions, and executes SQL statements.\n* **Server:** manages network communication, both with SQL clients and Raft nodes.\n* **Client:** provides a SQL user interface and communicates with the server.\n\nThis diagram illustrates the internal structure of a single toyDB node:\n\n![toyDB architecture](./images/architecture.svg)\n\nWe will go through each of these components from the bottom up.\n\n---\n\n

\n← toyDB Architecture   |   Storage Engine →\n

" }, { "path": "docs/architecture/raft.md", "content": "# Raft Consensus\n\n[Raft](https://raft.github.io) is a distributed consensus protocol which replicates data across a\ncluster of nodes in a consistent and durable manner. It is described in the very readable\n[Raft paper](https://raft.github.io/raft.pdf), and in the more comprehensive\n[Raft thesis](https://web.stanford.edu/~ouster/cgi-bin/papers/OngaroPhD.pdf).\n\nThe toyDB Raft implementation is in the [`raft`](https://github.com/erikgrinaker/toydb/tree/213e5c02b09f1a3cac6a8bbd0a81773462f367f5/src/raft)\nmodule, and is described in the module documentation:\n\nhttps://github.com/erikgrinaker/toydb/blob/d96c6dd5ae7c0af55ee609760dcd958c289a44f2/src/raft/mod.rs#L1-L240\n\nRaft is fundamentally the same protocol as [Paxos](https://lamport.azurewebsites.net/pubs/paxos-simple.pdf)\nand [Viewstamped Replication](https://pmg.csail.mit.edu/papers/vr-revisited.pdf), but an\nopinionated variant designed to be simple, understandable, and practical. It is widely used in the\nindustry: [CockroachDB](https://www.cockroachlabs.com), [TiDB](https://www.pingcap.com),\n[etcd](https://etcd.io), [Consul](https://developer.hashicorp.com/consul), and many others.\n\nBriefly, Raft elects a leader node which coordinates writes and replicates them to followers. Once a\nmajority (>50%) of nodes have acknowledged a write, it is considered durably committed. It is common\nfor the leader to also serve reads, since it always has the most recent data and is thus strongly\nconsistent.\n\nA cluster must have a majority of nodes (known as a [quorum](https://en.wikipedia.org/wiki/Quorum_(distributed_computing)))\nlive and connected to remain available, otherwise it will not commit writes in order to guarantee\ndata consistency and durability. Since there can only be one majority in the cluster, this prevents\na [split brain](https://en.wikipedia.org/wiki/Split-brain_(computing)) scenario where two active\nleaders can exist concurrently (e.g. during a [network partition](https://en.wikipedia.org/wiki/Network_partition))\nand store conflicting values.\n\nThe Raft leader appends writes to an ordered command log, which is then replicated to followers.\nOnce a majority has replicated the log up to a given entry, that log prefix is committed and then\napplied to a state machine. This ensures that all nodes will apply the same commands in the same\norder and eventually reach the same state (assuming the commands are deterministic). Raft itself\ndoesn't care what the state machine and commands are, but in toyDB's case it's SQL tables and rows\nstored in an MVCC key/value store.\n\nThis diagram from the Raft paper illustrates how a Raft node receives a command from a client (1),\nadds it to its log and reaches consensus with other nodes (2), then applies it to its state machine\n(3) before returning a result to the client (4):\n\n\"Raft\n\nYou may notice that Raft is not very scalable, since all reads and writes go via the leader node,\nand every node must store the entire dataset. Raft solves replication and availability, but not\nscalability. Real-world systems typically provide horizontal scalability by splitting a large\ndataset across many separate Raft clusters (i.e. sharding), but this is out of scope for toyDB.\n\nFor simplicitly, toyDB implements the bare minimum of Raft, and omits optimizations described in\nthe paper such as state snapshots, log truncation, leader leases, and more. The implementation is\nin the [`raft`](https://github.com/erikgrinaker/toydb/blob/d96c6dd5ae7c0af55ee609760dcd958c289a44f2/src/raft/mod.rs)\nmodule, and we'll walk through the main components next.\n\nThere is a comprehensive set of Raft test scripts in [`src/raft/testscripts/node`](https://github.com/erikgrinaker/toydb/blob/386153f5c00cb1a88b1ac8489ae132674d96f68a/src/raft/testscripts/node),\nwhich illustrate the protocol in a wide variety of scenarios.\n\n## Log Storage\n\nRaft replicates an ordered command log consisting of `raft::Entry`:\n\nhttps://github.com/erikgrinaker/toydb/blob/90a6cae47ac20481ac4eb2f20eea50f02e6c2b33/src/raft/log.rs#L10-L28\n\n`index` specifies the position in the log, and `command` contains the binary command to apply to the\nstate machine. The `term` identifies the leadership term in which the command was proposed: a new\nterm begins when a new leader election is held (we'll get back to this later).\n\nEntries are appended to the log by the leader and replicated to followers. Once acknowledged by a\nquorum, the log up to that index is committed and will never change. Entries that are not yet\ncommitted may be replaced or removed if the leader changes.\n\nThe Raft log enforces the following invariants:\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/log.rs#L80-L91\n\n`raft::Log` implements a Raft log, and stores log entries in a `storage::Engine` key/value store:\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/log.rs#L43-L116\n\nIt also stores some additional metadata that we'll need later: the current term, vote, and commit\nindex. These are stored as separate keys:\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/log.rs#L30-L39\n\nIndividual entries are appended to the log via `Log::append`, typically when the leader wants to\nreplicate a new write:\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/log.rs#L190-L203\n\nEntries can also be appended in bulk via `Log::splice`, typically when entries are replicated to\nfollowers. This also allows replacing existing uncommitted entries, e.g. after a leader change:\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/log.rs#L269-L343\n\nCommitted entries are marked by `Log::commit`, making them immutable and eligible for state machine\napplication:\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/log.rs#L205-L222\n\nThe log also has methods to read entries from the log, either individually as `Log::get` or by\niterating over a range with `Log::scan`:\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/log.rs#L224-L267\n\n## State Machine Interface\n\nRaft doesn't know or care what the log commands are, nor what the state machine does with them. It\nsimply takes `raft::Entry` from the log and gives them to the state machine.\n\nThe Raft state machine is represented by the `raft::State` trait. Raft will ask about the last\napplied entry via `State::get_applied_index`, and feed it newly committed entries via\n`State::apply`. It also allows reads via `State::read`, but we'll get back to that later.\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/state.rs#L4-L51\n\nThe state machine does not have to flush its state to durable storage after each transition; on node\ncrashes, the state machine is allowed to regress, and will be caught up by replaying the unapplied\nlog entries. It is also possible to implement a purely in-memory state machine (and in fact, toyDB\nallows running the state machine with a `Memory` storage engine).\n\nThe state machine must take care to be deterministic: the same commands applied in the same order\nmust result in the same state across all nodes. This means that a command can't e.g. read the\ncurrent time or generate a random number -- these values must be included in the command. It also\nmeans that non-deterministic errors, such as an IO error, must halt command application (in toyDB's\ncase, we just panic and crash the node).\n\nIn toyDB's, the state machine is an MVCC key/value store that stores SQL tables and rows, as we'll\nsee in the SQL Raft replication section.\n\n## Node Roles\n\nIn Raft, a node can have one out of three roles:\n\n* **Leader:** replicates writes to followers and serves client requests.\n* **Follower:** replicates writes from a leader.\n* **Candidate:** campaigns for leadership.\n\nThe Raft paper summarizes these roles and transitions in the following diagram (we'll discuss\nleader election in detail below):\n\n\"Raft\n\nIn toyDB, a node is represented by the `raft::Node` enum, with variants for each state:\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/node.rs#L47-L66\n\nThis wraps the `raft::RawNode` type which contains the inner node state. It is generic over\nthe role, and uses the [typestate pattern](http://cliffle.com/blog/rust-typestate/) to provide\nmethods and transitions depending on the node's current role. This enforces state transitions and\ninvariants at compile time via Rust's type system -- for example, only `RawNode` has an\n`into_leader()` method, since only candidates can transition to leaders (when they win an election).\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/node.rs#L156-L177\n\nThe `RawNode::role` field contains role-specific state as structs implementing the `Role` marker\ntrait:\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/node.rs#L661-L680\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/node.rs#L242-L255\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/node.rs#L523-L531\n\nWe'll see what the various fields are used for in the following sections.\n\n## Node Interface and Communication\n\nThe `raft::Node` enum has two main methods that drive the node: `tick()` and `step()`. These consume\nthe current node and return a new node, possibly with a different role.\n\n`tick()` advances time by a logical tick. This is used to measure the passage of time, e.g. to\ntrigger election timeouts or periodic leader heartbeats. toyDB uses a tick interval of 100\nmilliseconds (see `raft::TICK_INTERVAL`), and will call `tick()` on the node at this rate.\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/node.rs#L125-L132\n\n`step()` processes an inbound message from a different node or client:\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/node.rs#L107-L123\n\nOutbound messages to other nodes are sent via the `RawNode::tx` channel:\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/node.rs#L171-L172\n\nNodes are identified by a unique node ID, which is given at node startup:\n\nhttps://github.com/erikgrinaker/toydb/blob/90a6cae47ac20481ac4eb2f20eea50f02e6c2b33/src/raft/node.rs#L17-L18\n\nMessages are wrapped in a `raft::Envelope` specifying the sender and recipient:\n\nhttps://github.com/erikgrinaker/toydb/blob/d96c6dd5ae7c0af55ee609760dcd958c289a44f2/src/raft/message.rs#L10-L21\n\nThe envelope contains a `raft::Message`, an enum which encodes the Raft message protocol. We won't\ndwell on the specific message types here, but discuss them invididually in the following sections.\nRaft does not require reliable message delivery, so messages may be dropped or reordered at any\ntime, although toyDB's use of TCP provides stronger delivery guarantees.\n\nhttps://github.com/erikgrinaker/toydb/blob/d96c6dd5ae7c0af55ee609760dcd958c289a44f2/src/raft/message.rs#L25-L152\n\nThis is an entirely synchronous and deterministic model -- the same sequence of calls on a given\nnode in a given initial state will always produce the same result. This is very convenient for\ntesting and understandability. We will see in the server section how toyDB drives the node on a\nseparate thread, provides a network transport for messages, and ticks it at regular intervals.\n\n## Leader Election and Terms\n\nIn the steady state, Raft simply has a leader which replicates writes to followers. But to reach\nthis steady state, we must elect a leader, which is where much of the subtle complexity lies. See\nthe Raft paper for comprehensive details and safety arguments, we'll summarize it briefly below.\n\nRaft divides time into _terms_. The term is a monotonically increasing number starting at 1. There\ncan only be one leader in a term (or none if an election fails), and the term can never regress.\nReplicated commands belong to the specific term under which they were proposed.\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/node.rs#L20-L21\n\nLet's walk through an election, where we bootstrap a brand new, empty toyDB cluster with 3 nodes.\n\nNodes are initialized by calling `Node::new()`. Since this is a new cluster, they are given an empty\n`raft::Log` and `raft::State`, at term 0. Nodes start with role `Follower`, but without a leader.\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/node.rs#L68-L87\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/node.rs#L266-L290\n\nNow, nothing really happens for a while, as the nodes are waiting to maybe hear from an existing\nleader (there is none). Every 100 ms we call `tick()`, until we reach `election_timeout`:\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/node.rs#L489-L497\n\nNotice how `new()` set `election_timeout` to a random value (in the range `ELECTION_TIMEOUT_RANGE`\nof 10-20 ticks, i.e. 1-2 seconds). If all nodes had the same timeout, they would likely campaign for\nleadership simultaneously, resulting in an election tie -- Raft uses randomized election timeouts to\navoid such ties.\n\nOnce a node reaches `election_timeout` it transitions to role `Candidate`:\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/node.rs#L292-L312\n\nWhen it becomes a candidate it campaigns for leadership by increasing its term to 1, voting for\nitself, and sending `Message::Campaign` to all peers asking for their vote:\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/node.rs#L647-L658\n\nIn Raft, the term can't regress, and a node can only cast a single vote in each term (even across\nrestarts), so both of these are persisted to disk via `Log::set_term_vote()`.\n\nWhen the two other nodes (still in state `Follower`) receive the `Message::Campaign` asking for a\nvote, they will first increase their term to 1 (since this is a newer term than their local term 0):\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/node.rs#L347-L351\n\nThey then grant the vote since they haven't yet voted for anyone else in term 1. They persist the\nvote to disk via `Log::set_term_vote()` and return a `Message::CampaignResponse { vote: true }` to\nthe candidate:\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/node.rs#L424-L449\n\nThey also check that the candidate's log is at least as long as theirs, which is trivially true in\nthis case since the log is empty. This is necessary to ensure that a leader has all committed\nentries (see section 5.4.1 in the Raft paper).\n\nWhen the candidate receives the `Message::CampaignResponse` it records the vote from each node. Once\nit has a quorum (in this case 2 out of 3 votes including its own vote) it becomes leader in term 1:\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/node.rs#L599-L606\n\nWhen it becomes leader, it sends a `Message::Heartbeat` to all peers to tell them it is now the\nleader in term 1. It also appends an empty entry to its log and replicates it, but we will ignore\nthis for now (see section 5.4.2 in the Raft paper for why).\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/node.rs#L563-L583\n\nWhen the other nodes receive the heartbeat, they become followers of the new leader in its term:\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/node.rs#L359-L384\n\nFrom now on, the leader will send periodic `Message::Heartbeat` every 4 ticks (see\n`HEARTBEAT_INTERVAL`) to assert its leadership:\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/node.rs#L945-L953\n\nThe followers record when they last received any message from the leader (including heartbeats), and\nwill hold a new election if they haven't heard from the leader in an election timeout (e.g. due to a\nleader crash or network partition):\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/node.rs#L353-L356\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/node.rs#L489-L497\n\nThis entire process is illustrated in the test script [`election`](https://github.com/erikgrinaker/toydb/blob/cb234a0b776484608118fd9382869ee5bc30d4f0/src/raft/testscripts/node/election),\nalong with several other test scripts that show e.g. [election ties](https://github.com/erikgrinaker/toydb/blob/cb234a0b776484608118fd9382869ee5bc30d4f0/src/raft/testscripts/node/election_tie),\n[contested elections](https://github.com/erikgrinaker/toydb/blob/cb234a0b776484608118fd9382869ee5bc30d4f0/src/raft/testscripts/node/election_contested),\nand other scenarios:\n\nhttps://github.com/erikgrinaker/toydb/blob/cb234a0b776484608118fd9382869ee5bc30d4f0/src/raft/testscripts/node/election#L1-L72\n\n## Client Requests and Forwarding\n\nOnce a leader has been elected, we can submit read and write requests to it. This is done by\nstepping a `Message::ClientRequest` into the node using the local node ID, with a unique request ID\n(toyDB uses UUIDv4), and waiting for an outbound response message with the same ID:\n\nhttps://github.com/erikgrinaker/toydb/blob/d96c6dd5ae7c0af55ee609760dcd958c289a44f2/src/raft/message.rs#L134-L151\n\nhttps://github.com/erikgrinaker/toydb/blob/d96c6dd5ae7c0af55ee609760dcd958c289a44f2/src/raft/message.rs#L164-L188\n\nThe requests and responses themselves are arbitrary binary data which is interpreted by the state\nmachine. For our purposes here, let's pretend the requests are:\n\n* `Request::Write(\"key=value\")` → `Response::Write(\"ok\")`\n* `Request::Read(\"key\")` → `Response::Read(\"value\")`\n\nThe fundamental difference between read and write requests are that write requests are replicated\nthrough Raft and executed on all nodes, while read requests are only executed on the leader without\nbeing appended to the log. It would be possible to execute reads on followers too, for load\nbalancing, but these reads would be eventually consistent and thus violate linearizability, so toyDB\nonly executes reads on the leader.\n\nIf a request is submitted to a follower, it will be forwarded to the leader and the response\nforwarded back to the client (distinguished by the sender/recipient node ID -- a local client always\nuses the local node ID):\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/node.rs#L451-L474\n\nFor simplicity, we cancel the request with `Error::Abort` if a request is submitted to a candidate,\nand similarly if a follower changes its role to candidate or discovers a new leader. We could have\nheld on to these and redirected them to a new leader, but we keep it simple and ask the client to\nretry.\n\nWe'll look at the actual read and write request processing next.\n\n## Write Replication and Application\n\nWhen the leader receives a write request, it proposes the command for replication to followers. It\nkeeps track of the in-flight write and its log entry index in `writes`, such that it can respond to\nthe client with the command result once the entry has been committed and applied.\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/node.rs#L895-L904\n\nTo propose the command, the leader appends it to its log and sends a `Message::Append` to each\nfollower to replicate it to their logs:\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/node.rs#L966-L980\n\nIn steady state, `Message::Append` just contains the single log entry we appended above:\n\nhttps://github.com/erikgrinaker/toydb/blob/d96c6dd5ae7c0af55ee609760dcd958c289a44f2/src/raft/message.rs#L87-L108\n\nHowever, sometimes followers may be lagging behind the leader (e.g. after a crash), or their log may\nhave diverged from the leader (e.g. unsuccessful proposals from a stale leader after a network\npartition). To handle these cases, the leader tracks the replication progress of each follower as\n`raft::Progress`:\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/node.rs#L682-L698\n\nWe'll gloss over these cases here (see the Raft paper and the code in `raft::Progress` and\n`maybe_send_append()` for details). In the steady state, where each entry is successfully appended\nand replicated one at a time, `maybe_send_append()` will fall through to the bottom and send a\nsingle entry:\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/node.rs#L1068-L1128\n\nThe `Message::Append` contains the index/term of the entry immediately before the new entry as\n`base_index` and `base_term`. If the follower's log also contains an entry with this index and term\nthen its log is guaranteed to match (be equal to) the leader's log up to this entry (see section 5.3\nin the Raft paper). The follower can then append the new log entry and return a\n`Message::AppendResponse` confirming that the entry was appended and that its log matches the\nleader's log up to `match_index`:\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/node.rs#L386-L410\n\nWhen the leader receives the `Message::AppendResponse`, it will update its view of the follower's\n`match_index`.\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/node.rs#L844-L858\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/node.rs#L701-L710\n\nOnce a quorum of nodes (in our case 2 out of 3 including the leader) have the entry in their log,\nthe leader can commit the entry and apply it to the state machine. It also looks up the in-flight\nwrite request from `writes` and sends the command result back to the client as\n`Message::ClientResponse`:\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/node.rs#L982-L1032\n\nThe leader will also propagate the new commit index to followers via the next heartbeat, so that\nthey can also apply any pending log entries to their state machine. This isn't strictly necessary,\nsince reads are executed on the leader and nodes have to apply pending entries before becoming\nleaders, but we do it anyway so that they don't fall too far behind on application.\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/node.rs#L359-L384\n\nThis process is illustrated in the test scripts [`append`](https://github.com/erikgrinaker/toydb/blob/cb234a0b776484608118fd9382869ee5bc30d4f0/src/raft/testscripts/node/append) and [`heartbeat_commits_follower`](https://github.com/erikgrinaker/toydb/blob/cb234a0b776484608118fd9382869ee5bc30d4f0/src/raft/testscripts/node/heartbeat_commits_follower)\n(along with many other scenarios):\n\nhttps://github.com/erikgrinaker/toydb/blob/cb234a0b776484608118fd9382869ee5bc30d4f0/src/raft/testscripts/node/append#L1-L43\n\nhttps://github.com/erikgrinaker/toydb/blob/cb234a0b776484608118fd9382869ee5bc30d4f0/src/raft/testscripts/node/heartbeat_commits_follower#L1-L50\n\n## Read Processing\n\nFor linearizable (aka strongly consistent) reads, we must execute read requests on the leader, as\nmentioned above. However, this is not sufficient: under e.g. a network partition, a node may think\nit's still the leader while in fact a different leader has been elected elsewhere (in a later term)\nand executed writes there.\n\nTo handle this case, the leader must confirm that it is still the leader for each read, by sending a\n`Message::Read` to its followers containing a read sequence number. Only if a quorum confirms that\nit is still the leader can the read be executed. This incurs an additional network roundtrip, which\nis clearly inefficient, so real-world systems often use leader leases instead (see section 6.4.1 of\nthe Raft _thesis_, not the paper) -- but it's fine for toyDB.\n\nhttps://github.com/erikgrinaker/toydb/blob/d96c6dd5ae7c0af55ee609760dcd958c289a44f2/src/raft/message.rs#L125-L132\n\nWhen the leader receives the read request, it increments the read sequence number, stores the\npending read request in `reads`, and sends a `Message::Read` to all followers:\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/node.rs#L906-L917\n\nWhen the followers receive the `Message::Read`, they simply respond with a `Message::ReadResponse`\nif it's from their current leader (messages from stale terms are ignored):\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/node.rs#L342-L346\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/node.rs#L412-L422\n\nWhen the leader receives the `Message::ReadResponse` it records it in the peer's `Progress`, and\nexecutes the read once a quorum have confirmed the sequence number:\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/node.rs#L860-L866\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/node.rs#L1034-L1066\n\nWe now have a Raft-managed state machine with replicated writes and linearizable reads.\n\n---\n\n

\n← MVCC Transactions   |   SQL Engine →\n

" }, { "path": "docs/architecture/server.md", "content": "# Server\n\nNow that we've gone over the individual components, we'll tie them all together in the toyDB\nserver `toydb::Server`, located in the [`server`](https://github.com/erikgrinaker/toydb/blob/0839215770e31f1e693d5cccf20a68210deaaa3f/src/server.rs) module.\n\nThe server wraps an inner Raft node `raft::Node`, which manages the SQL state machine, and is\nresponsible for routing network traffic between the Raft node, its Raft peers, and SQL clients.\n\nhttps://github.com/erikgrinaker/toydb/blob/0839215770e31f1e693d5cccf20a68210deaaa3f/src/server.rs#L27-L44\n\nFor network protocol, the server uses the Bincode encoding that we've discussed in the encoding\nsection, sent over a TCP connection. There's no need for any further framing, since Bincode knows\nhow many bytes to expect for each message depending on the type it's decoding into.\n\nThe server does not use [async Rust](https://rust-lang.github.io/async-book/) and e.g.\n[Tokio](https://tokio.rs), instead opting for regular OS threads. Async Rust can significantly\ncomplicate the code, which would obscure the main concepts, and any efficiency gains would be\nentirely irrelevant for toyDB.\n\nInternally in the server, messages are passed around between threads using\n[Crossbeam channels](https://docs.rs/crossbeam/latest/crossbeam/channel/index.html).\n\nThe main server loop `Server::serve()` listens for inbound TCP connections on port 9705 for Raft\npeers and 9605 for SQL clients, and spawns threads to process them. We'll look at Raft and SQL\nservices separately.\n\nhttps://github.com/erikgrinaker/toydb/blob/0839215770e31f1e693d5cccf20a68210deaaa3f/src/server.rs#L66-L110\n\n## Raft Routing\n\nThe heart of the server is the Raft processing thread `Server::raft_route()`. This is responsible\nfor periodically ticking the Raft node via `raft::Node::tick()`, stepping inbound messages from\nRaft peers into the node via `raft::Node::step()`, and sending outbound messages to peers.\n\nIt also takes inbound Raft client requests from the `sql::engine::Raft` SQL engine, steps them\ninto the Raft node via `raft::Node::step()`, and passes responses back to the appropriate client\nas the node emits them.\n\nhttps://github.com/erikgrinaker/toydb/blob/0839215770e31f1e693d5cccf20a68210deaaa3f/src/server.rs#L169-L249\n\nWhen the node starts up, it spawns a `Server::raft_send_peer()` thread for each Raft peer to send\noutbound messages to them.\n\nhttps://github.com/erikgrinaker/toydb/blob/0839215770e31f1e693d5cccf20a68210deaaa3f/src/server.rs#L84-L91\n\nThese threads continually attempt to connect to the peer via TCP, and then read any outbound\n`raft::Envelope(raft::Message)` messages from `Server::raft_route()` via a channel and writes the\nmessages into the TCP connection using Bincode:\n\nhttps://github.com/erikgrinaker/toydb/blob/0839215770e31f1e693d5cccf20a68210deaaa3f/src/server.rs#L146-L167\n\nThe server also continually listens for inbound Raft TCP connections from peers in\n`Server::raft_accept()`:\n\nhttps://github.com/erikgrinaker/toydb/blob/0839215770e31f1e693d5cccf20a68210deaaa3f/src/server.rs#L112-L134\n\nWhen an inbound connection is accepted, a `Server::raft_receive_peer()` thread is spawned that reads\nBincode-encoded `raft::Envelope(raft::Message)` messages from the TCP connection and sends them to\n`Server::raft_route()` via a channel.\n\nhttps://github.com/erikgrinaker/toydb/blob/0839215770e31f1e693d5cccf20a68210deaaa3f/src/server.rs#L136-L144\n\nThe Raft cluster is now fully connected, and the nodes can all talk to each other.\n\n## SQL Service\n\nNext, let's serve some SQL clients. The SQL service uses the enums `toydb::Request` and\n`toydb::Response` as a client protocol, again Bincode-encoded over TCP.\n\nThe primary request type is `Request::Execute` which executes a SQL statement against a\n`sql::execution::Session` and returns a `sql::execution::StatementResult`, as we've seen previously.\n\nhttps://github.com/erikgrinaker/toydb/blob/0839215770e31f1e693d5cccf20a68210deaaa3f/src/server.rs#L312-L337\n\nThe server sets up a `sql::engine::Raft` SQL engine, with a Crossbeam channel that's used to send\n`raft::Request` Raft client requests to `Server::raft_route()` and onwards to the local\n`raft::Node`. It then spawns a `Server::sql_accept()` thread to listen for inbound SQL client\nconnections:\n\nhttps://github.com/erikgrinaker/toydb/blob/0839215770e31f1e693d5cccf20a68210deaaa3f/src/server.rs#L104-L106\n\nWhen a SQL client connection is accepted, a new client session `sql::execution::Session` is set up\nfor the client, and we spawn a `Server::sql_session()` thread to serve the connection:\n\nhttps://github.com/erikgrinaker/toydb/blob/0839215770e31f1e693d5cccf20a68210deaaa3f/src/server.rs#L251-L272\n\nThese session threads continually read `Request` messages from the client, execute them against the\nSQL session (and ultimately the Raft node), before sending a `Response` back to the client.\n\nhttps://github.com/erikgrinaker/toydb/blob/0839215770e31f1e693d5cccf20a68210deaaa3f/src/server.rs#L274-L309\n\n## `toydb` Binary\n\nThe `toydb` binary in `src/bin/toydb.rs` launches the server, and is a thin wrapper around\n`toydb::Server`. It is a tiny [`clap`](https://docs.rs/clap/latest/clap/) command:\n\nhttps://github.com/erikgrinaker/toydb/blob/8f8eae0dcf70b1a0df2e853b1f6600e0c7075340/src/bin/toydb.rs#L82-L89\n\nIt first parses a server configuration from the `toydb.yaml` file:\n\nhttps://github.com/erikgrinaker/toydb/blob/8f8eae0dcf70b1a0df2e853b1f6600e0c7075340/src/bin/toydb.rs#L30-L59\n\nThen it initializes the Raft log storage and SQL state machine:\n\nhttps://github.com/erikgrinaker/toydb/blob/8f8eae0dcf70b1a0df2e853b1f6600e0c7075340/src/bin/toydb.rs#L105-L133\n\nAnd finally it launches the `toydb::Server`:\n\nhttps://github.com/erikgrinaker/toydb/blob/8f8eae0dcf70b1a0df2e853b1f6600e0c7075340/src/bin/toydb.rs#L135-L137\n\ntoyDB is now up and running!\n\n---\n\n

\n← SQL Execution   |   Client →\n

" }, { "path": "docs/architecture/sql-data.md", "content": "# SQL Data Model\n\nThe SQL data model represents user data in tables and rows. It is made up of data types and schemas,\nin the [`sql::types`](https://github.com/erikgrinaker/toydb/tree/686d3971a253bfc9facc2ba1b0e716cff5c109fb/src/sql/types)\nmodule.\n\n## Data Types\n\ntoyDB supports four basic scalar data types as `sql::types::DataType`: booleans, integers, floats,\nand strings.\n\nhttps://github.com/erikgrinaker/toydb/blob/b2fe7b76ee634ca6ad31616becabfddb1c03d34b/src/sql/types/value.rs#L15-L27\n\nSpecific values are represented as `sql::types::Value`, using the corresponding Rust types. toyDB\nalso supports SQL `NULL` values, i.e. unknown values, following the rules of\n[three-valued logic](https://en.wikipedia.org/wiki/Three-valued_logic).\n\nhttps://github.com/erikgrinaker/toydb/blob/b2fe7b76ee634ca6ad31616becabfddb1c03d34b/src/sql/types/value.rs#L40-L64\n\nThe `Value` type provides basic formatting, conversion, and mathematical operations.\n\nhttps://github.com/erikgrinaker/toydb/blob/686d3971a253bfc9facc2ba1b0e716cff5c109fb/src/sql/types/value.rs#L68-L79\n\nhttps://github.com/erikgrinaker/toydb/blob/686d3971a253bfc9facc2ba1b0e716cff5c109fb/src/sql/types/value.rs#L164-L370\n\nIt also specifies comparison and ordering semantics, but these are subtly different from the SQL\nsemantics. For example, in Rust code `Value::Null == Value::Null` yields `true`, while in SQL\n`NULL = NULL` yields `NULL`. This mismatch is necessary for the Rust code to properly detect and\nprocess `Null` values, and the desired SQL semantics are implemented during expression evaluation\nwhich we'll cover below.\n\nhttps://github.com/erikgrinaker/toydb/blob/b2fe7b76ee634ca6ad31616becabfddb1c03d34b/src/sql/types/value.rs#L91-L162\n\nDuring execution, a row of values is represented as `sql::types::Row`, with multiple rows emitted\nvia `sql::types::Rows` row iterators:\n\nhttps://github.com/erikgrinaker/toydb/blob/b2fe7b76ee634ca6ad31616becabfddb1c03d34b/src/sql/types/value.rs#L378-L388\n\n## Schemas\n\ntoyDB schemas only support tables. There are no named indexes or constraints, and there's only a\nsingle unnamed database.\n\nTables are represented by `sql::types::Table`:\n\nhttps://github.com/erikgrinaker/toydb/blob/c2b0f7f1d6cbf6e2cdc09fc0aec7b050e840ec21/src/sql/types/schema.rs#L12-L25\n\nA table is made up of a set of columns, represented by `sql::types::Column`. These support the data\ntypes described above, along with unique constraints, foreign keys, and secondary indexes.\n\nhttps://github.com/erikgrinaker/toydb/blob/c2b0f7f1d6cbf6e2cdc09fc0aec7b050e840ec21/src/sql/types/schema.rs#L29-L53\n\nThe table name serves as a unique identifier, and can't be changed later. In fact, tables schemas\nare entirely static: they can only be created or dropped (there are no schema changes).\n\nTable schemas are stored in the catalog, represented by the `sql::engine::Catalog` trait. We'll\nrevisit the implementation of this trait in the SQL storage section.\n\nhttps://github.com/erikgrinaker/toydb/blob/0839215770e31f1e693d5cccf20a68210deaaa3f/src/sql/engine/engine.rs#L60-L79\n\nTable schemas are validated when created via `Table::validate()`, which enforces invariants and\ninternal consistency. It uses the catalog to look up information about other tables, e.g. that\nforeign key references point to a valid target column in a different table.\n\nhttps://github.com/erikgrinaker/toydb/blob/c2b0f7f1d6cbf6e2cdc09fc0aec7b050e840ec21/src/sql/types/schema.rs#L98-L170\n\nTable rows are validated via `Table::validate_row()`, which ensures that a `sql::types::Row`\nconforms to the schema (e.g. that value types match the column data types). It uses a\n`sql::engine::Transaction` to look up other rows in the database, e.g. to check for primary key\nconflicts (we'll get back to this later).\n\nhttps://github.com/erikgrinaker/toydb/blob/c2b0f7f1d6cbf6e2cdc09fc0aec7b050e840ec21/src/sql/types/schema.rs#L172-L236\n\n## Expressions\n\nDuring SQL execution, we also have to model _expressions_, such as `1 + 2 * 3`. These are\nrepresented as values and operations on them, and can be nested as a tree to represent compound\noperations.\n\nhttps://github.com/erikgrinaker/toydb/blob/9419bcf6aededf0e20b4e7485e2a5fa3e975d79f/src/sql/types/expression.rs#L11-L64\n\n\nFor example, the expression `1 + 2 * 3` (taking [precedence](https://en.wikipedia.org/wiki/Order_of_operations)\ninto account) is represented as:\n\n```rust\n// +\n// / \\\n// 1 *\n// / \\\n// 2 3\nExpression::Add(\n Expression::Constant(Value::Integer(1)),\n Expression::Multiply(\n Expression::Constant(Value::Integer(2)),\n Expression::Constant(Value::Integer(3)),\n ),\n)\n```\n\nAn `Expression` can contain two kinds of values: constant values as\n`Expression::Constant(sql::types::Value)`, and dynamic values as `Expression::Column(usize)` column\nreferences. The latter will fetch a `sql::types::Value` from a `sql::types::Row` at the specified\nindex during evaluation.\n\nWe'll see later how the SQL parser and planner transforms text expression like `1 + 2 * 3` into an\n`Expression`, and how it resolves column names to row indexes like `price * 0.25` to\n`row[3] * 0.25`.\n\nExpressions are evaluated recursively via `Expression::evalute()`, given a `sql::types::Row` with\ninput values for column references, and return a final `sql::types::Value` result:\n\nhttps://github.com/erikgrinaker/toydb/blob/9419bcf6aededf0e20b4e7485e2a5fa3e975d79f/src/sql/types/expression.rs#L73-L208\n\nMany of the comparison operations like `==` are implemented explicitly here instead of using\n`sql::types::Value` comparisons. This is where we implement the SQL semantics of special values like\n`NULL`, such that `NULL = NULL` yields `NULL` instead of `TRUE`.\n\nFor mathematical operations however, we generally dispatch to these methods on `sql::types::Value`:\n\nhttps://github.com/erikgrinaker/toydb/blob/b2fe7b76ee634ca6ad31616becabfddb1c03d34b/src/sql/types/value.rs#L185-L295\n\nExpression parsing and evaluation is tested via test scripts in\n[`sql/testscripts/expression`](https://github.com/erikgrinaker/toydb/tree/9419bcf6aededf0e20b4e7485e2a5fa3e975d79f/src/sql/testscripts/expressions).\n\n---\n\n

\n← SQL Engine   |   SQL Storage →\n

" }, { "path": "docs/architecture/sql-execution.md", "content": "# SQL Execution\n\nNow that the planner and optimizer have done all the hard work of figuring out how to execute a\nquery, it's time to actually execute it.\n\n## Plan Executor\n\nPlan execution is done by `sql::execution::Executor` in the\n[`sql::execution`](https://github.com/erikgrinaker/toydb/tree/9419bcf6aededf0e20b4e7485e2a5fa3e975d79f/src/sql/execution)\nmodule, using a `sql::engine::Transaction` to access the SQL storage engine.\n\nhttps://github.com/erikgrinaker/toydb/blob/213e5c02b09f1a3cac6a8bbd0a81773462f367f5/src/sql/execution/executor.rs#L14-L49\n\nThe executor takes a `sql::planner::Plan` as input, and will return an `ExecutionResult` depending\non the statement type.\n\nhttps://github.com/erikgrinaker/toydb/blob/686d3971a253bfc9facc2ba1b0e716cff5c109fb/src/sql/execution/executor.rs#L331-L339\n\nWhen executing the plan, the executor will branch off depending on the statement type:\n\nhttps://github.com/erikgrinaker/toydb/blob/686d3971a253bfc9facc2ba1b0e716cff5c109fb/src/sql/execution/executor.rs#L57-L101\n\nWe'll focus on `SELECT` queries here, which are the most interesting.\n\ntoyDB uses the iterator model (also known as the volcano model) for query execution. In the case of\na `SELECT` query, the result is a row iterator, and pulling from this iterator by calling `next()`\nwill drive the entire execution pipeline by recursively calling `next()` on the child nodes' row\niterators. This maps very naturally onto Rust's iterators, and we leverage these to construct the\nexecution pipeline as nested iterators.\n\nExecution itself is fairly straightforward, since we're just doing exactly what the planner tells us\nto do in the plan. We call `Executor::execute_node` recursively on each `sql::planner:Node`,\nstarting with the root node. Each node returns a result row iterator that the parent node can pull\nits input rows from, process them, and output the resulting rows via its own row iterator (with the\nroot node's iterator being returned to the caller):\n\nhttps://github.com/erikgrinaker/toydb/blob/686d3971a253bfc9facc2ba1b0e716cff5c109fb/src/sql/execution/executor.rs#L103-L104\n\n`Executor::execute_node()` will simply look at the type of `Node`, recursively call\n`Executor::execute_node()` on any child nodes, and then process the rows accordingly.\n\nhttps://github.com/erikgrinaker/toydb/blob/686d3971a253bfc9facc2ba1b0e716cff5c109fb/src/sql/execution/executor.rs#L103-L212\n\nWe won't discuss every plan node in detail, but let's consider the movie plan we've looked at\npreviously:\n\n```\nSelect\n└─ Order: movies.released desc\n └─ Projection: movies.title, movies.released, genres.name as genre\n └─ HashJoin: inner on movies.genre_id = genres.id\n ├─ Scan: movies (released >= 2000)\n └─ Scan: genres\n```\n\nWe'll recursively call `execute_node()` until we end up in the two `Scan` nodes. These simply\ncall through to the SQL engine (either using Raft or local disk) via `Transaction::scan()`, passing\nin the scan predicate if any, and return the resulting row iterator:\n\nhttps://github.com/erikgrinaker/toydb/blob/686d3971a253bfc9facc2ba1b0e716cff5c109fb/src/sql/execution/executor.rs#L203-L204\n\n`HashJoin` will then join the output rows from the `movies` and `genres` iterators by using a\nhash join. This builds an in-memory table for `genres` and then iterates over `movies`, joining\nthe rows:\n\nhttps://github.com/erikgrinaker/toydb/blob/686d3971a253bfc9facc2ba1b0e716cff5c109fb/src/sql/execution/executor.rs#L128-L141\n\nhttps://github.com/erikgrinaker/toydb/blob/889aef9f24c0fa4d58e314877fa17559a9f3d5d2/src/sql/execution/join.rs#L103-L183\n\nThe `Projection` node will simply evaluate the (trivial) column expressions using each joined\nrow as input:\n\nhttps://github.com/erikgrinaker/toydb/blob/686d3971a253bfc9facc2ba1b0e716cff5c109fb/src/sql/execution/executor.rs#L179-L186\n\nAnd finally the `Order` node will sort the results (which requires buffering them all in memory):\n\nhttps://github.com/erikgrinaker/toydb/blob/686d3971a253bfc9facc2ba1b0e716cff5c109fb/src/sql/execution/executor.rs#L173-L177\n\nhttps://github.com/erikgrinaker/toydb/blob/686d3971a253bfc9facc2ba1b0e716cff5c109fb/src/sql/execution/executor.rs#L298-L328\n\nThe output row iterator of `Order` is returned via `ExecutionResult::Select`, and the caller can now\ngo ahead and pull the resulting rows from it.\n\n## Session Management\n\nThe entry point to the SQL engine is the `sql::execution::Session`, which represents a single user\nsession. It is obtained via `sql::engine::Engine::session()`.\n\nhttps://github.com/erikgrinaker/toydb/blob/0839215770e31f1e693d5cccf20a68210deaaa3f/src/sql/execution/session.rs#L14-L21\n\nThe session takes a series of raw SQL statement strings as input and parses them:\n\nhttps://github.com/erikgrinaker/toydb/blob/0839215770e31f1e693d5cccf20a68210deaaa3f/src/sql/execution/session.rs#L29-L33\n\nFor each statement, it returns a result depending on the kind of statement:\n\nhttps://github.com/erikgrinaker/toydb/blob/0839215770e31f1e693d5cccf20a68210deaaa3f/src/sql/execution/session.rs#L132-L148\n\nThe session itself performs transaction control. It handles `BEGIN`, `COMMIT`, and `ROLLBACK`\nstatements, and modifies the transaction accordingly.\n\nhttps://github.com/erikgrinaker/toydb/blob/0839215770e31f1e693d5cccf20a68210deaaa3f/src/sql/execution/session.rs#L34-L70\n\nAny other statements are processed by the SQL planner, optimizer, and executor as we've seen in\nprevious sections.\n\nhttps://github.com/erikgrinaker/toydb/blob/0839215770e31f1e693d5cccf20a68210deaaa3f/src/sql/execution/session.rs#L77-L83\n\nThese statements are always executed using the session's current transaction. If there is no active\ntransaction, the session will create a new, implicit transaction for each statement.\n\nhttps://github.com/erikgrinaker/toydb/blob/0839215770e31f1e693d5cccf20a68210deaaa3f/src/sql/execution/session.rs#L87-L112\n\nAnd with that, we have a fully functional SQL engine!\n\n---\n\n

\n← SQL Optimization   |   Server →\n

" }, { "path": "docs/architecture/sql-optimizer.md", "content": "# SQL Optimization\n\n[Query optimization](https://en.wikipedia.org/wiki/Query_optimization) attempts to improve query\nperformance and efficiency by altering the execution plan. This is a deep and complex field, and\nwe can only scratch the surface here.\n\ntoyDB's query optimizer is very basic -- it only has a handful of rudimentary heuristic\noptimizations to illustrate how the process works. Real-world optimizers use much more sophisticated\nmethods, including statistical analysis, cost estimation, adaptive execution, etc.\n\nThe optimizers are located in the [`sql::planner::optimizer`](https://github.com/erikgrinaker/toydb/blob/213e5c02b09f1a3cac6a8bbd0a81773462f367f5/src/sql/planner/optimizer.rs) module.\nAn optimizer `sql::planner::Optimizer` just takes in a plan node `sql::planner::Node` (the root node\nin the plan), and returns an optimized node:\n\nhttps://github.com/erikgrinaker/toydb/blob/213e5c02b09f1a3cac6a8bbd0a81773462f367f5/src/sql/planner/optimizer.rs#L20-L25\n\nOptimizations are always implemented as recursive node transformations. To help with this, `Node`\nhas the helper methods `Node::transform` and `Node::transform_expressions` which recurse into a node\nor expression tree and call a given transformation closure on each node, as either\n[pre-order](https://en.wikipedia.org/wiki/Tree_traversal#Pre-order,_NLR) or\n[post-order](https://en.wikipedia.org/wiki/Tree_traversal#Post-order,_LRN) transforms:\n\nhttps://github.com/erikgrinaker/toydb/blob/213e5c02b09f1a3cac6a8bbd0a81773462f367f5/src/sql/planner/plan.rs#L269-L371\n\nA technique that's often useful during optimization is to convert expressions into\n[conjunctive normal form](https://en.wikipedia.org/wiki/Conjunctive_normal_form), i.e. \"an AND of\nORs\". For example, the two following expressions are equivalent, but the latter is in conjunctive\nnormal form (it's a chain of ANDs):\n\n```\n(a AND b) OR (c AND d) → (a OR c) AND (a OR d) AND (b OR c) AND (b OR d)\n```\n\nThis is useful because we can often move each AND operand independently around in the plan tree\nand still get the same result -- we'll see this in action later. Expressions are converted into\nconjunctive normal form via `Expression::into_cnf`, which is implemented using\n[De Morgan's laws](https://en.wikipedia.org/wiki/De_Morgan%27s_laws):\n\nhttps://github.com/erikgrinaker/toydb/blob/9419bcf6aededf0e20b4e7485e2a5fa3e975d79f/src/sql/types/expression.rs#L289-L351\n\nWe'll have a brief look at all of toyDB's optimizers, which are listed here in the order they're\napplied:\n\nhttps://github.com/erikgrinaker/toydb/blob/213e5c02b09f1a3cac6a8bbd0a81773462f367f5/src/sql/planner/optimizer.rs#L9-L18\n\nTest scripts for the optimizers are in [`src/sql/testscripts/optimizers`](https://github.com/erikgrinaker/toydb/tree/9419bcf6aededf0e20b4e7485e2a5fa3e975d79f/src/sql/testscripts/optimizers),\nand show how query plans evolve as each optimizer is applied.\n\n## Constant Folding\n\nThe `ConstantFolding` optimizer performs [constant folding](https://en.wikipedia.org/wiki/Constant_folding).\nThis pre-evaluates constant expressions in the plan during planning, instead of evaluating them\nfor every row during execution.\n\nhttps://github.com/erikgrinaker/toydb/blob/213e5c02b09f1a3cac6a8bbd0a81773462f367f5/src/sql/planner/optimizer.rs#L27-L30\n\nFor example, consider the query `SELECT 1 + 2 * 3 - foo FROM bar`. There is no point in\nre-evaluating `1 + 2 * 3` for every row in `bar`, because the result is always the same, so we can\njust evaluate this once during planning, transforming the expression into `7 - foo`.\n\nConcretely, this plan:\n\n```\nSelect\n└─ Projection: 1 + 2 * 3 - bar.foo\n └─ Scan: bar\n```\n\nShould be transformed into this plan:\n\n```\nSelect\n└─ Projection: 7 - bar.foo\n └─ Scan: bar\n```\n\nTo do this, `ConstantFolding` simply checks whether an `Expression` tree contains an\n`Expression::Column` node -- if it doesn't, then it much be a constant expression (since that's the\nonly dynamic value in an expression), and we can evaluate it with a `None` input row and replace the\noriginal expression node with an `Expression::Constant` node.\n\nThis is done recursively for each plan node, and recursively for each expression node (so it does\nthis both for `SELECT`, `WHERE`, `ORDER BY`, and all other parts of the query). Notably, it does a\npost-order expression transform, so it starts at the expression leaf nodes and attempts to transform\neach expression node as it moves back up the tree -- this allows it to iteratively evaluate constant\nparts as far as possible for each branch.\n\nhttps://github.com/erikgrinaker/toydb/blob/213e5c02b09f1a3cac6a8bbd0a81773462f367f5/src/sql/planner/optimizer.rs#L32-L56\n\nAdditionally, `ConstantFolding` also short-circuits logical expressions. For example, the expression\n`foo AND FALSE` will always be `FALSE`, regardless of what `foo` is, so we can replace it with\n`FALSE`:\n\nhttps://github.com/erikgrinaker/toydb/blob/213e5c02b09f1a3cac6a8bbd0a81773462f367f5/src/sql/planner/optimizer.rs#L58-L84\n\nAs the code comment mentions though, this doesn't fold optimally: it doesn't attempt to rearrange\nexpressions, which would require knowledge of precedence rules. For example, `(1 + foo) - 2` could\nbe folded into `foo - 1` by first rearranging it as `foo + (1 - 2)`, but we don't do this currently.\n\n## Filter Pushdown\n\nThe `FilterPushdown` optimizer attempts to push filter predicates as far down into the plan as\npossible, to reduce the number of rows each node has to process.\n\nhttps://github.com/erikgrinaker/toydb/blob/213e5c02b09f1a3cac6a8bbd0a81773462f367f5/src/sql/planner/optimizer.rs#L90-L95\n\nRecall the `movies` query plan from the planning section:\n\n```\nSelect\n└─ Order: movies.released desc\n └─ Projection: movies.title, movies.released, genres.name as genre\n └─ Filter: movies.released >= 2000\n └─ NestedLoopJoin: inner on movies.genre_id = genres.id\n ├─ Scan: movies\n └─ Scan: genres\n```\n\nEven though we're filtering on `release >= 2000`, the `Scan` node still has to read all of them from\ndisk and send them via Raft, and the `NestedLoopJoin` node still has to join all of them. It would\nbe nice if we could push this filtering into the `NestedLoopJoin` and `Scan` nodes and avoid this\nextra work, and this is exactly what `FilterPushdown` does.\n\nThe only plan nodes that have predicates that can be pushed down are `Filter` nodes and\n`NestedLoopJoin` nodes, so we recurse through the plan tree and look for these nodes, attempting\nto push down.\n\nhttps://github.com/erikgrinaker/toydb/blob/213e5c02b09f1a3cac6a8bbd0a81773462f367f5/src/sql/planner/optimizer.rs#L97-L110\n\nWhen it encounters the `Filter` node, it will extract the predicate and attempt to push it down\ninto its `source` node:\n\nhttps://github.com/erikgrinaker/toydb/blob/213e5c02b09f1a3cac6a8bbd0a81773462f367f5/src/sql/planner/optimizer.rs#L139-L153\n\nIf the source node is a `Filter`, `NestedLoopJoin`, or `Scan` node, then we can push the predicate\ndown into it by `AND`ing it with the existing predicate (if any).\n\nhttps://github.com/erikgrinaker/toydb/blob/213e5c02b09f1a3cac6a8bbd0a81773462f367f5/src/sql/planner/optimizer.rs#L112-L137\n\nIn our case, we were able to push the `Filter` into the `NestedLoopJoin`, and our plan now looks\nlike this:\n\n```\nSelect\n└─ Order: movies.released desc\n └─ Projection: movies.title, movies.released, genres.name as genre\n └─ NestedLoopJoin: inner on movies.genre_id = genres.id AND movies.released >= 2000\n ├─ Scan: movies\n └─ Scan: genres\n```\n\nBut we're still not done, as we'd like to push `movies.released >= 2000` down into the `Scan` node.\nPushdown for join nodes is a little more tricky, because we can only push down parts of the\nexpression that reference one of the source nodes.\n\nWe first have to convert the expression into conjunctive normal form, i.e. and AND of ORs, as we've\ndiscussed previously. This allows us to examine and push down each AND part in isolation, because it\nhas the same effect regardless of whether it is evaluated in the `NestedLoopJoin` node or one of\nthe source nodes. Our expression is already in conjunctive normal form, though.\n\nWe then look at each AND part, and check which side of the join it has column references for. If it\nonly references one of the sides, then the expression can be pushed down into it. We also make some\neffort here to move primary/foreign key constants across to both sides, but we'll gloss over that.\n\nhttps://github.com/erikgrinaker/toydb/blob/213e5c02b09f1a3cac6a8bbd0a81773462f367f5/src/sql/planner/optimizer.rs#L155-L247\n\nThis allows us to push down the `movies.released >= 2000` predicate into the corresponding `Scan`\nnode, significantly reducing the amount of data transferred across Raft:\n\n```\nSelect\n└─ Order: movies.released desc\n └─ Projection: movies.title, movies.released, genres.name as genre\n └─ NestedLoopJoin: inner on movies.genre_id = genres.id\n ├─ Scan: movies (released >= 2000)\n └─ Scan: genres\n```\n\n## Index Lookups\n\nThe `IndexLookup` optimizer uses primary key or secondary index lookups instead of full table\nscans where possible.\n\nhttps://github.com/erikgrinaker/toydb/blob/213e5c02b09f1a3cac6a8bbd0a81773462f367f5/src/sql/planner/optimizer.rs#L250-L252\n\nThe optimizer itself is fairly straightforward. It assumes that `FilterPushdown` has already pushed\npredicates down into `Scan` nodes, so it only needs to examine these. It converts the predicate into\nconjunctive normal form, and looks for any parts that are direct column lookups -- i.e.\n`column = value` (possibly a long OR chain of these).\n\nIf it finds any, and the column is either a primary key or secondary index column, then we convert\nthe `Scan` node into either a `KeyLookup` or `IndexLookup` node respectively. If there are any\nfurther AND predicates remaining, we add a parent `Filter` node to keep these predicates.\n\nFor example, the following plan:\n\n```\nSelect\n└─ Scan: movies ((id = 1 OR id = 7 OR id = 3) AND released >= 2000)\n```\n\nWill be transformed into one that does individual key lookups rather than a full table scan:\n\n```\nSelect\n└─ Filter: movies.released >= 2000\n └─ KeyLookup: movies (1, 3, 7)\n```\n\nThe code is as outlined above:\n\nhttps://github.com/erikgrinaker/toydb/blob/213e5c02b09f1a3cac6a8bbd0a81773462f367f5/src/sql/planner/optimizer.rs#L254-L303\n\nHelped by `Expression::is_column_lookup()` and `Expression::into_column_values()`:\n\nhttps://github.com/erikgrinaker/toydb/blob/9419bcf6aededf0e20b4e7485e2a5fa3e975d79f/src/sql/types/expression.rs#L363-L421\n\n## Hash Join\n\nThe `HashJoin` optimizer will replace a `NestedLoopJoin` with a `HashJoin` where possible.\n\nhttps://github.com/erikgrinaker/toydb/blob/213e5c02b09f1a3cac6a8bbd0a81773462f367f5/src/sql/planner/optimizer.rs#L305-L307\n\nA [nested loop join](https://en.wikipedia.org/wiki/Nested_loop_join) is a very inefficient O(n²)\nalgorithm, which iterates over all rows in the right source for each row in the left source to see\nif they match. However, it is completely general, and can join on arbitraily complex predicates.\n\nIn the common case where the join predicate is an equality comparison such as\n`movies.genre_id = genres.id` (i.e. an [equijoin](https://en.wikipedia.org/wiki/Relational_algebra#θ-join_and_equijoin)),\nthen we can instead use a [hash join](https://en.wikipedia.org/wiki/Hash_join). This scans the right\ntable once, builds an in-memory hash table from it, and for each left row it looks up any right rows\nin the hash table. This is a much more efficient O(n) algorithm.\n\nIn our previous movie example, we are in fact doing an equijoin:\n\n```\nSelect\n└─ Order: movies.released desc\n └─ Projection: movies.title, movies.released, genres.name as genre\n └─ NestedLoopJoin: inner on movies.genre_id = genres.id\n ├─ Scan: movies (released >= 2000)\n └─ Scan: genres\n```\n\nAnd so our `NestedLoopJoin` can be replaced by a `HashJoin`:\n\n```\nSelect\n└─ Order: movies.released desc\n └─ Projection: movies.title, movies.released, genres.name as genre\n └─ HashJoin: inner on movies.genre_id = genres.id\n ├─ Scan: movies (released >= 2000)\n └─ Scan: genres\n```\n\nThe `HashJoin` optimizer is extremely simple: if the join predicate is an equijoin, use a hash join.\nThis isn't always a good idea (the right source can be huge and we can run out of memory for the\nhash table), but we keep it simple.\n\nhttps://github.com/erikgrinaker/toydb/blob/213e5c02b09f1a3cac6a8bbd0a81773462f367f5/src/sql/planner/optimizer.rs#L309-L348\n\nOf course there are many other join algorithms out there, and one of the harder problems in SQL\noptimization is how to efficiently perform large N-way multijoins. We don't attempt to tackle these\nproblems here -- the `HashJoin` optimizer is just a very simple example of such join optimization.\n\n## Short Circuiting\n\nThe `ShortCircuit` optimizer tries to find nodes that can't possibly do any useful work, and either\nremoves them from the plan, or replaces them with trivial nodes that don't do anything. It is kind\nof similar to the `ConstantFolding` optimizer in spirit, but works on plan nodes rather than\nexpression nodes.\n\nhttps://github.com/erikgrinaker/toydb/blob/213e5c02b09f1a3cac6a8bbd0a81773462f367f5/src/sql/planner/optimizer.rs#L350-L354\n\nFor example, `Filter` nodes with a `TRUE` predicate won't actually filter anything:\n\n```\nSelect\n└─ Filter: true\n └─ Scan: movies\n```\n\nSo we can just remove them:\n\n```\nSelect\n└─ Scan: movies\n```\n\nSimilarly, `Filter` nodes with a `FALSE` predicate will never emit anything:\n\n```\nSelect\n└─ Filter: false\n └─ Scan: movies\n```\n\nThere's no point doing a scan in this case, so we can just replace it with a `Nothing` node that\ndoes no work and doesn't emit anything:\n\n```\nSelect\n└─ Nothing\n```\n\nThe optimizer tries to find a bunch of such patterns. This can also tidy up query plans a fair bit\nby removing unnecessary cruft.\n\nhttps://github.com/erikgrinaker/toydb/blob/213e5c02b09f1a3cac6a8bbd0a81773462f367f5/src/sql/planner/optimizer.rs#L356-L438\n\n---\n\n

\n← SQL Planning   |   SQL Execution →\n

" }, { "path": "docs/architecture/sql-parser.md", "content": "# SQL Parsing\n\nWe finally arrive at SQL. The SQL parser is the first stage in processing SQL queries and\nstatements, located in the [`sql::parser`](https://github.com/erikgrinaker/toydb/tree/39c6b60afc4c235f19113dc98087176748fa091d/src/sql/parser)\nmodule.\n\nThe SQL parser's job is to take a raw SQL string and turn it into a structured form that's more\nconvenient to work with. In doing so, it will validate that the string is in fact valid SQL\n_syntax_. However, it doesn't know if the SQL statement actually makes sense -- it has no idea which\ntables or columns exist, what their data types are, and so on. That's the job of the planner, which\nwe'll look at later.\n\nFor example, let's say the parser is given the following SQL query:\n\n```sql\nSELECT name, price, price * 25 / 100 AS vat\nFROM products JOIN categories ON products.category_id = categories.id\nWHERE categories.code = 'BLURAY' AND stock > 0\nORDER BY price DESC\nLIMIT 10\n```\n\nIt will generate a structure that looks something like this (in simplified syntax):\n\n```rust\n// A SELECT statement.\nStatement::Select {\n // SELECT name, price, price * 25 / 100 AS vat\n select: [\n (Column(\"name\"), None),\n (Column(\"price\"), None),\n (\n Divide(\n Multiply(Column(\"price\"), Integer(25)),\n Integer(100)\n ),\n Some(\"vat\"),\n ),\n ]\n\n // FROM products JOIN categories ON products.category_id = categories.id\n from: [\n Join {\n left: Table(\"products\"),\n right: Table(\"categories\"),\n type: Inner,\n predicate: Some(\n Equal(\n Column(\"products.category_id)\",\n Column(\"categories.id\"),\n )\n )\n }\n ]\n\n // WHERE categories.code = 'BLURAY' AND stock > 0\n where: Some(\n And(\n Equal(\n Column(\"categories.code\"),\n String(\"BLURAY\"),\n ),\n GreaterThan(\n Column(\"stock\"),\n Integer(0),\n )\n )\n )\n\n // ORDER BY price DESC\n order: [\n (Column(\"price\"), Descending),\n ]\n\n // LIMIT 10\n limit: Some(Integer(10))\n}\n```\n\nLet's have a look at how this happens.\n\n## Lexer\n\nWe begin with the `sql::parser::Lexer`, which takes the raw SQL string and performs\n[lexical analysis](https://en.wikipedia.org/wiki/Lexical_analysis) to convert it into a sequence of\ntokens. These tokens are things like number, string, identifier, SQL keyword, and so on.\n\nThis preprocessing is useful to deal with some of the \"noise\" of SQL text, such as whitespace,\nstring quotes, identifier normalization, and so on. It also specifies which symbols and keywords are\nvalid in our SQL queries. This makes the parser's life a lot easier.\n\nThe lexer doesn't care about SQL structure at all, only that the individual pieces (tokens) of a\nstring are well-formed. For example, the following input string:\n\n```\n'foo' ) 3.14 SELECT + x\n```\n\nWill result in these tokens:\n\n```\nString(\"foo\") CloseParen Number(\"3.14\") Keyword(Select) Plus Ident(\"x\")\n```\n\nTokens and keywords are represented by the `sql::parser::Token` and `sql::parser::Keyword` enums\nrespectively:\n\nhttps://github.com/erikgrinaker/toydb/blob/39c6b60afc4c235f19113dc98087176748fa091d/src/sql/parser/lexer.rs#L8-L47\n\nhttps://github.com/erikgrinaker/toydb/blob/39c6b60afc4c235f19113dc98087176748fa091d/src/sql/parser/lexer.rs#L86-L155\n\nThe lexer takes an input string and emits tokens as an iterator:\n\nhttps://github.com/erikgrinaker/toydb/blob/39c6b60afc4c235f19113dc98087176748fa091d/src/sql/parser/lexer.rs#L311-L337\n\nIt does this by repeatedly attempting to scan the next token until it reaches the end of the string\n(or errors). It can determine the kind of token by looking at the first character:\n\nhttps://github.com/erikgrinaker/toydb/blob/39c6b60afc4c235f19113dc98087176748fa091d/src/sql/parser/lexer.rs#L358-L373\n\nAnd then scan across the following characters as appropriate to generate a valid token. For example,\nthis is how a quoted string (e.g. `'foo'`) is lexed into a `Token::String` (including handling of\nany escaped quotes inside the string):\n\nhttps://github.com/erikgrinaker/toydb/blob/39c6b60afc4c235f19113dc98087176748fa091d/src/sql/parser/lexer.rs#L435-L451\n\nThese tokens become the input to the parser.\n\n## Abstract Syntax Tree\n\nThe end result of the parsing process will be an [abstract syntax tree](https://en.wikipedia.org/wiki/Abstract_syntax_tree)\n(AST), which is a structured representation of a SQL statement, located in the\n[`sql::parser::ast`](https://github.com/erikgrinaker/toydb/blob/39c6b60afc4c235f19113dc98087176748fa091d/src/sql/parser/ast.rs) module.\n\nThe root of this tree is the `sql::parser::ast::Statement` enum, which represents all the different\nkinds of SQL statements that we support, along with their contents:\n\nhttps://github.com/erikgrinaker/toydb/blob/39c6b60afc4c235f19113dc98087176748fa091d/src/sql/parser/ast.rs#L6-L145\n\nThe nested tree structure is particularly apparent with expressions, which represent values and\noperations on them. For example, the expression `2 * 3 - 4 / 2`, which evaluates to the value `4`.\n\nWe've seen in the data model section how such expressions are represented as\n`sql::types::Expression`, but before we get there we have to parse them. The parser has its own\nrepresentation `sql::parser::ast::Expression` -- this is necessary e.g. because in the AST, we\nrepresent columns as names rather than numeric indexes (we don't know yet which columns exist or\nwhat their names are, we'll get to that during planning).\n\nhttps://github.com/erikgrinaker/toydb/blob/39c6b60afc4c235f19113dc98087176748fa091d/src/sql/parser/ast.rs#L147-L170\n\nhttps://github.com/erikgrinaker/toydb/blob/39c6b60afc4c235f19113dc98087176748fa091d/src/sql/parser/ast.rs#L204-L234\n\nFor example, `2 * 3 - 4 / 2` is represented as:\n\n```rust\nExpression::Operator(Operator::Subtract(\n // The left-hand operand of -\n Expression::Operator(Operator::Multiply(\n // The left-hand operand of *\n Expression::Literal(Literal::Integer(2)),\n // The right-hand operand of *\n Expression::Literal(Literal::Integer(3)),\n )),\n // The right-hand operand of -\n Expression::Operator(Operator::Divide(\n // The left-hand operand of /\n Expression::Literal(Literal::Integer(4)),\n // The right-hand operand of /\n Expression::Literal(Literal::Integer(2)),\n )),\n))\n```\n\n## Parser\n\nThe parser, `sql::parser::Parser`, takes lexer tokens as input and builds an `ast::Statement`\nfrom them:\n\nhttps://github.com/erikgrinaker/toydb/blob/39c6b60afc4c235f19113dc98087176748fa091d/src/sql/parser/parser.rs#L9-L32\n\nWe can determine the kind of statement we're parsing simply by looking at the first keyword:\n\nhttps://github.com/erikgrinaker/toydb/blob/39c6b60afc4c235f19113dc98087176748fa091d/src/sql/parser/parser.rs#L109-L130\n\nLet's see how a `SELECT` statement is parsed. The different clauses in a `SELECT` (e.g. `FROM`,\n`WHERE`, etc.) must always be given in a specific order, and they always begin with the appropriate\nkeyword, so we can simply try to parse each clause in the expected order:\n\nhttps://github.com/erikgrinaker/toydb/blob/39c6b60afc4c235f19113dc98087176748fa091d/src/sql/parser/parser.rs#L330-L342\n\nParsing each clause is also just a matter of parsing the expected parts in order. For example, the\ninitial `SELECT` clause is just a comma-separated list of expressions with an optional alias:\n\nhttps://github.com/erikgrinaker/toydb/blob/39c6b60afc4c235f19113dc98087176748fa091d/src/sql/parser/parser.rs#L344-L365\n\nThe `FROM` clause is a comma-separated list of table name, optionally joined with other tables:\n\nhttps://github.com/erikgrinaker/toydb/blob/39c6b60afc4c235f19113dc98087176748fa091d/src/sql/parser/parser.rs#L367-L427\n\nAnd the `WHERE` clause is just a predicate expression to filter by:\n\nhttps://github.com/erikgrinaker/toydb/blob/39c6b60afc4c235f19113dc98087176748fa091d/src/sql/parser/parser.rs#L429-L435\n\nExpression parsing is where this gets tricky, because we have to respect the rules of operator\nprecedence and associativity. For example, according to mathematical order of operations (aka\n\"PEMDAS\") the expression `2 * 3 - 4 / 2` must be parsed as `(2 * 3) - (4 / 2)` which yields 4, not\n`2 * (3 - 4) / 2` which yields -1.\n\ntoyDB does this using the [precedence climbing algorithm](https://en.wikipedia.org/wiki/Operator-precedence_parser#Precedence_climbing_method),\nwhich is a fairly simple and compact algorithm as far as these things go. In a nutshell, it will\ngreedily and recursively group operators together as long as their precedence is the same or higher\nthan that of the operators preceding them (hence \"precedence climbing\"). For example:\n\n```\n----- ----- Precedence 2: * and /\n------------- Precedence 1: -\n2 * 3 - 4 / 2\n```\n\nThe algorithm is documented in more detail on `Parser::parse_expression()`:\n\nhttps://github.com/erikgrinaker/toydb/blob/39c6b60afc4c235f19113dc98087176748fa091d/src/sql/parser/parser.rs#L501-L696\n\n---\n\n

\n← SQL Raft Replication   |   SQL Planning →\n

" }, { "path": "docs/architecture/sql-planner.md", "content": "# SQL Planning\n\nThe SQL planner in the [`sql::planner`](https://github.com/erikgrinaker/toydb/tree/c64012e29c5712d6fe028d3d5375a98b8faea266/src/sql/planner)\nmodule takes a SQL statement AST from the parser and generates an execution plan for it. We won't\nactually execute it just yet though, only figure out how to execute it.\n\n## Execution Plan\n\nA plan is represented by the `sql::planner::Plan` enum. The variant specifies the operation to\nexecute (e.g. `SELECT`, `INSERT`, `UPDATE`, `DELETE`):\n\nhttps://github.com/erikgrinaker/toydb/blob/213e5c02b09f1a3cac6a8bbd0a81773462f367f5/src/sql/planner/plan.rs#L15-L73\n\nBelow the root, the plan is typically made of up of a tree of nested `sql::planner::Node`. Each node\nemits a stream of SQL rows as output, and may take streams of input rows from child nodes.\n\nhttps://github.com/erikgrinaker/toydb/blob/213e5c02b09f1a3cac6a8bbd0a81773462f367f5/src/sql/planner/plan.rs#L106-L175\n\nHere is an example, taken from the `Plan` code comment above:\n\n```sql\nSELECT title, released, genres.name AS genre\nFROM movies INNER JOIN genres ON movies.genre_id = genres.id\nWHERE released >= 2000\nORDER BY released\n```\n\nWhich results in this query plan:\n\n```\nSelect\n└─ Order: movies.released desc\n └─ Projection: movies.title, movies.released, genres.name as genre\n └─ Filter: movies.released >= 2000\n └─ NestedLoopJoin: inner on movies.genre_id = genres.id\n ├─ Scan: movies\n └─ Scan: genres\n```\n\nRows flow from the tree leaves to the root:\n\n1. `Scan` nodes read rows from the tables `movies` and `genres`.\n2. `NestedLoopJoin` joins the rows from `movies` and `genres`.\n3. `Filter` discards rows with release dates older than 2000.\n4. `Projection` picks out the requested column values from the rows.\n5. `Order` sorts the rows by release date.\n6. `Select` returns the final rows to the client.\n\n## Scope and Name Resolution\n\nOne of the main jobs of the planner is to resolve column names to column indexes in the input rows\nof each node.\n\nIn the query example above, the `WHERE released >= 2000` filter may refer to a column `released`\nfrom either the joined `movies` table or the `genres` tables. The planner needs to figure out which\ntable has a `released` column, and also figure out which column number in the `NestedLoopJoin`\noutput rows corresponds to the `released` column (for example column number 2).\n\nThis job is further complicated by the fact that many nodes can alias, reorder, or drop columns,\nand some nodes may also refer to columns that shouldn't be part of the result at all (for example,\nit's possible to `ORDER BY` a column that won't be output by a `SELECT` projection at all, but\nthe `Order` node still needs access to the column data to sort by it).\n\nThe planner uses a `sql::planner::Scope` to keep track of which column names are currently visible,\nand which column indexes they refer to. For each node the planner builds, starting from the leaves,\nit creates a new `Scope` that contains the currently visible columns, tracking how they are modified\nand rearranged by each node.\n\nhttps://github.com/erikgrinaker/toydb/blob/6f6cec4db10bc015a37ee47ff6c7dae383147dd5/src/sql/planner/planner.rs#L577-L610\n\nWhen an AST expression refers to a column name, the planner can use `Scope::lookup_column()` to find\nout which column number the expression should take its input value from.\n\nhttps://github.com/erikgrinaker/toydb/blob/6f6cec4db10bc015a37ee47ff6c7dae383147dd5/src/sql/planner/planner.rs#L660-L686\n\n## Planner\n\nThe planner itself is `sql:planner::Planner`. It uses a `sql::engine::Catalog` to look up\ninformation about tables and columns from storage.\n\nhttps://github.com/erikgrinaker/toydb/blob/6f6cec4db10bc015a37ee47ff6c7dae383147dd5/src/sql/planner/planner.rs#L12-L20\n\nTo build an execution plan, the planner first looks at the `ast::Statement` kind to determine\nwhat kind of plan to build:\n\nhttps://github.com/erikgrinaker/toydb/blob/6f6cec4db10bc015a37ee47ff6c7dae383147dd5/src/sql/planner/planner.rs#L28-L47\n\nLet's build this `SELECT` plan from above:\n\n```sql\nSELECT title, released, genres.name AS genre\nFROM movies INNER JOIN genres ON movies.genre_id = genres.id\nWHERE released >= 2000\nORDER BY released\n```\n\nWhich should result in this plan:\n\n```\nSelect\n└─ Order: movies.released desc\n └─ Projection: movies.title, movies.released, genres.name as genre\n └─ Filter: movies.released >= 2000\n └─ NestedLoopJoin: inner on movies.genre_id = genres.id\n ├─ Scan: movies\n └─ Scan: genres\n```\n\nThe planner is given the following (simplified) AST from the parser as input:\n\n```rust\n// A SELECT statement.\nStatement::Select {\n // SELECT title, released, genres.name AS genre\n select: [\n (Column(\"title\"), None),\n (Column(\"released\"), None),\n (Column(\"genres.name\"), \"genre\"),\n ]\n\n // FROM movies INNER JOIN genres ON movies.genre_id = genres.id\n from: [\n Join {\n left: Table(\"movies\"),\n right: Table(\"genres\"),\n type: Inner,\n predicate: Some(\n Equal(\n Column(\"movies.genre_id\"),\n Column(\"genres.id\"),\n )\n )\n }\n ]\n\n // WHERE released >= 2000\n where: Some(\n GreaterThanOrEqual(\n Column(\"released\"),\n Integer(2000),\n )\n )\n\n // ORDER BY released\n order: [\n (Column(\"released\"), Ascending),\n ]\n}\n```\n\nThe first thing `Planner::build_select` does is to create an empty scope (which will track column\nnames and indexes) and build the `FROM` clause which will generate the initial input rows:\n\nhttps://github.com/erikgrinaker/toydb/blob/6f6cec4db10bc015a37ee47ff6c7dae383147dd5/src/sql/planner/planner.rs#L170-L179\n\nhttps://github.com/erikgrinaker/toydb/blob/6f6cec4db10bc015a37ee47ff6c7dae383147dd5/src/sql/planner/planner.rs#L283-L289\n\n`Planner::build_from()` first encounters the `ast::From::Join` item, which joins `movies` and\n`genres`. This will build a `Node::NestedLoopJoin` plan node for the join, which is the simplest and\nmost straightforward join algorithm -- it simply iterates over all rows in the `genres` table for\nevery row in the `movies` table and emits the joined rows (we'll see how to optimize it with a\nbetter join algorithm later).\n\nhttps://github.com/erikgrinaker/toydb/blob/6f6cec4db10bc015a37ee47ff6c7dae383147dd5/src/sql/planner/planner.rs#L319-L344\n\nIt first recurses into `Planner::build_from()` to build each of the `ast::From::Table` nodes for\neach table. This will look up the table schemas in the catalog, add them to the current scope, and\nbuild a `Node::Scan` node which will emit all rows from each table. The `Node::Scan` nodes are\nplaced into the `Node::NestedLoopJoin` above.\n\nhttps://github.com/erikgrinaker/toydb/blob/6f6cec4db10bc015a37ee47ff6c7dae383147dd5/src/sql/planner/planner.rs#L312-L317\n\nWhile building the `Node::NestedLoopJoin`, it also needs to convert the join expression\n`movies.genre_id = genres.id` into a proper `sql::types::Expression`. This is done by\n`Planner::build_expression()`:\n\nhttps://github.com/erikgrinaker/toydb/blob/6f6cec4db10bc015a37ee47ff6c7dae383147dd5/src/sql/planner/planner.rs#L493-L568\n\nExpression building is mostly a direct translation from an `ast::Expression` variant to a\ncorresponding `sql::types::Expression` variant (for example from\n`ast::Expression::Operator(ast::Operator::Equal)` to `sql::types::Expression::Equal`). However, as\nmentioned earlier, `ast::Expression` contains column references by name, while\n`sql::types::Expression` contains column references as row indexes. This name resolution is done\nhere, by looking up the column names in the scope:\n\nhttps://github.com/erikgrinaker/toydb/blob/6f6cec4db10bc015a37ee47ff6c7dae383147dd5/src/sql/planner/planner.rs#L521-L523\n\nThe expression we're building is the join predicate of `Node::NestedLoopJoin`, so it operates on\njoined rows containing all columns of `movies` then all columns of `genres`. It also operates on all\ncombinations of joined rows (the [Cartesian product](https://en.wikipedia.org/wiki/Cartesian_product)),\nand the purpose of the join predicate is to determine which joined rows to actually keep. For\nexample, the full set of joined rows that are evaluated might be:\n\n| movies.id | movies.title | movies.released | movies.genre_id | genres.id | genres.name |\n|-----------|--------------|-----------------|-----------------|-----------|-------------|\n| 1 | Sicario | 2015 | 2 | 1 | Drama |\n| 2 | Sicario | 2015 | 2 | 2 | Action |\n| 3 | 21 Grams | 2003 | 1 | 1 | Drama |\n| 4 | 21 Grams | 2003 | 1 | 2 | Action |\n| 5 | Heat | 1995 | 2 | 1 | Drama |\n| 6 | Heat | 1995 | 2 | 2 | Action |\n\nThe join predicate should pick out the rows where `movies.genre_id = genres.id`. The scope will\nreflect the column layout in the example above, and can resolve the column names to zero-based row\nindexes as `#3 = #4`, which will be the final built `Expression`.\n\nNow that we've built the `FROM` clause into a `Node::NestedLoopJoin` of two `Node::Scan` nodes, we\nmove on to the `WHERE` clause. This simply builds the `WHERE` expression `released >= 2000`, like\nwe've already seen with the join predicate, and creates a `Node::Filter` node which takes its input\nrows from the `Node::NestedLoopJoin` and filters them by the given expression. Again, the scope\nkeeps track of which input columns we're getting from the join node and resolves the `released`\ncolumn reference in the expression.\n\nhttps://github.com/erikgrinaker/toydb/blob/6f6cec4db10bc015a37ee47ff6c7dae383147dd5/src/sql/planner/planner.rs#L202-L206\n\nWe then build the `SELECT` clause, which emits the `title, released, genres.name AS genre` columns.\nThis is just a list of expressions that are built in the current scope and placed into a\n`Node::Projection` (the expressions could be arbitrarily complex). However, we also have to make\nsure to update the scope with the final three columns that are output to subsequent nodes, taking\ninto account the `genre` alias for the original `genres.name` column (we won't dwell on the \"hidden\ncolumns\" mentioned there -- they're not relevant for our query).\n\nhttps://github.com/erikgrinaker/toydb/blob/6f6cec4db10bc015a37ee47ff6c7dae383147dd5/src/sql/planner/planner.rs#L214-L234\n\nFinally, we build the `ORDER BY` clause. Again, this just builds a trivial expression for `released`\nand places it into an `Node::Order` node which takes input rows from the `Node::Projection` and\nsorts them by the order expression.\n\nhttps://github.com/erikgrinaker/toydb/blob/6f6cec4db10bc015a37ee47ff6c7dae383147dd5/src/sql/planner/planner.rs#L245-L252\n\nAnd that's it. The `Node::Order` is placed into the root `Plan::Select`, and we have our final plan.\n\n```\nSelect\n└─ Order: movies.released desc\n └─ Projection: movies.title, movies.released, genres.name as genre\n └─ Filter: movies.released >= 2000\n └─ NestedLoopJoin: inner on movies.genre_id = genres.id\n ├─ Scan: movies\n └─ Scan: genres\n```\n\nWe'll see how to execute it soon, but first we should optimize it to see if we can make it run\nfaster -- in particular, to see if we can avoid reading all movies from storage, and if we can do\nbetter than the very slow nested loop join.\n\n---\n\n

\n← SQL Parsing   |   SQL Optimization →\n

" }, { "path": "docs/architecture/sql-raft.md", "content": "# SQL Raft Replication\n\ntoyDB uses Raft to replicate SQL storage across a cluster of nodes (see the Raft section for\ndetails). All nodes will store a full copy of the SQL database, and the Raft leader will replicate\nwrites across nodes and execute reads.\n\nRecall the Raft state machine interface `raft::State`:\n\nhttps://github.com/erikgrinaker/toydb/blob/8782c2b05f11333c1586ef248f1a13dc1c8dec4a/src/raft/state.rs#L4-L51\n\nIn toyDB, the state machine is just a `sql::engine::Local` storage engine with a thin wrapper:\n\nhttps://github.com/erikgrinaker/toydb/blob/c2b0f7f1d6cbf6e2cdc09fc0aec7b050e840ec21/src/sql/engine/raft.rs#L278-L291\n\nRaft will submit read and write commands to this state machine as binary `Vec` data, so we have\nto represent the methods of `sql::engine::Engine` as binary Raft commands. We do this as two\nenums, `sql::engine::raft::Read` and `sql::engine::raft::Write`, which we'll Bincode-encode:\n\nhttps://github.com/erikgrinaker/toydb/blob/c2b0f7f1d6cbf6e2cdc09fc0aec7b050e840ec21/src/sql/engine/raft.rs#L16-L71\n\nNotice that almost all requests include a `mvcc::TransactionState`. Most of the useful methods of\n`sql::engine::Engine` are on the `sql::engine::Transaction`, but unlike the `Local` engine, below\nRaft we can't hold on to a `Transaction` object in memory between each command -- nodes may restart\nand leadership may move, and we want client transactions to keep working despite this. Instead, we\nwill use the client-supplied `mvcc::TransactionState` to reconstruct a `Transaction` for every\ncommand via `mvcc::Transaction::resume()` and call methods on it.\n\nWhen the state machine receives a write command, it decodes it as a `Write` and calls the\nappropriate `Local` method. The result is Bincode-encoded and returned to the caller, who knows what\nreturn type to expect for a given command. The state machine also keeps track of the Raft applied\nindex of each command as a separate key in the key/value store.\n\nhttps://github.com/erikgrinaker/toydb/blob/c2b0f7f1d6cbf6e2cdc09fc0aec7b050e840ec21/src/sql/engine/raft.rs#L346-L367\n\nhttps://github.com/erikgrinaker/toydb/blob/c2b0f7f1d6cbf6e2cdc09fc0aec7b050e840ec21/src/sql/engine/raft.rs#L306-L338\n\nSimilarly, read commands are decoded as a `Read` and the appropriate `Local` method is called:\n\nhttps://github.com/erikgrinaker/toydb/blob/c2b0f7f1d6cbf6e2cdc09fc0aec7b050e840ec21/src/sql/engine/raft.rs#L369-L404\n\nThat's the state machine running below Raft. But how do we actually send these commands to Raft and\nreceive results? That's handled by the `sql::engine::Raft` implementation, which uses a channel to\nsend requests to the local Raft node (we'll see how this plumbing works in the server section):\n\nhttps://github.com/erikgrinaker/toydb/blob/c2b0f7f1d6cbf6e2cdc09fc0aec7b050e840ec21/src/sql/engine/raft.rs#L80-L95\n\nThe channel takes a `raft::Request` containing binary Raft client requests and a return channel\nwhere the Raft node can send back a `raft::Response`. The Raft engine has a few convenience methods\nto send requests and receive responses, for both read and write requests:\n\nhttps://github.com/erikgrinaker/toydb/blob/c2b0f7f1d6cbf6e2cdc09fc0aec7b050e840ec21/src/sql/engine/raft.rs#L114-L135\n\nAnd the implementation of the `sql::engine::Engine` and `sql::engine::Transaction` traits simply\nsend these requests via Raft:\n\nhttps://github.com/erikgrinaker/toydb/blob/c2b0f7f1d6cbf6e2cdc09fc0aec7b050e840ec21/src/sql/engine/raft.rs#L194-L276\n\nOne thing to note here is that we don't support streaming data via Raft, so e.g. the\n`Transaction::scan` method will buffer the entire result in a `Vec`. With a full table scan, this\nwill load the entire table into memory -- that's unfortunate, but we keep it simple.\n\nTo summarize, this is what happens when `Transaction::insert()` is called to insert a row via Raft:\n\n1. `sql::engine::raft::Transaction::insert()`: called to insert a row.\n2. `sql::engine::raft::Write::Insert`: enum representation of the insert command.\n3. `raft::Request::Write`: raft request containing the Bincode-encoded `Write::Insert` command.\n4. `sql::engine::raft::Engine::tx`: sends the `Request::Write` and response channel to Raft.\n5. `raft::Node::step()`: the `Request::Write` is given to Raft in a `Message::ClientRequest`.\n6. Raft does its replication thing, and commits the command's log entry.\n7. `raft::State::apply()`: the Bincode-encoded `Write::Insert` is passed to the state machine.\n8. `sql::engine::raft::State::apply()`: decodes the command to a `Write::Insert`.\n9. `sql::engine::raft::State::local`: contains the `Local` engine on each node.\n10. `sql::engine::local::Engine::resume()`: called to obtain the SQL/MVCC transaction.\n11. `sql::engine::local::Transaction::insert()`: the row is inserted to the local engine.\n12. `raft::RawNode::tx`: the `Ok(())` result is sent as a Bincode-encoded `Message::ClientResponse`.\n13. `sql::engine::raft::Transaction::insert()`: receives the result and returns it to the caller.\n\nThe plumbing here will be covered in more details in the server section.\n\n---\n\n

\n← SQL Storage   |   SQL Parsing →\n

" }, { "path": "docs/architecture/sql-storage.md", "content": "# SQL Storage\n\nThe SQL storage engine, in the [`sql::engine`](https://github.com/erikgrinaker/toydb/tree/213e5c02b09f1a3cac6a8bbd0a81773462f367f5/src/sql/engine)\nmodule, stores tables and rows. toyDB has two SQL storage implementations:\n\n* `sql::engine::Local`: local storage using a `storage::Engine` key/value store.\n* `sql::engine::Raft`: Raft-replicated storage, using `Local` on each node below Raft.\n\nThese implement the `sql::engine::Engine` trait, which specifies the SQL storage API. SQL execution\ncan use either simple local storage or Raft-replicated storage -- toyDB itself always uses the\nRaft-replicated engine, but many tests use a local in-memory engine.\n\nThe `sql::engine::Engine` trait is fully transactional, based on the `storage::MVCC` transaction\nengine discussed previously. As such, the trait just has a few methods that begin transactions --\nthe storage logic itself is implemented in the transaction, which we'll cover in next. The trait\nalso has a `session()` method to start SQL sessions for query execution, which we'll revisit in the\nexecution section.\n\nhttps://github.com/erikgrinaker/toydb/blob/0839215770e31f1e693d5cccf20a68210deaaa3f/src/sql/engine/engine.rs#L9-L29\n\nHere, we'll only look at the `Local` engine, and we'll discuss Raft replication afterwards. `Local`\nitself is just a thin wrapper around a `storage::MVCC` to create transactions:\n\nhttps://github.com/erikgrinaker/toydb/blob/39c6b60afc4c235f19113dc98087176748fa091d/src/sql/engine/local.rs#L50-L97\n\n## Key/Value Representation\n\n`Local` uses a `storage::Engine` key/value store to store SQL table schemas, table rows, and\nsecondary index entries. But how do we represent these as keys and values?\n\nThe keys are represented by the `sql::engine::Key` enum, and encoded using the Keycode encoding\nthat we've discussed in the encoding section:\n\nhttps://github.com/erikgrinaker/toydb/blob/39c6b60afc4c235f19113dc98087176748fa091d/src/sql/engine/local.rs#L15-L31\n\nThe values are encoded using the Bincode encoding, where the value type is given by the key:\n\n* `Key::Table` → `sql::types::Table` (table schemas)\n* `Key::Index` → `BTreeSet` (indexed primary keys)\n* `Key::Row` → `sql::types::Row` (table rows)\n\nRecall that the Keycode encoding will store keys in sorted order. This means that all `Key::Table`\nentries come first, then all `Key::Index`, then all `Key::Row`. These are further grouped and\nsorted by their fields.\n\nFor example, consider these SQL tables containing movies and genres, with a secondary index on\n`movies.genre_id` for fast lookups of movies with a given genre:\n\n```sql\nCREATE TABLE genres (\n id INTEGER PRIMARY KEY,\n name STRING NOT NULL\n);\n\nCREATE TABLE movies (\n id INTEGER PRIMARY KEY,\n title STRING NOT NULL,\n released INTEGER NOT NULL,\n genre_id INTEGER NOT NULL INDEX REFERENCES genres\n);\n\nINSERT INTO genres VALUES (1, 'Drama'), (2, 'Action');\n\nINSERT INTO movies VALUES\n (1, 'Sicario', 2015, 2),\n (2, '21 Grams', 2003, 1),\n (3, 'Heat', 1995, 2);\n```\n\nThis would result in the following illustrated keys and values, in the given order:\n\n```\n/Table/genres → Table { name: \"genres\", primary_key: 0, columns: ... }\n/Table/movies → Table { name: \"movies\", primary_key: 0, columns: ... }\n/Index/movies/genre_id/Integer(1) → BTreeSet { Integer(2) }\n/Index/movies/genre_id/Integer(2) → BTreeSet { Integer(1), Integer(3) }\n/Row/genres/Integer(1) → Row { Integer(1), String(\"Action\") }\n/Row/genres/Integer(2) → Row { Integer(2), String(\"Drama\") }\n/Row/movies/Integer(1) → Row { Integer(1), String(\"Sicario\"), Integer(2015), Integer(2) }\n/Row/movies/Integer(2) → Row { Integer(2), String(\"21 Grams\"), Integer(2003), Integer(1) }\n/Row/movies/Integer(3) → Row { Integer(3), String(\"Heat\"), Integer(1995), Integer(2) }\n```\n\nThus, if we want to do a full table scan of the `movies` table, we just do a prefix scan of\n`/Row/movies/`. If we want to do a secondary index lookup of all movies with `genre_id = 2`, we\nfetch `/Index/movies/genre_id/Integer(2)` and find that movies with `id = {1,3}` have this genre.\n\nTo help with prefix scans, the valid key prefixes are represented as `sql::engine::KeyPrefix`:\n\nhttps://github.com/erikgrinaker/toydb/blob/39c6b60afc4c235f19113dc98087176748fa091d/src/sql/engine/local.rs#L35-L48\n\nFor a look at the actual on-disk binary storage format, see the test scripts under\n[`src/sql/testscripts/writes`](https://github.com/erikgrinaker/toydb/tree/c2b0f7f1d6cbf6e2cdc09fc0aec7b050e840ec21/src/sql/testscripts/writes),\nwhich output the logical and raw binary representation of write operations.\n\n## Schema Catalog\n\nThe `sql::engine::Catalog` trait is used to store table schemas, i.e. `sql::types::Table`. It has a\nhandful of methods for creating, dropping and fetching tables (recall that toyDB does not support\nschema changes). The `Table::name` field is used as a unique table identifier throughout.\n\nhttps://github.com/erikgrinaker/toydb/blob/0839215770e31f1e693d5cccf20a68210deaaa3f/src/sql/engine/engine.rs#L60-L79\n\nThe `Catalog` trait is also fully transactional, as it must be implemented on a transaction via the\n`type Transaction: Transaction + Catalog` trait bound on `sql::engine::Engine`.\n\nCreating a table is straightforward: insert a key/value pair with a Keycode-encoded `Key::Table`\nfor the key and a Bincode-encoded `sql::types::Table` for the value. We first check that the\ntable doesn't already exist, and validate the table schema using `Table::validate()`.\n\nhttps://github.com/erikgrinaker/toydb/blob/39c6b60afc4c235f19113dc98087176748fa091d/src/sql/engine/local.rs#L340-L347\n\nSimilarly, fetching and listing tables is straightforward: just key/value gets or scans using the\nappropriate keys.\n\nhttps://github.com/erikgrinaker/toydb/blob/39c6b60afc4c235f19113dc98087176748fa091d/src/sql/engine/local.rs#L390-L399\n\nDropping tables is a bit more involved, since we have to perform some validation and also delete the\nactual table rows and any secondary index entries, but it's not terribly complicated:\n\nhttps://github.com/erikgrinaker/toydb/blob/39c6b60afc4c235f19113dc98087176748fa091d/src/sql/engine/local.rs#L349-L388\n\n## Row Storage and Transactions\n\nThe workhorse of the SQL storage engine is the `Transaction` trait, which provides\n[CRUD](https://en.wikipedia.org/wiki/Create,_read,_update_and_delete) operations (create, read,\nupdate, delete) on table rows and secondary index entries. For performance (especially with Raft),\nit operates on row batches rather than individual rows.\n\nhttps://github.com/erikgrinaker/toydb/blob/0839215770e31f1e693d5cccf20a68210deaaa3f/src/sql/engine/engine.rs#L31-L58\n\nThe `Local::Transaction` implementation is just a wrapper around an MVCC transaction, and the\ncommit/rollback methods just call straight through to it:\n\nhttps://github.com/erikgrinaker/toydb/blob/39c6b60afc4c235f19113dc98087176748fa091d/src/sql/engine/local.rs#L99-L102\n\nhttps://github.com/erikgrinaker/toydb/blob/39c6b60afc4c235f19113dc98087176748fa091d/src/sql/engine/local.rs#L182-L192\n\nTo insert new rows into a table, we first have to perform some validation: check that the table\nexists and validate the rows against the table schema (including checking for e.g. primary key\nconflicts and foreign key references). We then store the rows as a key/value pairs, using a\n`Key::Row` with the table name and primary key value. And finally, we update secondary index entries\n(if any).\n\nhttps://github.com/erikgrinaker/toydb/blob/39c6b60afc4c235f19113dc98087176748fa091d/src/sql/engine/local.rs#L252-L268\n\nRow updates are similar to inserts, but in the case of a primary key change we instead delete the\nold row and insert a new one, for simplicity. Secondary index updates also have to update both the\nold and new entries.\n\nhttps://github.com/erikgrinaker/toydb/blob/39c6b60afc4c235f19113dc98087176748fa091d/src/sql/engine/local.rs#L296-L337\n\nRow deletions are also similar: validate that the deletion is safe (e.g. check that there are no\nforeign key references to it), then delete the `Key::Row` keys and any secondary index entries:\n\nhttps://github.com/erikgrinaker/toydb/blob/39c6b60afc4c235f19113dc98087176748fa091d/src/sql/engine/local.rs#L194-L246\n\nTo fetch rows by primary key, we simply call through to key/value gets using the appropriate\n`Key::Row`:\n\nhttps://github.com/erikgrinaker/toydb/blob/39c6b60afc4c235f19113dc98087176748fa091d/src/sql/engine/local.rs#L248-L250\n\nhttps://github.com/erikgrinaker/toydb/blob/39c6b60afc4c235f19113dc98087176748fa091d/src/sql/engine/local.rs#L127-L133\n\nSimilarly, index lookups fetch a `Key::Index` for the indexed value, returning matching primary\nkeys:\n\nhttps://github.com/erikgrinaker/toydb/blob/39c6b60afc4c235f19113dc98087176748fa091d/src/sql/engine/local.rs#L270-L273\n\nhttps://github.com/erikgrinaker/toydb/blob/39c6b60afc4c235f19113dc98087176748fa091d/src/sql/engine/local.rs#L115-L125\n\nScanning table rows just performs a prefix scan with the appropriate `KeyPrefix::Row`, returning a\nrow iterator. This can optionally also do row filtering via filter pushdowns, which we'll revisit\nwhen we look at the SQL optimizer.\n\nhttps://github.com/erikgrinaker/toydb/blob/39c6b60afc4c235f19113dc98087176748fa091d/src/sql/engine/local.rs#L275-L294\n\nAnd with that, we can now store and retrieve SQL tables and rows on disk. Let's see how to replicate\nit across nodes via Raft.\n\n---\n\n

\n← SQL Data Model   |   SQL Raft Replication →\n

" }, { "path": "docs/architecture/sql.md", "content": "# SQL Engine\n\nThe SQL engine provides support for the SQL query language, and is the main database interface. It\nuses a key/value store for data storage, MVCC for transactions, and Raft for replication. The SQL\nengine itself consists of several distinct components that form a pipeline:\n\n> Client → Session → Lexer → Parser → Planner → Optimizer → Executor → Storage\n\nThe SQL engine is located in the [`sql`](https://github.com/erikgrinaker/toydb/tree/b2fe7b76ee634ca6ad31616becabfddb1c03d34b/src/sql)\nmodule. We'll discuss each of the components in a bottom-up manner.\n\nThe SQL engine is tested as a whole by test scripts under\n[`src/sql/testscripts`](https://github.com/erikgrinaker/toydb/tree/9419bcf6aededf0e20b4e7485e2a5fa3e975d79f/src/sql/testscripts).\nThese typically take a raw SQL string as input, execute them against an in-memory storage engine,\nand output the result along with intermediate state such as the query plan, storage operations,\nand binary key/value data.\n\n---\n\n

\n← Raft Consensus   |   SQL Data Model →\n

" }, { "path": "docs/architecture/storage.md", "content": "# Storage Engine\n\ntoyDB uses an embedded [key/value store](https://en.wikipedia.org/wiki/Key–value_database) for data\nstorage, located in the [`storage`](https://github.com/erikgrinaker/toydb/tree/213e5c02b09f1a3cac6a8bbd0a81773462f367f5/src/storage)\nmodule. This stores arbitrary keys and values as binary byte strings. The storage engine doesn't\nknow or care what the keys and values contain -- we'll see later how the SQL data model, with tables\nand rows, is mapped onto this key/value structure.\n\nThe storage engine supports simple set/get/delete operations on individual keys. It does not itself\nsupport transactions -- this is built on top, and we'll get back to it shortly.\n\nKeys are stored in sorted order. This allows range scans, where we can iterate over all key/value\npairs between two specific keys, or with a specific key prefix. This will be needed by other\ncomponents in the system, e.g. to scan all rows in a specific SQL table, to scan all versions of an\nMVCC key, to scan the tail of the Raft log, etc.\n\nThe storage engine is pluggable: there are multiple implementations, and the user can choose which\none to use in the config file. These implement the `storage::Engine` trait:\n\nhttps://github.com/erikgrinaker/toydb/blob/4804df254034c51f367d1380d389d80695cd7054/src/storage/engine.rs#L8-L58\n\nLet's look at the existing storage engine implementations.\n\n## `Memory` Storage Engine\n\nThe simplest storage engine is the `storage::Memory` engine. This is a trivial implementation which\nstores data in memory using the Rust standard library's\n[`BTreeMap`](https://doc.rust-lang.org/std/collections/struct.BTreeMap.html), without persisting\nit to disk. It is primarily used for testing.\n\nSince this is just a wrapper around the `BTreeMap` we can include it in its entirety here:\n\nhttps://github.com/erikgrinaker/toydb/blob/8f8eae0dcf70b1a0df2e853b1f6600e0c7075340/src/storage/memory.rs#L8-L77\n\n## `BitCask` Storage Engine\n\nThe main storage engine is `storage::BitCask`. This is a very simple variant of\n[BitCask](https://riak.com/assets/bitcask-intro.pdf), used in the [Riak](https://riak.com/)\ndatabase. It is kind of like the [LSM-tree](https://en.wikipedia.org/wiki/Log-structured_merge-tree)'s\nbaby cousin.\n\nhttps://github.com/erikgrinaker/toydb/blob/3e467512dca55843f0b071b3e239f14724f59a41/src/storage/bitcask.rs#L15-L55\n\ntoyDB's BitCask implementation uses a single append-only log file for storage. To write a key/value\npair, we simply append it to the file. To delete a key, we append a special tombstone value. When\nreading a key, the last entry for that key in the file is used.\n\nThe file format for a key/value pair is simply:\n\n1. The key length, as a big-endian `u32` (4 bytes).\n2. The value length, as a big-endian `i32` (4 bytes). -1 if tombstone.\n3. The binary key (n bytes).\n4. The binary value (n bytes).\n\nFor example, the key/value pair `foo=bar` would be written as follows (in hexadecimal):\n\n```\nkeylen valuelen key value\n00000003 00000003 666f6f 626172\n```\n\nBecause the data file is a simple log, we don't need a separate [write-ahead log](https://en.wikipedia.org/wiki/Write-ahead_logging)\nfor crash recovery -- the data file _is_ the write-ahead log.\n\nTo quickly look up key/value pairs when reading, we maintain an in-memory `KeyDir` index which maps\na key to the latest value's position in the file. All keys must therefore fit in memory.\n\nhttps://github.com/erikgrinaker/toydb/blob/3e467512dca55843f0b071b3e239f14724f59a41/src/storage/bitcask.rs#L57-L65\n\nWe initially generate this index by scanning through the entire file when it is opened:\n\nhttps://github.com/erikgrinaker/toydb/blob/3e467512dca55843f0b071b3e239f14724f59a41/src/storage/bitcask.rs#L267-L332\n\nTo write a key, we append it to the file and update the `KeyDir`:\n\nhttps://github.com/erikgrinaker/toydb/blob/3e467512dca55843f0b071b3e239f14724f59a41/src/storage/bitcask.rs#L155-L159\n\nhttps://github.com/erikgrinaker/toydb/blob/3e467512dca55843f0b071b3e239f14724f59a41/src/storage/bitcask.rs#L342-L366\n\nTo delete a key, we append a tombstone value instead:\n\nhttps://github.com/erikgrinaker/toydb/blob/3e467512dca55843f0b071b3e239f14724f59a41/src/storage/bitcask.rs#L122-L126\n\nTo read a value for a key, we look up the key's file location in the `KeyDir` index (if the key\nexists), and then read it from the file:\n\nhttps://github.com/erikgrinaker/toydb/blob/3e467512dca55843f0b071b3e239f14724f59a41/src/storage/bitcask.rs#L334-L340\n\nThe `KeyDir` uses an inner stdlib `BTreeMap` to keep track of keys. This allows range scans, where\nwe iterate over a sorted set of keys between the range bounds, loading each key from the file:\n\nhttps://github.com/erikgrinaker/toydb/blob/3e467512dca55843f0b071b3e239f14724f59a41/src/storage/bitcask.rs#L144-L146\n\nhttps://github.com/erikgrinaker/toydb/blob/3e467512dca55843f0b071b3e239f14724f59a41/src/storage/bitcask.rs#L207-L225\n\nAs keys are updated and deleted, we'll keep accumulating old versions in the log file. To remove\nthese, the log file is compacted on startup. This writes out the latest value of every live\nkey/value pair to a new file, and replaces the old file. The keys are written in sorted order, to\nmake later scans faster.\n\nhttps://github.com/erikgrinaker/toydb/blob/3e467512dca55843f0b071b3e239f14724f59a41/src/storage/bitcask.rs#L172-L195\n\n---\n\n

\n← Overview   |   Key/Value Encoding →\n

" }, { "path": "docs/architecture.md", "content": "Moved to [`architecture/index.md`](architecture/index.md)." }, { "path": "docs/crate/Cargo.toml", "content": "[package]\nname = \"toydb\"\nversion = \"1.0.1\"\ndescription = \"A simple distributed SQL database, built for education\"\nauthors = [\"Erik Grinaker \"]\nlicense = \"Apache-2.0\"\nhomepage = \"https://github.com/erikgrinaker/toydb\"\nrepository = \"https://github.com/erikgrinaker/toydb\"\nedition = \"2024\"\n" }, { "path": "docs/crate/README.md", "content": "# toyDB\n\ntoyDB is a distributed SQL database in Rust, built from scratch as an educational project. Main\nfeatures:\n\n* Raft distributed consensus for linearizable state machine replication.\n\n* ACID transactions with MVCC-based snapshot isolation.\n\n* Pluggable storage engine with BitCask and in-memory backends.\n\n* Iterator-based query engine with heuristic optimization and time-travel support.\n\n* SQL interface including joins, aggregates, and transactions.\n\ntoyDB is not distributed as a crate, see for more.\n\nThis crate used to contain the [joydb](https://crates.io/crates/joydb) database. Thanks to Serhii\nPotapov for donating the crate name." }, { "path": "docs/crate/src/lib.rs", "content": "//! This crate is just a simple README.md placeholder. toydb is not intended to be used as a\n//! library, and is not distributed as a crate. See .\n" }, { "path": "docs/examples.md", "content": "# SQL Examples\n\nThe following examples demonstrate some of toyDB's SQL features. For more details, see the\n[SQL reference](sql.md).\n\n- [Setup](#setup)\n- [Creating Tables and Data](#creating-tables-and-data)\n- [Constraints and Referential Integrity](#constraints-and-referential-integrity)\n- [Basic SQL Queries](#basic-sql-queries)\n- [Expressions](#expressions)\n- [Joins](#joins)\n- [Explain](#explain)\n- [Aggregates](#aggregates)\n- [Transactions](#transactions)\n- [Time-Travel Queries](#time-travel-queries)\n\n## Setup\n\nTo start a five-node cluster on the local machine (requires a working\n[Rust compiler](https://www.rust-lang.org/tools/install)), run:\n\n```\n$ ./cluster/run.sh\ntoydb2 19:06:28 [ INFO] Listening on 0.0.0.0:9602 (SQL) and 0.0.0.0:9702 (Raft)\ntoydb2 19:06:28 [ERROR] Failed connecting to Raft peer 127.0.0.1:9705: Connection refused\ntoydb5 19:06:28 [ INFO] Listening on 0.0.0.0:9605 (SQL) and 0.0.0.0:9705 (Raft)\n[...]\ntoydb5 19:06:29 [ INFO] Voting for toydb-d in term 1 election\ntoydb3 19:06:29 [ INFO] Voting for toydb-d in term 1 election\ntoydb4 19:06:29 [ INFO] Won election for term 1, becoming leader\n```\n\nIn a separate terminal, start a `toysql` client and check the server status:\n\n```\n$ cargo run --release --bin toysql\nConnected to toyDB node \"toydb-a\". Enter !help for instructions.\ntoydb> !status\n\nServer: 5 (leader 4 in term 1 with 5 nodes)\nRaft log: 1 committed, 0 applied, 0.000 MB (hybrid storage)\nNode logs: 1:1 2:1 3:1 4:1 5:1\nSQL txns: 0 active, 0 total (bitcask storage)\n```\n\nThe cluster is shut down by pressing Ctrl-C. Data is saved under `clusters/toydb-?/data/`,\ndelete the contents to start over.\n\n## Creating Tables and Data\n\nAs a basis for later examples, we'll create a small movie database. The following SQL statements\ncan be pasted into `toysql`:\n\n```sql\nCREATE TABLE genres (\n id INTEGER PRIMARY KEY,\n name STRING NOT NULL\n);\nINSERT INTO genres VALUES\n (1, 'Science Fiction'),\n (2, 'Action'),\n (3, 'Drama'),\n (4, 'Comedy');\n\nCREATE TABLE studios (\n id INTEGER PRIMARY KEY,\n name STRING NOT NULL\n);\nINSERT INTO studios VALUES\n (1, 'Mosfilm'),\n (2, 'Lionsgate'),\n (3, 'StudioCanal'),\n (4, 'Warner Bros'),\n (5, 'Focus Features');\n\nCREATE TABLE movies (\n id INTEGER PRIMARY KEY,\n title STRING NOT NULL,\n studio_id INTEGER NOT NULL INDEX REFERENCES studios,\n genre_id INTEGER NOT NULL INDEX REFERENCES genres,\n released INTEGER NOT NULL,\n rating FLOAT\n);\nINSERT INTO movies VALUES\n (1, 'Stalker', 1, 1, 1979, 8.2),\n (2, 'Sicario', 2, 2, 2015, 7.6),\n (3, 'Primer', 3, 1, 2004, 6.9),\n (4, 'Heat', 4, 2, 1995, 8.2),\n (5, 'The Fountain', 4, 1, 2006, 7.2),\n (6, 'Solaris', 1, 1, 1972, 8.1),\n (7, 'Gravity', 4, 1, 2013, 7.7),\n (8, '21 Grams', 5, 3, 2003, 7.7),\n (9, 'Birdman', 4, 4, 2014, 7.7),\n (10, 'Inception', 4, 1, 2010, 8.8),\n (11, 'Lost in Translation', 5, 4, 2003, 7.7),\n (12, 'Eternal Sunshine of the Spotless Mind', 5, 3, 2004, 8.3);\n```\n\ntoyDB supports some basic datatypes, as well as primary keys, foreign keys, and column indexes.\nFor more information on these, see the [SQL reference](sql.md). Schema changes such as\n`ALTER TABLE` are not supported, only `CREATE TABLE` and `DROP TABLE`.\n\nThe tables can be inspected via the `!tables` and `!table` commands:\n\n```sql\ntoydb> !tables\ngenres\nmovies\nstudios\n\ntoydb> !table genres\nCREATE TABLE genres (\n id INTEGER PRIMARY KEY,\n name STRING NOT NULL\n)\n```\n\n## Constraints and Referential Integrity\n\nSchemas enforce referential integrity and other constraints:\n\n```sql\ntoydb> DROP TABLE studios;\nError: Table studios is referenced by table movies column studio_id\n\ntoydb> DELETE FROM studios WHERE id = 1;\nError: Primary key 1 is referenced by table movies column studio_id\n\ntoydb> UPDATE movies SET id = 1;\nError: Primary key 1 already exists for table movies\n\ntoydb> INSERT INTO movies VALUES (13, 'Nebraska', 6, 3, 2013, 7.7);\nError: Referenced primary key 6 in table studios does not exist\n\ntoydb> INSERT INTO movies VALUES (13, 'Nebraska', NULL, 3, 2013, 7.7);\nError: NULL value not allowed for column studio_id\n\ntoydb> INSERT INTO movies VALUES (13, 'Nebraska', 'Unknown', 3, 2013, 7.7);\nError: Invalid datatype STRING for INTEGER column studio_id\n```\n\n## Basic SQL Queries\n\nMost basic SQL query functionality is supported:\n\n```sql\ntoydb> SELECT * FROM studios;\n1|Mosfilm\n2|Lionsgate\n3|StudioCanal\n4|Warner Bros\n5|Focus Features\n\ntoydb> SELECT title, rating FROM movies WHERE released >= 2000 ORDER BY rating DESC LIMIT 3;\nInception|8.8\nEternal Sunshine of the Spotless Mind|8.3\nGravity|7.7\n```\n\nColumn headers can be enabled with `!headers on`:\n\n```sql\ntoydb> !headers on\nHeaders enabled\n\ntoydb> SELECT id, name AS genre FROM genres;\nid|genre\n1|Science Fiction\n2|Action\n3|Drama\n4|Comedy\n```\n\n## Expressions\n\nAll common mathematical operators are implemented:\n\n```sql\ntoydb> SELECT 1 + 2 * 3;\n7\n\ntoydb> SELECT (1 + 2) * 4 / -3;\n-4\n\nSELECT 3! + 7 % 4 - 2 ^ 3;\n1\n```\n\n64-bit floating point arithmetic is also supported, including infinity and NaN:\n\n```sql\ntoydb> SELECT 3.14 * 2.718;\n8.53452\n\ntoydb> SELECT 1.0 / 0.0;\ninf\n\ntoydb> SELECT 1e10 ^ 8;\n100000000000000000000000000000000000000000000000000000000000000000000000000000000\n\ntoydb> SELECT 1e10 ^ 8 / INFINITY, 1e10 ^ 1e10, INFINITY / INFINITY;\n0|inf|NaN\n```\n\nAnd of course three-valued logic:\n\n```sql\ntoydb> SELECT TRUE AND TRUE, TRUE AND FALSE, TRUE AND NULL, FALSE AND NULL;\nTRUE|FALSE|NULL|FALSE\n\ntoydb> SELECT TRUE OR FALSE, FALSE OR FALSE, TRUE OR NULL, FALSE OR NULL;\nTRUE|FALSE|TRUE|NULL\n\ntoydb> SELECT NOT TRUE, NOT FALSE, NOT NULL;\nFALSE|TRUE|NULL\n```\n\nWhich would be useless without comparison operators for all types:\n\n```sql\ntoydb> SELECT 3 > 1, 3 <= 1, 3 = 3.0;\nTRUE|FALSE|TRUE\n\ntoydb> SELECT 'a' = 'A', 'foo' > 'bar', '👍' != '👎';\nFALSE|TRUE|TRUE\n\ntoydb> SELECT INFINITY > -INFINITY, NULL = NULL;\nTRUE|NULL\n```\n\n## Joins\n\nNo SQL database would be complete without joins, and toyDB supports most join types such as\ninner joins (both implicit and explicit):\n\n```sql\ntoydb> SELECT m.id, m.title, g.name FROM movies m JOIN genres g ON m.genre_id = g.id LIMIT 4;\n1|Stalker|Science Fiction\n2|Sicario|Action\n3|Primer|Science Fiction\n4|Heat|Action\n\ntoydb> SELECT m.id, m.title, g.name FROM movies m, genres g WHERE m.genre_id = g.id LIMIT 4;\n1|Stalker|Science Fiction\n2|Sicario|Action\n3|Primer|Science Fiction\n4|Heat|Action\n```\n\nLeft and right outer joins:\n\n```sql\ntoydb> SELECT s.id, s.name, g.name FROM studios s LEFT JOIN genres g ON s.id = g.id;\n1|Mosfilm|Science Fiction\n2|Lionsgate|Action\n3|StudioCanal|Drama\n4|Warner Bros|Comedy\n5|Focus Features|NULL\n\ntoydb> SELECT g.id, g.name, s.name FROM genres g RIGHT JOIN studios s ON g.id = s.id;\n1|Science Fiction|Mosfilm\n2|Action|Lionsgate\n3|Drama|StudioCanal\n4|Comedy|Warner Bros\nNULL|NULL|Focus Features\n```\n\nAnd cross joins (both implicit and explicit):\n\n```sql\ntoydb> SELECT g.name, s.name FROM genres g, studios s WHERE s.name < 'S';\nScience Fiction|Mosfilm\nScience Fiction|Lionsgate\nScience Fiction|Focus Features\nAction|Mosfilm\nAction|Lionsgate\nAction|Focus Features\nDrama|Mosfilm\nDrama|Lionsgate\nDrama|Focus Features\nComedy|Mosfilm\nComedy|Lionsgate\nComedy|Focus Features\n```\n\nWe can join on arbitrary predicates, such as joining movies with any genres whose name is\nordered after the movie's title:\n\n```sql\ntoydb> SELECT m.title, g.name\n FROM movies m JOIN genres g ON g.name > m.title\n ORDER BY m.title, g.name;\n\n21 Grams|Action\n21 Grams|Comedy\n21 Grams|Drama\n21 Grams|Science Fiction\nBirdman|Comedy\nBirdman|Drama\nBirdman|Science Fiction\nEternal Sunshine of the Spotless Mind|Science Fiction\nGravity|Science Fiction\nHeat|Science Fiction\nInception|Science Fiction\nLost in Translation|Science Fiction\nPrimer|Science Fiction\n```\n\nAnd we can join multiple tables, even using the same table multiple times - like in this example\nwhere we find all science fiction movies released since 2000 by studios that have released any \nmovie rated 8 or higher:\n\n```sql\ntoydb> SELECT m.id, m.title, g.name AS genre, m.released, s.name AS studio\n FROM movies m JOIN genres g ON m.genre_id = g.id,\n studios s JOIN movies good ON good.studio_id = s.id AND good.rating >= 8\n WHERE m.studio_id = s.id AND m.released >= 2000 AND g.id = 1\n ORDER BY m.title ASC;\n\n7|Gravity|Science Fiction|2013|Warner Bros\n10|Inception|Science Fiction|2010|Warner Bros\n5|The Fountain|Science Fiction|2006|Warner Bros\n```\n\n## Explain\n\nWhen optimizing complex queries with several joins, it can often be useful to inspect the query\nplan via an `EXPLAIN` query:\n\n```sql\ntoydb> EXPLAIN\n SELECT m.id, m.title, g.name AS genre, m.released, s.name AS studio\n FROM movies m JOIN genres g ON m.genre_id = g.id,\n studios s JOIN movies good ON good.studio_id = s.id AND good.rating >= 8\n WHERE m.studio_id = s.id AND m.released >= 2000 AND g.id = 1\n ORDER BY m.title ASC;\n\nOrder: m.title asc\n└─ Projection: m.id, m.title, g.name, m.released, s.name\n └─ HashJoin: inner on m.studio_id = s.id\n ├─ HashJoin: inner on m.genre_id = g.id\n │ ├─ Filter: m.released > 2000 OR m.released = 2000\n │ │ └─ IndexLookup: movies as m column genre_id (1)\n │ └─ KeyLookup: genres as g (1)\n └─ HashJoin: inner on s.id = good.studio_id\n ├─ Scan: studios as s\n └─ Scan: movies as good (good.rating > 8 OR good.rating = 8)\n```\n\nHere, we can see that the planner does a primary key lookup on `genres` and an index lookup on\n`movies.genre_id`, filtering the resulting movies by release year and joining them. It also\ndoes full table scans of `studios` and `movies` (to find the good movies) and joins them, pusing\nthe `rating >= 8` filter down to the `movies` table scan. The results of these two joins are also\njoined to produce the final result, which is then formatted and sorted.\n\n## Aggregates\n\nMost basic aggregate functions are supported:\n\n```sql\ntoydb> SELECT COUNT(*), MIN(rating), MAX(rating), AVG(rating), SUM(rating) FROM movies;\n12|6.9|8.8|7.841666666666668|94.10000000000001\n```\n\nWe can group by values and filter the aggregate results:\n\n```sql\ntoydb> SELECT s.id, s.name, AVG(m.rating) AS average\n FROM movies m JOIN studios s ON m.studio_id = s.id\n GROUP BY s.id, s.name\n HAVING average > 7.8\n ORDER BY average DESC, s.name ASC;\n1|Mosfilm|8.149999999999999\n4|Warner Bros|7.919999999999999\n5|Focus Features|7.900000000000001\n```\n\nAnd we can combine aggregate functions with arbitrary expressions, both inside and outside:\n\n```sql\ntoydb> SELECT s.id, s.name, ((MAX(rating^2) - MIN(rating^2)) / AVG(rating^2)) ^ (0.5) AS spread\n FROM movies m JOIN studios s ON m.studio_id = s.id\n GROUP BY s.id, s.name\n HAVING MAX(rating) - MIN(rating) > 0.5\n ORDER BY spread DESC;\n4|Warner Bros|0.6373540990222496\n5|Focus Features|0.39194971607693424\n```\n\n## Transactions\n\ntoyDB supports ACID transactions via MVCC-based snapshot isolation. This provides atomic\ntransactions with good isolation, without taking out locks or blocking reads on writes. As a basic\nexample, the below transaction is rolled back without taking effect, as opposed to `COMMIT`\nwhich would make it permanent:\n\n```sql\ntoydb> BEGIN;\nBegan transaction 131\n\ntoydb:131> INSERT INTO genres VALUES (5, 'Western');\ntoydb:131> SELECT * FROM genres;\n1|Science Fiction\n2|Action\n3|Drama\n4|Comedy\n5|Western\ntoydb:131> ROLLBACK;\nRolled back transaction 131\n\ntoydb> SELECT * FROM genres;\n1|Science Fiction\n2|Action\n3|Drama\n4|Comedy\n```\n\nWe'll demonstrate transactions by covering most common transaction anomalies given two\nconcurrent sessions, and show how toyDB prevents these anomalies in all cases but one. In these\nexamples, the left half is user A and the right is user B. Time flows downwards such that\ncommands on the same line happen at the same time.\n\n**Dirty write:** an uncommitted write by A should not be affected by a concurrent B write.\n\n```sql\na> BEGIN;\na> INSERT INTO genres VALUES (5, 'Western');\n b> INSERT INTO genres VALUES (5, 'Romance');\n Error: Serialization failure, retry transaction\na> SELECT * FROM genres WHERE id = 5;\n5|Western\n```\n\nThe serialization failure here occurs because the first write always wins. This may not be an\noptimal strategy, but it is correct in terms of preventing serialization anomalies.\n\n**Dirty read:** an uncommitted write by A should not be visible to B until committed.\n\n```sql\na> BEGIN;\na> INSERT INTO genres VALUES (5, 'Western');\n b> SELECT * FROM genres WHERE id = 5;\n No rows returned\na> COMMIT;\n b> SELECT * FROM genres WHERE id = 5;\n 5|Western\n```\n\n**Lost update:** when A and B both read a value, before updating it in turn, the first write should\nnot be overwritten by the second.\n\n```sql\na> BEGIN; b> BEGIN;\na> SELECT title, rating FROM movies WHERE id = 2; b> SELECT title, rating FROM movies WHERE id = 2;\nSicario|7.6 Sicario|7.6\na> UPDATE movies SET rating = 7.8 WHERE id = 2;\n b> UPDATE movies SET rating = 7.7 WHERE id = 2;\n Error: Serialization failure, retry transaction\na> COMMIT;\n```\n\n**Fuzzy read:** B should not see a value suddenly change in its transaction, even if A commits a \nnew value.\n\n```sql\na> BEGIN; b> BEGIN;\n b> SELECT * FROM genres WHERE id = 1;\n 1|Science Fiction\na> UPDATE genres SET name = 'Scifi' WHERE id = 1;\na> COMMIT;\n b> SELECT * FROM genres WHERE id = 1;\n 1|Science Fiction\n b> COMMIT;\n\n b> SELECT * FROM genres WHERE id = 1;\n 1|Scifi\n```\n\n**Read skew:** if A reads two values, and B modifies the second value in between the reads, A \nshould see the old second value.\n\n```sql\na> BEGIN;\na> SELECT * FROM genres WHERE id = 2;\n2|Action\n b> BEGIN;\n b> UPDATE genres SET name = 'Drama' WHERE id = 2;\n b> UPDATE genres SET name = 'Action' WHERE id = 3;\n b> COMMIT;\na> SELECT * FROM genres WHERE id = 3;\n3|Drama\n```\n\n**Phantom read:** when A runs a query with a predicate, and B commits a matching write, A should\nnot see the write when rerunning it.\n\n```sql\na> BEGIN;\na> SELECT * FROM genres WHERE id > 2;\n3|Drama\n4|Comedy\n b> INSERT INTO genres VALUES (5, 'Western');\na> SELECT * FROM genres WHERE id > 2;\n3|Drama\n4|Comedy\n```\n\n**Write skew:** when A reads row X and writes it to row Y, B should not concurrently be able to\nread row Y and write it to row X.\n\n```sql\na> BEGIN; b> BEGIN;\na> SELECT * FROM genres WHERE id = 2;\n2|Action\n b> SELECT * FROM genres WHERE id = 3;\n 3|Drama\n b> UPDATE genres SET name = 'Drama' WHERE id = 2;\na> UPDATE genres SET name = 'Action' WHERE id = 3;\na> COMMIT; b> COMMIT;\n```\n\nHere, the writes actually go through. This anomaly is not protected against by snapshot isolation, \nand thus not by toyDB either - doing so would require implementing serializable snapshot isolation. \nHowever, this is the only common serialization anomaly not handled by toyDB, and is not among the\nmost severe.\n\n## Time-Travel Queries\n\nSince toyDB uses MVCC for transactions and keeps all historical versions, the state of the database\ncan be queried at any arbitrary point in the past. toyDB uses incremental transaction IDs as\nlogical timestamps:\n\n```sql\ntoydb> SELECT * FROM genres;\n1|Science Fiction\n2|Drama\n3|Action\n4|Comedy\n\ntoydb> BEGIN;\nBegan transaction 173\ntoydb:173> UPDATE genres SET name = 'Scifi' WHERE id = 1;\ntoydb:173> INSERT INTO genres VALUES (5, 'Western');\ntoydb:173> COMMIT;\nCommitted transaction 173\n\ntoydb> SELECT * FROM genres;\n1|Scifi\n2|Drama\n3|Action\n4|Comedy\n5|Western\n\ntoydb> BEGIN READ ONLY AS OF SYSTEM TIME 172;\nBegan read-only transaction 175 in snapshot at version 172\ntoydb@172> SELECT * FROM genres;\n1|Science Fiction\n2|Drama\n3|Action\n4|Comedy\n```" }, { "path": "docs/references.md", "content": "# References\n\nThis is the main research material I used while building toyDB. It is a subset of my\n[reading list](https://github.com/erikgrinaker/readings).\n\n## Introduction\n\nAndy Pavlo's CMU lectures are an absolutely fantastic introduction to database internals:\n\n- 🎥 [CMU 15-445 Intro to Database Systems](https://www.youtube.com/playlist?list=PLSE8ODhjZXjbohkNBWQs_otTrBTrjyohi) (A Pavlo 2019)\n- 🎥 [CMU 15-721 Advanced Database Systems](https://www.youtube.com/playlist?list=PLSE8ODhjZXjasmrEd2_Yi1deeE360zv5O) (A Pavlo 2020)\n\nMartin Kleppman has written an excellent overview of database technologies and concepts, while Alex\nPetrov goes in depth on implementation of storage engines and distributed systems algorithms:\n\n- 📖 [Designing Data-Intensive Applications](https://dataintensive.net/) (M Kleppmann 2017)\n- 📖 [Database Internals](https://www.databass.dev) (A Petrov 2019)\n\n## Raft\n\nThe Raft consensus algorithm is described in a very readable paper by Diego Ongaro, and in a talk\ngiven by his advisor John Ousterhout:\n\n- 📄 [In Search of an Understandable Consensus Algorithm](https://raft.github.io/raft.pdf) (D Ongaro, J Ousterhout 2014)\n- 🎥 [Designing for Understandability: The Raft Consensus Algorithm](https://www.youtube.com/watch?v=vYp4LYbnnW8) (J Ousterhout 2016)\n\nHowever, Raft has several subtle pitfalls, and Jon Gjengset's student guide was very helpful in\ndrawing attention to these:\n\n- 🔗 [Students' Guide to Raft](https://thesquareplanet.com/blog/students-guide-to-raft/) (J Gjengset 2016)\n\n## Parsing\n\nThorsten Ball has written a very enjoyable hands-on introduction to parsers where he implements\nfirst an interpreter and then a compiler for the made-up Monkey programming language (in Go):\n\n- 📖 [Writing An Interpreter In Go](https://interpreterbook.com) (T Ball 2016) \n- 📖 [Writing A Compiler In Go](https://compilerbook.com) (T Ball 2018)\n\nThe toyDB expression parser is inspired by a blog post by Eli Bendersky describing the precedence\nclimbing algorithm, which is the algorithm I found the most elegant:\n\n- 💬 [Parsing Expressions by Precedence Climbing](https://eli.thegreenplace.net/2012/08/02/parsing-expressions-by-precedence-climbing) (E Bendersky 2012)\n\n## Transactions\n\nJepsen (i.e. Kyle Kingsbury) has an excellent overview of consistency and isolation models, which \nis very helpful in making sense of the jungle of overlapping and ill-defined terms:\n\n- 🔗 [Consistency Models](https://jepsen.io/consistency) (Jepsen 2016)\n\nFor more background on this, in particular on how snapshot isolation provided by the MVCC\ntransaction engine used in toyDB does not fit into the traditional SQL isolation levels, the\nfollowing classic papers were useful:\n\n- 📄 [A Critique of ANSI SQL Isolation Levels](https://www.microsoft.com/en-us/research/wp-content/uploads/2016/02/tr-95-51.pdf) (H Berenson et al 1995)\n- 📄 [Generalized Isolation Level Definitions](http://pmg.csail.mit.edu/papers/icde00.pdf) (A Adya, B Liskov, P ONeil 2000)\n\nAs for actually implementing MVCC, I found blog posts to be the most helpful:\n\n- 💬 [Implementing Your Own Transactions with MVCC](https://levelup.gitconnected.com/implementing-your-own-transactions-with-mvcc-bba11cab8e70) (E Chance 2015)\n- 💬 [How Postgres Makes Transactions Atomic](https://brandur.org/postgres-atomicity) (B Leach 2017)\n" }, { "path": "docs/sql.md", "content": "# SQL Reference\n\n## Data Types\n\nThe following data types are supported:\n\n* `BOOLEAN` (`BOOL`): logical truth values, i.e. true and false.\n* `FLOAT` (`DOUBLE`): 64-bit signed floating point numbers, using [IEEE 754 `binary64`](https://en.wikipedia.org/wiki/binary64) encoding. Supports magnitudes of 10⁻³⁰⁷ to 10³⁰⁸ with 53-bit precision (~15 significant figures), as well as the special values infinity and NaN.\n* `INTEGER` (`INT`): 64-bit signed integer numbers with a range of ±2⁶³-1.\n* `STRING` (`TEXT`, `VARCHAR`): UTF-8 encoded strings.\n\nIn addition, the special `NULL` value is used for an unknown value, following the rules of [three-valued logic](https://en.wikipedia.org/wiki/Three-valued_logic).\n\nNumeric types are not interchangable; a float value (even without a fractional part) cannot be stored in an integer column and vice-versa.\n\n## SQL Syntax\n\n### Keywords\n\nKeywords are reserved words with special meaning in SQL statements. They are case-insensitive, and must be quoted with `\"` to be used as identifiers. The complete list is:\n\n`AS`, `ASC`, `AND`, `BEGIN`, `BOOL`, `BOOLEAN`, `BY`, `COMMIT`, `CREATE`, `CROSS`, `DEFAULT`,`DELETE`, `DESC`, `DOUBLE`, `DROP`, `EXISTS`, `EXPLAIN`, `FALSE`, `FLOAT`, `FROM`, `GROUP`, `HAVING`, `IF`, `INDEX`, `INFINITY`, `INNER`, `INSERT`, `INT`, `INTEGER`, `INTO`, `IS`, `JOIN`, `KEY`, `LEFT`, `LIKE`, `LIMIT`, `NAN`, `NOT`, `NULL`, `OF`, `OFFSET`, `ON`, `ONLY`, `OR`, `ORDER`, `OUTER`, `PRIMARY`, `READ`, `REFERENCES`, `RIGHT`, `ROLLBACK`, `SELECT`, `SET`, `STRING`, `SYSTEM`, `TABLE`, `TEXT`, `TIME`, `TRANSACTION`, `TRUE`, `UNIQUE`, `UPDATE`, `VALUES`, `VARCHAR`, `WHERE`, `WRITE`\n\n### Identifiers\n\nIdentifiers are names for database objects such as tables and columns. Unless quoted with `\"`, they must begin with a Unicode letter followed by any combination of letters, numbers, and `_`, and cannot be reserved keywords. `\"\"` can be used to escape a double quote character. They are always converted to lowercase.\n\n### Constants\n\n#### Named constants\n\nThe following keywords evaluate to constants:\n\n* `FALSE`: the boolean false value.\n* `INFINITY`: the floating-point value for infinity.\n* `NAN`: the floating-point value for NaN (not a number).\n* `NULL`: an unknown value.\n* `TRUE`: the boolean true value.\n\n#### String literals\n\nString literals are surrounded by single quotes `'`, and can contain any valid UTF-8 character. Single quotes must be escaped by an additional single quote, i.e. `''`, no other escape sequences are supported. For example:\n\n```\n'A string with ''quotes'' and emojis 😀'\n```\n\n#### Numeric literals\n\nSequences of digits `0-9` are parsed as a 64-bit signed integer. Numbers with decimal points or in scientific notation are parsed as 64-bit floating point numbers. The following pattern is supported:\n\n```\n999[.[999]][e[+-]999]\n```\n\nThe `-` prefix operator can be used to take negative numbers.\n\n### Expressions\n\nExpressions can be used wherever a value is expected, e.g. as `SELECT` columns nd `INSERT` values. They are made up of constants, a column references, an operator invocations, and a function calls.\n\nColumn references can either be unqualified, e.g. `name`, or prefixed with the relation identifier separated by `.`, e.g. `person.name`. Unqualified identifiers must be unambiguous.\n\n## SQL Operators\n\n### Logical operators\n\nLogical operators apply standard logic operations on boolean operands.\n\n* `AND`: the logical conjunction, e.g. `TRUE AND TRUE` yields `TRUE`.\n* `OR`: the logical disjunction, e.g. `TRUE OR FALSE` yields `TRUE`.\n* `NOT`: the logical negation, e.g. `NOT TRUE` yields `FALSE`.\n\nThe complete truth tables are:\n\n| `AND` | `TRUE` | `FALSE` | `NULL` |\n|-------------|---------|---------|---------|\n| **`TRUE`** | `TRUE` | `FALSE` | `NULL` |\n| **`FALSE`** | `FALSE` | `FALSE` | `FALSE` |\n| **`NULL`** | `NULL` | `FALSE` | `NULL` |\n\n| `OR` | `TRUE` | `FALSE` | `NULL` |\n|-------------|--------|---------|--------|\n| **`TRUE`** | `TRUE` | `TRUE` | `TRUE` |\n| **`FALSE`** | `TRUE` | `FALSE` | `NULL` |\n| **`NULL`** | `TRUE` | `NULL` | `NULL` |\n\n| `NOT` | |\n|-------------|---------|\n| **`TRUE`** | `FALSE` |\n| **`FALSE`** | `TRUE` |\n| **`NULL`** | `NULL` |\n\n### Comparison operators\n\nComparison operators compare values of the same data type, and return `TRUE` if the comparison holds or `FALSE` otherwise. `INTEGER` and `FLOAT` values are interchangeable. `STRING` comparisons use the string's byte values, i.e. case-sensitive with `'B' < 'a'` due to their UTF-8 code points. `FALSE` is considered lesser than `TRUE`. Comparison with `NULL` always yields `NULL` (even `NULL = NULL`).\n\nBinary operators:\n\n* `=`: equality, e.g. `1 = 1` yields `TRUE`.\n* `!=`: inequality, e.g. `1 != 2` yields `TRUE`.\n* `>`: greater than, e.g. `2 > 1` yields `TRUE`.\n* `>=`: greater than or equal, e.g. `1 >= 1` yields `TRUE`.\n* `<`: lesser than, e.g. `1 < 2` yields `TRUE`.\n* `<=`: lesser than or equal, e.g. `1 <= 1` yields `TRUE`.\n\nUnary operators:\n\n* `IS NULL`: checks if the value is `NULL`, e.g. `NULL IS NULL` yields `TRUE`.\n* `IS NOT NULL`: checks if the value is not `NULL`, e.g. `TRUE IS NOT NULL` yields `TRUE`.\n* `IS NAN`: checks if the value is a float `NAN`, e.g. `NAN IS NAN` yields `TRUE`. Errors on \n non-float datatypes, except `NULL` which yields `NULL`.\n* `IS NOT NAN`: checks if the value is not a float `NAN`, e.g. `3.14 IS NOT NAN` yields `TRUE`.\n\n### Mathematical operators\n\nMathematical operators apply standard math operations on numeric (`INTEGER` or `FLOAT`) operands. If either operand is a `FLOAT`, both operands are converted to `FLOAT` and the result is a `FLOAT`. If either operand is `NULL`, the result is `NULL`. The special values `INFINITY` and `NAN` are handled according to the IEEE 754 spec.\n\nFor `INTEGER` operands, failure conditions such as overflow and division by zero yield an error. For `FLOAT` operands, these return `INFINITY` or `NAN` as appropriate.\n\nBinary operators:\n\n* `+`: addition, e.g. `1 + 2` yields `3`.\n* `-`: subtraction, e.g. `3 - 2` yields `1`.\n* `*`: multiplication, e.g. `3 * 2` yields `6`.\n* `/`: division, e.g. `6 / 2` yields `3`.\n* `^`: exponentiation, e.g. `2 ^ 4` yields `16`.\n* `%`: remainder, e.g. `8 % 3` yields `2`. Unlike modulo, the result has the sign of the dividend.\n\nUnary operators:\n\n* `+` (prefix): identity, e.g. `+1` yields `1`.\n* `-` (prefix): negation, e.g. `- -2` yields `2`.\n* `!` (postfix): factorial, e.g. `5!` yields `15`.\n\n### String operators\n\nString operators operate on string operands.\n\n* `LIKE`: compares a string with the given pattern, using `%` as multi-character wildcard and `_` as single-character wildcard, returning `TRUE` if the string matches the pattern - e.g. `'abc' LIKE 'a%'` yields `TRUE`.\n\n### Operator precedence\n\nThe operator precedence (order of operations) is as follows:\n\n| Precedence | Operator | Associativity |\n|------------|-------------------------|---------------|\n| 10 | `+`, `-` (prefix) | Right |\n| 9 | `!` (postfix) | Left |\n| 8 | `^` | Right |\n| 7 | `*`, `/`, `%` | Left |\n| 6 | `+`, `-` | Left |\n| 5 | `>`, `>=`, `<`, `<=` | Left |\n| 4 | `=`, `!=`, `LIKE`, `IS` | Left |\n| 3 | `NOT` | Right |\n| 2 | `AND` | Left |\n| 1 | `OR` | Left |\n\nPrecedence can be overridden by wrapping an expression in parentheses, e.g. `(1 + 2) * 3`.\n\n### Functions\n\n* `sqrt(expr)`: returns the square root of a numerical argument.\n\n### Aggregate functions\n\nAggregate function aggregate an expression across all rows, optionally grouped into buckets given by `GROUP BY`, and results can be filtered via `HAVING`.\n\n* `AVG(expr)`: returns the average of numerical values.\n\n* `COUNT(expr)`: returns the number of rows for which ***`expr`*** evaluates to a non-`NULL` value. `COUNT(*)` can be used to count all rows.\n\n* `MAX(expr)`: returns the maximum value, according to the datatype's ordering.\n\n* `MIN(expr)`: returns the minimum value, according to the datatype's ordering.\n\n* `SUM(expr)`: returns the sum of numerical values.\n\n## SQL Statements\n\n### `BEGIN`\n\nStarts a new [transaction](#transactions).\n\n
\nBEGIN [ TRANSACTION ] [ READ ONLY | READ WRITE ] [ AS OF SYSTEM TIME txn_id ]\n
\n\n* ***`txn_id`***: A past transaction ID to run a read-only transaction for, for time-travel queries.\n\n### `COMMIT`\n\nCommits an active [transaction](#transactions).\n\n### `CREATE TABLE`\n\nCreates a new table.\n\n
\nCREATE TABLE table_name (\n    [ column_name data_type [ column_constraint [ ... ] ]  [ INDEX ] [, ... ] ]\n)\n\nwhere column_constraint is:\n\n{ NOT NULL | NULL | PRIMARY KEY | DEFAULT expr | REFERENCES ref_table | UNIQUE }\n
\n\n* ***`table_name`***: The name of the table. Must be a [valid identifier](#identifiers). Errors if a table with this name already exists.\n\n* ***`column_name`***: The name of the column. Must be a [valid identifier](#identifiers), and unique within the table.\n\n* ***`data_type`***: The data type of the column, see [data types](#data-types) for valid types.\n\n* `NOT NULL`: The column may not contain `NULL` values.\n\n* `NULL`: The column may contain `NULL` values. This is the default.\n\n* `PRIMARY KEY`: The column should act as a primary key, i.e. the main row identifier. A table must have exactly one primary key column, and it must be unique and non-nullable.\n\n* `DEFAULT`***`expr`***: Specifies a default value for the column when `INSERT` statements do not give a value. ***`expr`*** can be any constant expression of an appropriate data type, e.g. `'abc'` or `1 + 2 * 3`. For nullable columns, the default value is `NULL` unless specified otherwise.\n\n* `REFERENCES`***`ref_table`***: The column is a foreign key to ***`ref_table`***'s primary key, enforcing referential integrity.\n\n* `UNIQUE`: The column may only contain unique (distinct) values. `NULL` values are not considered equal, thus a `UNIQUE` column which allows `NULL` may contain multiple `NULL` values. `PRIMARY KEY` columns are implicitly `UNIQUE`.\n\n* `INDEX`: Create an index for the column.\n\n#### Example\n\n```sql\nCREATE TABLE movie (\n id INTEGER PRIMARY KEY,\n title STRING NOT NULL,\n release_year INTEGER INDEX,\n imdb_id STRING INDEX UNIQUE,\n bluray BOOLEAN NOT NULL DEFAULT TRUE\n)\n```\n\n### `DELETE`\n\nDeletes rows in a table.\n\n
\nDELETE FROM table_name\n    [ WHERE predicate ]\n
\n\nDeletes rows where ***`predicate`*** evaluates to `TRUE`, or all rows if no `WHERE` clause is given.\n\n* ***`table_name`***: the table to delete from. Errors if it does not exist.\n\n* ***`predicate`***: an expression which determines which rows to delete by evaluting to `TRUE`. Must evaluate to a `BOOLEAN` or `NULL`, otherwise an error is returned.\n\n#### Example\n\n```sql\nDELETE FROM movie\nWHERE release_year < 2000 AND bluray = FALSE\n```\n\n### `DROP TABLE`\n\nDeletes a table and all contained data. Errors if the table does not\nexist, unless `IF EXISTS` is given.\n\n
\nDROP TABLE [ IF EXISTS ] table_name\n
\n\n* ***`table_name`***: the table to delete.\n\n### `EXPLAIN`\n\nOutputs the execution plan for the given statement.\n\n
\nEXPLAIN [ statement ]\n
\n\n### `INSERT`\n\nInserts rows into a table.\n\n
\nINSERT INTO table_name\n    [ ( column_name [, ... ] ) ]\n    VALUES ( expression [, ... ] ) [, ... ]\n
\n\nIf column names are given, an identical number of values must be given. If no column names are given, values must be given in the table's column order. Omitted columns will get a default value if specified, otherwise an error will be returned.\n\n* ***`table_name`***: the table to insert into. Errors if it does not exist.\n\n* ***`column_name`***: a column to insert into in the given table. Errors if it does not exist.\n\n* ***`expression`***: an expression to insert into the corresponding column. Must be a constant expression, i.e. it cannot refer to table columns.\n\n#### Example\n\n```sql\nINSERT INTO movie\n (id, title, release_year)\nVALUES\n (1, 'Sicario', 2015),\n (2, 'Stalker', 1979),\n (3, 'Her', 2013)\n```\n\n### `ROLLBACK`\n\nRolls back an active [transaction](#transactions).\n\n### `SELECT`\n\nSelects rows from a table.\n\n
\nSELECT [ * | expression [ [ AS ] output_name [, ...] ] ]\n    [ FROM from_item [, ...] ]\n    [ WHERE predicate ]\n    [ GROUP BY group_expr [, ...] ]\n    [ HAVING having_expr ]\n    [ ORDER BY order_expr [ ASC | DESC ] [, ...] ]\n    [ LIMIT count ]\n    [ OFFSET start ]\n\nwhere from_item is one of:\n\ntable_name [ [ AS ] alias ]\nfrom_item join_type from_item [ ON join_predicate ]\n\nwhere join_type is one of:\n\nCROSS JOIN\n[ INNER ] JOIN\nLEFT [ OUTER ] JOIN\nRIGHT [ OUTER ] JOIN\n\n
\n\nFetches rows or expressions, either from table ***`table_name`*** (if given) or generated.\n\n* ***`expression`***: [expression](#expressions) to fetch (can be a simple column name).\n\n* ***`output_name`***: output column [identifier](#identifier), defaults to column name (if single column) otherwise nothing (displayed as `?`).\n\n* ***`table_name`***: table to fetch rows from.\n\n* ***`alias`***: table alias.\n\n* ***`predicate`***: only return rows for which this [expression](#expressions) evaluates to `TRUE`.\n\n* ***`group_expr`***: an expression to group aggregates by. Non-aggregate `SELECT` expressions must either reference a column given in `group_expr`, be idential with a `group_expr`, or have an `output_name` that is referenced by a `group_expr` column.\n\n* ***`having_expr`***: only return aggregate results for which this [expression](#expressions) evaluates to `TRUE`.\n\n* ***`order_expr`***: order rows by this expression (can be a simple column name).\n\n* ***`count`***: maximum number of rows to return. Must be a constant integer expression.\n\n* ***`start`***: number of rows to skip. Must be a constant integer expression.\n\n* ***`join_predicate`***: only return rows for which this [expression](#expressions) evaluates to `TRUE`.\n\nJoin types:\n\n* `CROSS JOIN`: returns the Carthesian product of the joined tables. Does not accept a join predicate (`ON` clause).\n\n* `INNER JOIN`: returns the rows of the tables' Carthesian product for which ***`join_predicate`*** evaluates to `TRUE`.\n\n* `LEFT OUTER JOIN`: returns the rows joined on the ***`join_predicate`***, or for any rows in the left table that does not have a match in the right table a single row is returned with the right table's columns set to `NULL`.\n\n* `RIGHT OUTER JOIN`: the same as a `LEFT OUTER JOIN` but with the left and right tables switched.\n\n#### Example\n\n```sql\nSELECT id, title, 2020 - released AS age\nFROM movies\nWHERE released >= 2000 AND ultrahd\nORDER BY released DESC, title ASC\nLIMIT 10\nOFFSET 10\n```\n\n### `UPDATE`\n\nUpdates rows in a table.\n\n
\nUPDATE table_name\n    SET column_name = expression | DEFAULT [, ... ]\n    [ WHERE predicate ]\n
\n\nUpdates columns given by ***`column_name`*** to the corresponding ***`expression`*** for all rows where ***`predicate`*** evaluates to `TRUE`. If no `WHERE` clause is given, all rows are updated.\n\n* ***`table_name`***: the table to update. Errors if it does not exist.\n\n* ***`column_name`***: a column to update. Errors if it does not exist.\n\n* ***`expression`***: an expression whose evaluated value will be set for the corresponding column and row. Expressions can refer to column values, and must evaluate to the same datatype as the updated column. Using `DEFAULT` will set the column's default value, if any.\n\n* ***`predicate`***: an expression which determines which rows to update by evaluting to `TRUE`. Must evaluate to a `BOOLEAN` or `NULL`, otherwise an error is returned.\n\n#### Example\n\n```sql\nUPDATE movie\nSET bluray = TRUE\nWHERE release_year >= 2000 AND bluray = FALSE\n```\n\n## Transactions\n\ntoyDB supports ACID transactions using MVCC-based snapshot isolation, protecting from the following anomalies: dirty writes, dirty reads, lost updates, fuzzy reads, read skew, and phantom reads. However, write skew anomalies are possible since serializable snapshot isolation is not implemented.\n\nA new transaction is started with `BEGIN`, and ended with either `COMMIT` (atomically writing all changes) or `ROLLBACK` (discarding all changes). If any conflicts occur between concurrent transactions, the lowest transaction ID wins and the others will fail with a serialization error and must retry.\n\nAll past data is versioned and retained, and can be queried as of a given transaction ID via `BEGIN TRANSACTION READ ONLY AS OF SYSTEM TIME `.\n\nA transaction is still valid for use if a contained statement returns an error. It is up to the client to take appropriate action.\n" }, { "path": "docs/tools/update-links.py", "content": "#!/usr/bin/env python3\n#\n# Updates GitHub code links to the latest commit SHA.\n\nimport os, re, sys, argparse\nimport requests\n\nGITHUB_API = \"https://api.github.com\"\n\ndef get_latest_sha(owner, repo, path, token):\n url = f\"{GITHUB_API}/repos/{owner}/{repo}/commits\"\n headers = {}\n if token:\n headers[\"Authorization\"] = f\"token {token}\"\n params = {\"path\": path, \"sha\": \"main\", \"per_page\": 1}\n resp = requests.get(url, headers=headers, params=params)\n resp.raise_for_status()\n data = resp.json()\n return data[0][\"sha\"] if data else None\n\ndef process_markdown(text, token):\n pattern = re.compile(\n r\"https://github\\.com/(?P[^/]+)/(?P[^/]+)/blob/\"\n r\"(?P[0-9a-f]{7,40})/(?P[^#)\\s]+)\"\n )\n cache = {}\n def replacer(m):\n print(f\"Checking {m.group(0)}\")\n owner, repo, oldsha, path = m.group(\"owner\",\"repo\",\"oldsha\",\"path\")\n key = (owner, repo, path)\n print(f\"Key: {key}\")\n if key not in cache:\n cache[key] = get_latest_sha(owner, repo, path, token)\n newsha = cache[key]\n if newsha and newsha != oldsha:\n print(f\"Updating {m.group(0)} to {newsha}\")\n return m.group(0).replace(oldsha, newsha)\n return m.group(0)\n return pattern.sub(replacer, text)\n\ndef main():\n p = argparse.ArgumentParser(description=\"Update GitHub blob links to latest SHAs\")\n p.add_argument(\"file\", nargs=\"?\", help=\"Markdown file to update (defaults to stdin/stdout)\")\n args = p.parse_args()\n token = os.getenv(\"GITHUB_TOKEN\")\n if args.file:\n text = open(args.file, encoding=\"utf-8\").read()\n updated = process_markdown(text, token)\n with open(args.file, \"w\", encoding=\"utf-8\") as f:\n f.write(updated)\n else:\n text = sys.stdin.read()\n sys.stdout.write(process_markdown(text, token))\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "rust-toolchain", "content": "1.93.1" }, { "path": "rustfmt.toml", "content": "use_small_heuristics = \"Max\"\n" }, { "path": "src/bin/toydb.rs", "content": "//! The toyDB server. Takes configuration from a config file (default\n//! config/toydb.yaml) or corresponding TOYDB_ environment variables. Listens\n//! for SQL clients (default port 9601) and Raft connections from other toyDB\n//! peers (default port 9701). The Raft log and SQL database are stored at\n//! data/raft and data/sql by default.\n//!\n//! Use the toysql command-line client to connect to the server.\n\n#![warn(clippy::all)]\n\nuse std::collections::HashMap;\nuse std::path::Path;\n\nuse clap::Parser as _;\nuse serde::Deserialize;\n\nuse toydb::Server;\nuse toydb::errinput;\nuse toydb::error::Result;\nuse toydb::raft;\nuse toydb::sql;\nuse toydb::storage;\n\nfn main() {\n if let Err(error) = Command::parse().run() {\n eprintln!(\"Error: {error}\")\n }\n}\n\n/// The toyDB server configuration. Can be provided via config file (default\n/// config/toydb.yaml) or TOYDB_ environment variables.\n#[derive(Debug, Deserialize)]\nstruct Config {\n /// The node ID. Must be unique in the cluster.\n id: raft::NodeID,\n /// The other nodes in the cluster, and their Raft TCP addresses.\n peers: HashMap,\n /// The Raft listen address.\n listen_raft: String,\n /// The SQL listen address.\n listen_sql: String,\n /// The log level.\n log_level: String,\n /// The path to this node's data directory. The Raft log is stored in\n /// the file \"raft\", and the SQL state machine in \"sql\".\n data_dir: String,\n /// The Raft storage engine: bitcask or memory.\n storage_raft: String,\n /// The SQL storage engine: bitcask or memory.\n storage_sql: String,\n /// If false, don't fsync Raft log writes to disk. Disabling this\n /// will yield much better write performance, but may lose data on\n /// host crashes which compromises Raft safety guarantees.\n fsync: bool,\n /// The garbage fraction threshold at which to trigger compaction.\n compact_threshold: f64,\n /// The minimum bytes of garbage before triggering compaction.\n compact_min_bytes: u64,\n}\n\nimpl Config {\n /// Loads the configuration from the given file.\n fn load(file: &str) -> Result {\n Ok(config::Config::builder()\n .set_default(\"id\", \"1\")?\n .set_default(\"listen_sql\", \"localhost:9601\")?\n .set_default(\"listen_raft\", \"localhost:9701\")?\n .set_default(\"log_level\", \"info\")?\n .set_default(\"data_dir\", \"data\")?\n .set_default(\"storage_raft\", \"bitcask\")?\n .set_default(\"storage_sql\", \"bitcask\")?\n .set_default(\"fsync\", true)?\n .set_default(\"compact_threshold\", 0.2)?\n .set_default(\"compact_min_bytes\", 1_000_000)?\n .add_source(config::File::with_name(file))\n .add_source(config::Environment::with_prefix(\"TOYDB\"))\n .build()?\n .try_deserialize()?)\n }\n}\n\n/// The toyDB server command.\n#[derive(clap::Parser)]\n#[command(about = \"Starts a toyDB server.\", version, propagate_version = true)]\nstruct Command {\n /// The configuration file path.\n #[arg(short = 'c', long, default_value = \"config/toydb.yaml\")]\n config: String,\n}\n\nimpl Command {\n /// Runs the toyDB server.\n fn run(self) -> Result<()> {\n // Load the configuration.\n let cfg = Config::load(&self.config)?;\n\n // Initialize logging.\n let loglevel = cfg.log_level.parse()?;\n let mut logconfig = simplelog::ConfigBuilder::new();\n if loglevel != simplelog::LevelFilter::Debug {\n logconfig.add_filter_allow_str(\"toydb\");\n }\n simplelog::SimpleLogger::init(loglevel, logconfig.build())?;\n\n // Initialize the Raft log storage engine.\n let datadir = Path::new(&cfg.data_dir);\n let mut raft_log = match cfg.storage_raft.as_str() {\n \"bitcask\" | \"\" => {\n let engine = storage::BitCask::new_maybe_compact(\n datadir.join(\"raft\"),\n cfg.compact_threshold,\n cfg.compact_min_bytes,\n )?;\n raft::Log::new(Box::new(engine))?\n }\n \"memory\" => raft::Log::new(Box::new(storage::Memory::new()))?,\n name => return errinput!(\"invalid Raft storage engine {name}\"),\n };\n raft_log.enable_fsync(cfg.fsync);\n\n // Initialize the SQL storage engine.\n let raft_state: Box = match cfg.storage_sql.as_str() {\n \"bitcask\" | \"\" => {\n let engine = storage::BitCask::new_maybe_compact(\n datadir.join(\"sql\"),\n cfg.compact_threshold,\n cfg.compact_min_bytes,\n )?;\n Box::new(sql::engine::Raft::new_state(engine)?)\n }\n \"memory\" => Box::new(sql::engine::Raft::new_state(storage::Memory::new())?),\n name => return errinput!(\"invalid SQL storage engine {name}\"),\n };\n\n // Start the server.\n Server::new(cfg.id, cfg.peers, raft_log, raft_state)?\n .serve(&cfg.listen_raft, &cfg.listen_sql)\n }\n}\n" }, { "path": "src/bin/toydump.rs", "content": "//! toydump is a debug tool that prints a toyDB BitCask database in\n//! human-readable form. It can print both the SQL database and the Raft log\n//! (via --raft). It only outputs live BitCask data, not garbage entries.\n\n#![warn(clippy::all)]\n\nuse clap::Parser as _;\n\nuse toydb::encoding::format::{self, Formatter as _};\nuse toydb::error::Result;\nuse toydb::storage::{BitCask, Engine as _};\n\nfn main() {\n if let Err(error) = Command::parse().run() {\n eprintln!(\"Error: {error}\")\n }\n}\n\n/// The toydump command.\n#[derive(clap::Parser)]\n#[command(about = \"Prints toyDB file contents.\", version, propagate_version = true)]\nstruct Command {\n /// The BitCask file to dump (SQL database unless --raft).\n file: String,\n /// The file is a Raft log, not SQL database.\n #[arg(long)]\n raft: bool,\n /// Also show raw key and value.\n #[arg(long)]\n raw: bool,\n}\n\nimpl Command {\n /// Runs the command.\n fn run(self) -> Result<()> {\n let mut engine = BitCask::new(self.file.into())?;\n let mut scan = engine.scan(..);\n while let Some((key, value)) = scan.next().transpose()? {\n let mut string = match self.raft {\n true => format::Raft::::key_value(&key, &value),\n false => format::MVCC::::key_value(&key, &value),\n };\n if self.raw {\n string = format!(\"{string} [{}]\", format::Raw::key_value(&key, &value))\n }\n println!(\"{string}\");\n }\n Ok(())\n }\n}\n" }, { "path": "src/bin/toysql.rs", "content": "//! toySQL is a command-line client for toyDB. It connects to a toyDB node\n//! (default localhost:9601) and executes SQL statements against it via an\n//! interactive shell interface. Command history is stored in .toysql.history.\n\n#![warn(clippy::all)]\n\nuse std::path::PathBuf;\n\nuse clap::Parser as _;\nuse itertools::Itertools as _;\nuse rustyline::error::ReadlineError;\nuse rustyline::history::DefaultHistory;\nuse rustyline::validate::{ValidationContext, ValidationResult, Validator};\nuse rustyline::{Editor, Modifiers};\nuse rustyline_derive::{Completer, Helper, Highlighter, Hinter};\n\nuse toydb::Client;\nuse toydb::errinput;\nuse toydb::error::Result;\nuse toydb::sql::execution::StatementResult;\nuse toydb::sql::parser::{Lexer, Token};\n\nfn main() {\n if let Err(error) = Command::parse().run() {\n eprintln!(\"Error: {error}\");\n }\n}\n\n/// The toySQL command.\n#[derive(clap::Parser)]\n#[command(about = \"A toyDB client.\", version, propagate_version = true)]\nstruct Command {\n /// A SQL statement to execute, then exit.\n #[arg()]\n statement: Option,\n /// Host to connect to.\n #[arg(short = 'H', long, default_value = \"localhost\")]\n host: String,\n /// Port number to connect to.\n #[arg(short = 'p', long, default_value = \"9601\")]\n port: u16,\n}\n\nimpl Command {\n /// Runs the command.\n fn run(self) -> Result<()> {\n let mut shell = Shell::new(&self.host, self.port)?;\n match self.statement {\n Some(statement) => shell.execute(&statement),\n None => shell.run(),\n }\n }\n}\n\n/// An interactive toySQL shell.\nstruct Shell {\n /// The toyDB client.\n client: Client,\n /// The Rustyline command editor.\n editor: Editor,\n /// The path to the history file, if any.\n history_path: Option,\n /// If true, SELECT column headers will be displayed.\n show_headers: bool,\n}\n\nimpl Shell {\n /// Creates a new shell connected to the given server.\n fn new(host: &str, port: u16) -> Result {\n let client = Client::connect((host, port))?;\n // Set up Rustyline. Make sure multiline pastes are handled normally.\n let mut editor = Editor::new()?;\n editor.set_helper(Some(InputValidator));\n editor.bind_sequence(\n rustyline::KeyEvent(rustyline::KeyCode::BracketedPasteStart, Modifiers::NONE),\n rustyline::Cmd::Noop,\n );\n let history_path =\n std::env::var_os(\"HOME\").map(|home| PathBuf::from(home).join(\".toysql.history\"));\n Ok(Self { client, editor, history_path, show_headers: false })\n }\n\n /// Executes a SQL statement or ! command.\n fn execute(&mut self, input: &str) -> Result<()> {\n if input.starts_with('!') {\n self.execute_command(input)\n } else if !input.is_empty() {\n self.execute_sql(input)\n } else {\n Ok(())\n }\n }\n\n /// Executes a toySQL ! command (e.g. !help)\n fn execute_command(&mut self, input: &str) -> Result<()> {\n let mut input = input.split_ascii_whitespace();\n let Some(command) = input.next() else {\n return errinput!(\"expected command\");\n };\n let args = input.collect_vec();\n\n match (command, args.as_slice()) {\n // Toggles column headers.\n (\"!headers\", []) => {\n self.show_headers = !self.show_headers;\n match self.show_headers {\n true => println!(\"Headers enabled\"),\n false => println!(\"Headers disabled\"),\n }\n }\n (\"!headers\", _) => return errinput!(\"!headers takes no arguments\"),\n\n // Displays help.\n (\"!help\", []) => println!(\n r#\"\nEnter a SQL statement terminated by a semicolon (;) to execute it, or Ctrl-D to\nexit. The following commands are also available:\n\n !headers Toggles column headers\n !help This help message\n !status Display server status\n !table NAME Display a table schema\n !tables List tables\n\"#\n ),\n (\"!help\", _) => return errinput!(\"!help takes no arguments\"),\n\n // Displays server status.\n (\"!status\", []) => {\n let status = self.client.status()?;\n println!(\n r#\"\nServer: n{server} with Raft leader n{leader} in term {term} for {nodes} nodes\nRaft log: {committed} committed, {applied} applied, {raft_size} MB, {raft_garbage}% garbage ({raft_storage} engine)\nReplication: {raft_match}\nSQL storage: {sql_keys} keys, {sql_size} MB logical, {nodes}x {sql_disk_size} MB disk, {sql_garbage}% garbage ({sql_storage} engine)\nTransactions: {active_txns} active, {versions} total\n\"#,\n server = status.server,\n leader = status.raft.leader,\n term = status.raft.term,\n nodes = status.raft.match_index.len(),\n committed = status.raft.commit_index,\n applied = status.raft.applied_index,\n raft_size =\n format_args!(\"{:.3}\", status.raft.storage.size as f64 / 1_000_000.0),\n raft_garbage =\n format_args!(\"{:.0}\", status.raft.storage.garbage_disk_percent()),\n raft_storage = status.raft.storage.name,\n raft_match =\n status.raft.match_index.iter().map(|(n, m)| format!(\"n{n}:{m}\")).join(\" \"),\n sql_keys = status.mvcc.storage.keys,\n sql_size = format_args!(\"{:.3}\", status.mvcc.storage.size as f64 / 1_000_000.0),\n sql_disk_size =\n format_args!(\"{:.3}\", status.mvcc.storage.disk_size as f64 / 1_000_000.0),\n sql_garbage = format_args!(\"{:.0}\", status.mvcc.storage.garbage_disk_percent()),\n sql_storage = status.mvcc.storage.name,\n active_txns = status.mvcc.active_txns,\n versions = status.mvcc.versions,\n )\n }\n (\"!status\", _) => return errinput!(\"!status takes no arguments\"),\n\n (\"!table\", [name]) => println!(\"{}\", self.client.get_table(name)?),\n (\"!table\", _) => return errinput!(\"!table takes 1 argument\"),\n\n (\"!tables\", []) => self.client.list_tables()?.iter().for_each(|t| println!(\"{t}\")),\n (\"!tables\", _) => return errinput!(\"!tables takes no arguments\"),\n\n (command, _) => return errinput!(\"unknown command {command}\"),\n }\n Ok(())\n }\n\n /// Executes a SQL statement and displays the results.\n fn execute_sql(&mut self, statement: &str) -> Result<()> {\n use StatementResult::*;\n match self.client.execute(statement)? {\n Begin(state) => match state.read_only {\n true => println!(\"Began read-only transaction at version {}\", state.version),\n false => println!(\"Began transaction {}\", state.version),\n },\n Commit { version } => println!(\"Committed transaction {version}\"),\n Rollback { version } => println!(\"Rolled back transaction {version}\"),\n Insert { count } => println!(\"Inserted {count} rows\"),\n Delete { count } => println!(\"Deleted {count} rows\"),\n Update { count } => println!(\"Updated {count} rows\"),\n CreateTable { name } => println!(\"Created table {name}\"),\n DropTable { name, existed } => match existed {\n true => println!(\"Dropped table {name}\"),\n false => println!(\"Table {name} does not exist\"),\n },\n Explain(plan) => println!(\"{plan}\"),\n Select { columns, rows } => {\n if self.show_headers {\n println!(\"{}\", columns.iter().map(|c| c.as_header()).join(\", \"));\n }\n for row in rows {\n println!(\"{}\", row.iter().join(\", \"));\n }\n }\n }\n Ok(())\n }\n\n /// Prompts the user for input. Returns None if the shell should close.\n fn prompt(&mut self) -> rustyline::Result {\n let prompt = match self.client.txn() {\n Some(txn) if txn.read_only => format!(\"toydb@{}> \", txn.version),\n Some(txn) => format!(\"toydb:{}> \", txn.version),\n None => \"toydb> \".to_string(),\n };\n self.editor.readline(&prompt)\n }\n\n /// Runs the interactive shell.\n fn run(&mut self) -> Result<()> {\n // Load the history file, if any.\n if let Some(history_path) = &self.history_path {\n match self.editor.load_history(history_path) {\n Ok(()) => {}\n Err(ReadlineError::Io(error)) if error.kind() == std::io::ErrorKind::NotFound => {}\n Err(error) => return Err(error.into()),\n }\n }\n\n // Print welcome message.\n let server = self.client.status()?.server;\n println!(\"Connected to toyDB node n{server}. Enter !help for instructions.\");\n\n // Prompt for commands and execute them.\n loop {\n let input = match self.prompt() {\n Ok(input) => input.trim().to_string(),\n Err(ReadlineError::Interrupted) => continue,\n Err(ReadlineError::Eof) => break,\n Err(error) => return Err(error.into()),\n };\n self.editor.add_history_entry(&input)?;\n if let Err(error) = self.execute(&input) {\n eprintln!(\"Error: {error}\");\n };\n }\n\n // Save the history file.\n if let Some(history_path) = &self.history_path {\n self.editor.save_history(history_path)?;\n }\n Ok(())\n }\n}\n\n/// A Rustyline helper for multiline editing. After a new line is entered, it\n/// determines whether the input makes up a complete SQL statement that should\n/// be submitted to the server (i.e. it's terminated by ;), or wait for further\n/// input.\n#[derive(Completer, Helper, Highlighter, Hinter)]\nstruct InputValidator;\n\nimpl Validator for InputValidator {\n fn validate(&self, ctx: &mut ValidationContext) -> rustyline::Result {\n let input = ctx.input();\n // Empty lines and ! commands are ready.\n if input.is_empty() || input.starts_with('!') || input == \";\" {\n return Ok(ValidationResult::Valid(None));\n }\n // For SQL statements, just look for any semicolon or lexer error, and\n // rely on the server for further validation and error handling.\n if Lexer::new(input).any(|r| matches!(r, Ok(Token::Semicolon) | Err(_))) {\n return Ok(ValidationResult::Valid(None));\n }\n // Otherwise, wait for more input.\n Ok(ValidationResult::Incomplete)\n }\n\n fn validate_while_typing(&self) -> bool {\n false // only check after completed lines\n }\n}\n" }, { "path": "src/bin/workload.rs", "content": "//! Runs toyDB workload benchmarks. By default, it assumes a running 5-node\n//! cluster as launched via cluster/run.sh, but this can be modified via -H.\n//! For example, a read-only workload can be run as:\n//!\n//! cargo run --release --bin workload -- read\n//!\n//! See --help for a list of available workloads and arguments.\n\n#![warn(clippy::all)]\n\nuse std::cmp::min;\nuse std::collections::HashSet;\nuse std::io::Write as _;\nuse std::time::{Duration, Instant};\n\nuse clap::Parser;\nuse hdrhistogram::Histogram;\nuse itertools::Itertools as _;\nuse rand::SeedableRng as _;\nuse rand::distr::Distribution as _;\nuse rand::rngs::StdRng;\nuse rand::seq::IndexedRandom as _;\n\nuse toydb::error::Result;\nuse toydb::sql::types::{Row, Rows};\nuse toydb::{Client, StatementResult};\n\nfn main() {\n let Command { runner, subcommand } = Command::parse();\n let result = match subcommand {\n Subcommand::Read(read) => runner.run(read),\n Subcommand::Write(write) => runner.run(write),\n Subcommand::Bank(bank) => runner.run(bank),\n };\n if let Err(error) = result {\n eprintln!(\"Error: {error}\")\n }\n}\n\n/// Handles command-line parsing.\n#[derive(clap::Parser)]\n#[command(about = \"Runs toyDB workload benchmarks.\", version, propagate_version = true)]\nstruct Command {\n #[command(flatten)]\n runner: Runner,\n\n #[command(subcommand)]\n subcommand: Subcommand,\n}\n\n#[derive(clap::Subcommand)]\nenum Subcommand {\n Read(Read),\n Write(Write),\n Bank(Bank),\n}\n\n/// Runs a workload benchmark.\n#[derive(clap::Args)]\nstruct Runner {\n /// Hosts to connect to (optionally with port number).\n #[arg(\n short = 'H',\n long,\n value_delimiter = ',',\n default_value = \"localhost:9601,localhost:9602,localhost:9603,localhost:9604,localhost:9605\"\n )]\n hosts: Vec,\n\n /// Number of concurrent workers to spawn.\n #[arg(short, long, default_value = \"16\")]\n concurrency: usize,\n\n /// Number of transactions to execute.\n #[arg(short = 'n', long, default_value = \"100000\")]\n count: usize,\n\n /// Seed to use for random number generation.\n #[arg(short, long, default_value = \"16791084677885396490\")]\n seed: u64,\n}\n\nimpl Runner {\n /// Runs the specified workload.\n fn run(self, workload: W) -> Result<()> {\n let mut rng = StdRng::seed_from_u64(self.seed);\n let mut client = Client::connect(&self.hosts[0])?;\n\n // Set up a histogram recording txn latencies as nanoseconds. The\n // buckets range from 0.001s to 10s.\n let mut hist = Histogram::::new_with_bounds(1_000, 10_000_000_000, 3)?.into_sync();\n\n // Prepare the dataset.\n print!(\"Preparing initial dataset... \");\n std::io::stdout().flush()?;\n let start = Instant::now();\n workload.prepare(&mut client, &mut rng)?;\n println!(\"done ({:.3}s)\", start.elapsed().as_secs_f64());\n\n // Spawn workers, round robin across hosts.\n std::thread::scope(|s| -> Result<()> {\n print!(\"Spawning {} workers... \", self.concurrency);\n std::io::stdout().flush()?;\n let start = Instant::now();\n\n let (work_tx, work_rx) = crossbeam::channel::bounded(self.concurrency);\n let (done_tx, done_rx) = crossbeam::channel::bounded::<()>(0);\n\n for addr in self.hosts.iter().cycle().take(self.concurrency) {\n let mut client = Client::connect(addr)?;\n let mut recorder = hist.recorder();\n let work_rx = work_rx.clone();\n let done_tx = done_tx.clone();\n s.spawn(move || -> Result<()> {\n while let Ok(item) = work_rx.recv() {\n let start = Instant::now();\n client.with_retry(|client| W::execute(client, &item))?;\n recorder.record(start.elapsed().as_nanos() as u64)?;\n }\n drop(done_tx); // disconnects done_rx once all workers exit\n Ok(())\n });\n }\n drop(done_tx); // drop local copy\n\n println!(\"done ({:.3}s)\", start.elapsed().as_secs_f64());\n\n // Spawn work generator.\n {\n println!(\"Running workload {}...\", workload);\n let generator = workload.generate(rng)?.take(self.count);\n s.spawn(move || -> Result<()> {\n for item in generator {\n work_tx.send(item)?;\n }\n Ok(())\n });\n }\n\n // Periodically print stats until all workers are done.\n let start = Instant::now();\n let ticker = crossbeam::channel::tick(Duration::from_secs(1));\n\n println!();\n println!(\"Time Progress Txns Rate p50 p90 p99 pMax\");\n\n while let Err(crossbeam::channel::TryRecvError::Empty) = done_rx.try_recv() {\n crossbeam::select! {\n recv(ticker) -> _ => {},\n recv(done_rx) -> _ => {},\n }\n\n let duration = start.elapsed().as_secs_f64();\n hist.refresh_timeout(Duration::from_secs(1));\n\n println!(\n \"{:<8} {:>5.1}% {:>7} {:>6.0}/s {:>6.1}ms {:>6.1}ms {:>6.1}ms {:>6.1}ms\",\n format!(\"{:.1}s\", duration),\n hist.len() as f64 / self.count as f64 * 100.0,\n hist.len(),\n hist.len() as f64 / duration,\n Duration::from_nanos(hist.value_at_quantile(0.5)).as_secs_f64() * 1000.0,\n Duration::from_nanos(hist.value_at_quantile(0.9)).as_secs_f64() * 1000.0,\n Duration::from_nanos(hist.value_at_quantile(0.99)).as_secs_f64() * 1000.0,\n Duration::from_nanos(hist.max()).as_secs_f64() * 1000.0,\n );\n }\n Ok(())\n })?;\n\n // Verify the final dataset.\n println!();\n print!(\"Verifying dataset... \");\n std::io::stdout().flush()?;\n let start = Instant::now();\n workload.verify(&mut client, self.count)?;\n println!(\"done ({:.3}s)\", start.elapsed().as_secs_f64());\n\n Ok(())\n }\n}\n\n/// A workload.\ntrait Workload: std::fmt::Display {\n /// A work item.\n type Item: Send;\n\n /// Prepares the workload by creating initial tables and data.\n fn prepare(&self, client: &mut Client, rng: &mut StdRng) -> Result<()>;\n\n /// Generates work items as an iterator.\n fn generate(&self, rng: StdRng) -> Result + Send + 'static>;\n\n /// Executes a single work item. This will automatically be retried on\n /// certain errors, and must use a transaction where appropriate.\n fn execute(client: &mut Client, item: &Self::Item) -> Result<()>;\n\n /// Verifies the dataset after the workload has completed.\n fn verify(&self, _client: &mut Client, _txns: usize) -> Result<()> {\n Ok(())\n }\n}\n\n/// A read-only workload. Creates an id,value table and populates it with the\n/// given row count and value size. Then runs batches of random primary key\n/// lookups (SELECT * FROM read WHERE id = 1 OR id = 2 ...).\n#[derive(clap::Args, Clone)]\n#[command(about = \"A read-only workload using primary key lookups\")]\nstruct Read {\n /// Total number of rows in data set.\n #[arg(short, long, default_value = \"1000\")]\n rows: u64,\n\n /// Row value size (excluding primary key).\n #[arg(short, long, default_value = \"64\")]\n size: usize,\n\n /// Number of rows to fetch in a single select.\n #[arg(short, long, default_value = \"1\")]\n batch: usize,\n}\n\nimpl std::fmt::Display for Read {\n fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {\n write!(f, \"read (rows={} size={} batch={})\", self.rows, self.size, self.batch)\n }\n}\n\nimpl Workload for Read {\n type Item = HashSet;\n\n fn prepare(&self, client: &mut Client, rng: &mut StdRng) -> Result<()> {\n client.execute(\"BEGIN\")?;\n client.execute(r#\"DROP TABLE IF EXISTS \"read\"\"#)?;\n client.execute(r#\"CREATE TABLE \"read\" (id INT PRIMARY KEY, value STRING NOT NULL)\"#)?;\n\n let chars = &mut rand::distr::Alphanumeric.sample_iter(rng).map(|b| b as char);\n let rows = (1..=self.rows).map(|id| (id, chars.take(self.size).collect::()));\n let chunks = rows.chunks(100);\n let queries = chunks.into_iter().map(|chunk| {\n format!(\n r#\"INSERT INTO \"read\" (id, value) VALUES ({})\"#,\n chunk.map(|(id, value)| format!(\"{}, '{}'\", id, value)).join(\"), (\")\n )\n });\n for query in queries {\n client.execute(&query)?;\n }\n client.execute(\"COMMIT\")?;\n Ok(())\n }\n\n fn generate(&self, rng: StdRng) -> Result + 'static> {\n Ok(ReadGenerator {\n batch: self.batch,\n dist: rand::distr::Uniform::new(1, self.rows + 1)?,\n rng,\n })\n }\n\n fn execute(client: &mut Client, item: &Self::Item) -> Result<()> {\n let batch_size = item.len();\n let query = format!(\n r#\"SELECT * FROM \"read\" WHERE {}\"#,\n item.iter().map(|id| format!(\"id = {}\", id)).join(\" OR \")\n );\n let rows: Rows = client.execute(&query)?.try_into()?;\n assert_eq!(rows.count(), batch_size, \"Unexpected row count\");\n Ok(())\n }\n\n fn verify(&self, client: &mut Client, _: usize) -> Result<()> {\n let count: i64 = client.execute(r#\"SELECT COUNT(*) FROM \"read\"\"#)?.try_into()?;\n assert_eq!(count, self.rows as i64, \"Unexpected row count\");\n Ok(())\n }\n}\n\n/// A Read workload generator, yielding batches of random, unique primary keys.\nstruct ReadGenerator {\n batch: usize,\n rng: StdRng,\n dist: rand::distr::Uniform,\n}\n\nimpl Iterator for ReadGenerator {\n type Item = ::Item;\n\n fn next(&mut self) -> Option {\n let mut ids = HashSet::new();\n for id in self.dist.sample_iter(&mut self.rng) {\n ids.insert(id);\n if ids.len() >= self.batch {\n break;\n }\n }\n Some(ids)\n }\n}\n\n/// A write-only workload. Creates an id,value table, and writes rows with\n/// sequential primary keys and the given value size, in the given batch size\n/// (INSERT INTO write (id, value) VALUES ...). The number of rows written\n/// is given by Runner.count * Write.batch.\n#[derive(clap::Args, Clone)]\n#[command(about = \"A write-only workload writing sequential rows\")]\nstruct Write {\n /// Row value size (excluding primary key).\n #[arg(short, long, default_value = \"64\")]\n size: usize,\n\n /// Number of rows to write in a single insert query.\n #[arg(short, long, default_value = \"1\")]\n batch: usize,\n}\n\nimpl std::fmt::Display for Write {\n fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {\n write!(f, \"write (size={} batch={})\", self.size, self.batch)\n }\n}\n\nimpl Workload for Write {\n type Item = Vec<(u64, String)>;\n\n fn prepare(&self, client: &mut Client, _: &mut StdRng) -> Result<()> {\n client.execute(\"BEGIN\")?;\n client.execute(r#\"DROP TABLE IF EXISTS \"write\"\"#)?;\n client.execute(r#\"CREATE TABLE \"write\" (id INT PRIMARY KEY, value STRING NOT NULL)\"#)?;\n client.execute(\"COMMIT\")?;\n Ok(())\n }\n\n fn generate(&self, rng: StdRng) -> Result + 'static> {\n Ok(WriteGenerator { next_id: 1, size: self.size, batch: self.batch, rng })\n }\n\n fn execute(client: &mut Client, item: &Self::Item) -> Result<()> {\n let batch_size = item.len();\n let query = format!(\n r#\"INSERT INTO \"write\" (id, value) VALUES {}\"#,\n item.iter().map(|(id, value)| format!(\"({}, '{}')\", id, value)).join(\", \")\n );\n if let StatementResult::Insert { count } = client.execute(&query)? {\n assert_eq!(count as usize, batch_size, \"Unexpected row count\");\n } else {\n panic!(\"Unexpected result\")\n }\n Ok(())\n }\n\n fn verify(&self, client: &mut Client, txns: usize) -> Result<()> {\n let count: i64 = client.execute(r#\"SELECT COUNT(*) FROM \"write\"\"#)?.try_into()?;\n assert_eq!(count as usize, txns * self.batch, \"Unexpected row count\");\n Ok(())\n }\n}\n\n/// A Write workload generator, yielding batches of sequential primary keys and\n/// random rows.\nstruct WriteGenerator {\n next_id: u64,\n size: usize,\n batch: usize,\n rng: StdRng,\n}\n\nimpl Iterator for WriteGenerator {\n type Item = ::Item;\n\n fn next(&mut self) -> Option {\n let chars = &mut rand::distr::Alphanumeric.sample_iter(&mut self.rng).map(|b| b as char);\n let mut rows = Vec::with_capacity(self.batch);\n while rows.len() < self.batch {\n rows.push((self.next_id, chars.take(self.size).collect()));\n self.next_id += 1;\n }\n Some(rows)\n }\n}\n\n/// A bank workload. Creates a set of customers and accounts, and makes random\n/// transfers between them. Specifically, it picks two random customers A and B,\n/// and then finds A's highest-balance account and B's lowest-balance account,\n/// and transfers a random amount without overdrawing the account. This\n/// somewhat convoluted scheme is used to make the workload slightly less\n/// trivial, including joins, ordering, and secondary indexes.\n#[derive(clap::Args, Clone)]\n#[command(about = \"A bank workload, making transfers between customer accounts\")]\nstruct Bank {\n /// Number of customers.\n #[arg(short, long, default_value = \"100\")]\n customers: u64,\n\n /// Number of accounts per customer.\n #[arg(short, long, default_value = \"10\")]\n accounts: u64,\n\n /// Initial account balance.\n #[arg(short, long, default_value = \"100\")]\n balance: u64,\n\n /// Max amount to transfer.\n #[arg(short, long, default_value = \"50\")]\n max_transfer: u64,\n}\n\nimpl std::fmt::Display for Bank {\n fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {\n write!(f, \"bank (customers={} accounts={})\", self.customers, self.accounts)\n }\n}\n\nimpl Workload for Bank {\n type Item = (u64, u64, u64); // from,to,amount\n\n fn prepare(&self, client: &mut Client, rng: &mut StdRng) -> Result<()> {\n let petnames = petname::Petnames::default();\n client.execute(\"BEGIN\")?;\n client.execute(\"DROP TABLE IF EXISTS account\")?;\n client.execute(\"DROP TABLE IF EXISTS customer\")?;\n client.execute(\n \"CREATE TABLE customer (\n id INTEGER PRIMARY KEY,\n name STRING NOT NULL\n )\",\n )?;\n client.execute(\n \"CREATE TABLE account (\n id INTEGER PRIMARY KEY,\n customer_id INTEGER NOT NULL INDEX REFERENCES customer,\n balance INTEGER NOT NULL\n )\",\n )?;\n client.execute(&format!(\n \"INSERT INTO customer VALUES {}\",\n (1..=self.customers)\n .map(|id| {\n let name = [\n *petnames.adverbs.choose(rng).expect(\"no adverb\"),\n *petnames.adjectives.choose(rng).expect(\"no adjective\"),\n *petnames.nouns.choose(rng).expect(\"no noun\"),\n ]\n .join(\" \");\n (id, name)\n })\n .map(|(id, name)| format!(\"({}, '{}')\", id, name))\n .join(\", \")\n ))?;\n client.execute(&format!(\n \"INSERT INTO account VALUES {}\",\n (1..=self.customers)\n .flat_map(|c| (1..=self.accounts).map(move |a| (c, (c - 1) * self.accounts + a)))\n .map(|(c, a)| format!(\"({}, {}, {})\", a, c, self.balance))\n .join(\", \")\n ))?;\n client.execute(\"COMMIT\")?;\n Ok(())\n }\n\n fn generate(&self, rng: StdRng) -> Result + 'static> {\n let customers = self.customers;\n let max_transfer = self.max_transfer;\n // Generate random u64s, then pick random from,to,amount as the\n // remainder of the max customer and amount.\n Ok(rand::distr::Uniform::new_inclusive(0, u64::MAX)?\n .sample_iter(rng)\n .tuples()\n .map(move |(a, b, c)| (a % customers + 1, b % customers + 1, c % max_transfer + 1))\n .filter(|(from, to, _)| from != to))\n }\n\n fn execute(client: &mut Client, item: &Self::Item) -> Result<()> {\n let &(from, to, mut amount) = item;\n\n client.execute(\"BEGIN\")?;\n\n let row: Row = client\n .execute(&format!(\n \"SELECT a.id, a.balance\n FROM account a JOIN customer c ON a.customer_id = c.id\n WHERE c.id = {}\n ORDER BY a.balance DESC\n LIMIT 1\",\n from\n ))?\n .try_into()?;\n let mut row = row.into_iter();\n let from_account: i64 = row.next().unwrap().try_into()?;\n let from_balance: i64 = row.next().unwrap().try_into()?;\n amount = min(amount, from_balance as u64);\n\n let to_account: i64 = client\n .execute(&format!(\n \"SELECT a.id, a.balance\n FROM account a JOIN customer c ON a.customer_id = c.id\n WHERE c.id = {}\n ORDER BY a.balance ASC\n LIMIT 1\",\n to\n ))?\n .try_into()?;\n\n client.execute(&format!(\n \"UPDATE account SET balance = balance - {} WHERE id = {}\",\n amount, from_account,\n ))?;\n client.execute(&format!(\n \"UPDATE account SET balance = balance + {} WHERE id = {}\",\n amount, to_account,\n ))?;\n\n client.execute(\"COMMIT\")?;\n\n Ok(())\n }\n\n fn verify(&self, client: &mut Client, _: usize) -> Result<()> {\n let balance: i64 = client.execute(\"SELECT SUM(balance) FROM account\")?.try_into()?;\n assert_eq!(balance as u64, self.customers * self.accounts * self.balance);\n let negative: i64 =\n client.execute(\"SELECT COUNT(*) FROM account WHERE balance < 0\")?.try_into()?;\n assert_eq!(negative, 0);\n Ok(())\n }\n}\n" }, { "path": "src/client.rs", "content": "use std::io::{BufReader, BufWriter, Write as _};\nuse std::net::{TcpStream, ToSocketAddrs};\nuse std::time::Duration;\n\nuse rand::RngExt as _;\n\nuse crate::encoding::Value as _;\nuse crate::errdata;\nuse crate::error::{Error, Result};\nuse crate::server::{Request, Response, Status};\nuse crate::sql::execution::StatementResult;\nuse crate::sql::types::Table;\nuse crate::storage::mvcc;\n\n/// A toyDB client. Connects to a server via TCP and submits SQL statements and\n/// other requests.\npub struct Client {\n /// Inbound response stream.\n reader: BufReader,\n /// Outbound request stream.\n writer: BufWriter,\n /// The current transaction, if any.\n txn: Option,\n}\n\nimpl Client {\n /// Connects to a toyDB server, creating a new client.\n pub fn connect(addr: impl ToSocketAddrs) -> Result {\n let socket = TcpStream::connect(addr)?;\n let reader = BufReader::new(socket.try_clone()?);\n let writer = BufWriter::new(socket);\n Ok(Self { reader, writer, txn: None })\n }\n\n /// Sends a request to the server, returning the response.\n fn request(&mut self, request: Request) -> Result {\n request.encode_into(&mut self.writer)?;\n self.writer.flush()?;\n Result::decode_from(&mut self.reader)?\n }\n\n /// Executes a SQL statement.\n pub fn execute(&mut self, statement: &str) -> Result {\n let result = match self.request(Request::Execute(statement.to_string()))? {\n Response::Execute(result) => result,\n response => return errdata!(\"unexpected response {response:?}\"),\n };\n // Update the transaction state.\n match &result {\n StatementResult::Begin(state) => self.txn = Some(state.clone()),\n StatementResult::Commit { .. } => self.txn = None,\n StatementResult::Rollback { .. } => self.txn = None,\n _ => {}\n }\n Ok(result)\n }\n\n /// Fetches a table schema.\n pub fn get_table(&mut self, table: &str) -> Result {\n match self.request(Request::GetTable(table.to_string()))? {\n Response::GetTable(table) => Ok(table),\n response => errdata!(\"unexpected response: {response:?}\"),\n }\n }\n\n /// Lists database tables.\n pub fn list_tables(&mut self) -> Result> {\n match self.request(Request::ListTables)? {\n Response::ListTables(tables) => Ok(tables),\n response => errdata!(\"unexpected response: {response:?}\"),\n }\n }\n\n /// Returns server status.\n pub fn status(&mut self) -> Result {\n match self.request(Request::Status)? {\n Response::Status(status) => Ok(status),\n response => errdata!(\"unexpected response: {response:?}\"),\n }\n }\n\n /// Returns the transaction state.\n pub fn txn(&self) -> Option<&mvcc::TransactionState> {\n self.txn.as_ref()\n }\n\n /// Runs the given closure, automatically retrying serialization and abort\n /// errors. If a transaction is open following an error, it is automatically\n /// rolled back. It is the caller's responsibility to use a transaction in\n /// the closure where appropriate (i.e. when it is not idempotent).\n pub fn with_retry(&mut self, f: impl Fn(&mut Client) -> Result) -> Result {\n const MAX_RETRIES: u32 = 10;\n const MIN_WAIT: u64 = 10;\n const MAX_WAIT: u64 = 2_000;\n let mut retries: u32 = 0;\n loop {\n match f(self) {\n Ok(result) => return Ok(result),\n Err(Error::Serialization | Error::Abort) if retries < MAX_RETRIES => {\n if self.txn().is_some() {\n self.execute(\"ROLLBACK\")?;\n }\n // Use exponential backoff starting at MIN_WAIT doubling up\n // to MAX_WAIT, but randomize the wait time in this interval\n // to reduce the chance of collisions.\n let mut wait = MAX_WAIT.min(MIN_WAIT * 2_u64.pow(retries));\n wait = rand::rng().random_range(MIN_WAIT..=wait);\n std::thread::sleep(Duration::from_millis(wait));\n retries += 1;\n }\n Err(error) => {\n if self.txn().is_some() {\n self.execute(\"ROLLBACK\").ok(); // ignore rollback error\n }\n return Err(error);\n }\n }\n }\n }\n}\n" }, { "path": "src/encoding/bincode.rs", "content": "//! Bincode is used to encode values, both in key/value stores and the toyDB\n//! network protocol. It is a Rust-specific encoding that depends on the\n//! internal data structures being stable, but it's sufficient for toyDB. See:\n//! \n//!\n//! This module wraps the [`bincode`] crate and uses the standard config.\n\nuse std::io::{Read, Write};\n\nuse serde::de::DeserializeOwned;\nuse serde::{Deserialize, Serialize};\n\nuse crate::error::{Error, Result};\n\n/// Use the standard Bincode configuration.\nconst CONFIG: bincode::config::Configuration = bincode::config::standard();\n\n/// Serializes a value using Bincode.\npub fn serialize(value: &T) -> Vec {\n // Panic on failure, as this is a problem with the data structure.\n bincode::serde::encode_to_vec(value, CONFIG).expect(\"value must be serializable\")\n}\n\n/// Deserializes a value using Bincode.\npub fn deserialize<'de, T: Deserialize<'de>>(bytes: &'de [u8]) -> Result {\n Ok(bincode::serde::borrow_decode_from_slice(bytes, CONFIG)?.0)\n}\n\n/// Serializes a value to a writer using Bincode.\npub fn serialize_into(mut writer: W, value: &T) -> Result<()> {\n bincode::serde::encode_into_std_write(value, &mut writer, CONFIG)?;\n Ok(())\n}\n\n/// Deserializes a value from a reader using Bincode.\npub fn deserialize_from(mut reader: R) -> Result {\n Ok(bincode::serde::decode_from_std_read(&mut reader, CONFIG)?)\n}\n\n/// Deserializes a value from a reader using Bincode, or returns None if the\n/// reader is closed.\npub fn maybe_deserialize_from(mut reader: R) -> Result> {\n match bincode::serde::decode_from_std_read(&mut reader, CONFIG) {\n Ok(t) => Ok(Some(t)),\n Err(bincode::error::DecodeError::Io { inner, .. })\n if inner.kind() == std::io::ErrorKind::UnexpectedEof\n || inner.kind() == std::io::ErrorKind::ConnectionReset =>\n {\n Ok(None)\n }\n Err(err) => Err(Error::from(err)),\n }\n}\n" }, { "path": "src/encoding/format.rs", "content": "//! Formats raw keys and values, recursively where necessary. Handles both both\n//! Raft, MVCC, SQL, and raw binary data.\n\nuse std::collections::BTreeSet;\nuse std::marker::PhantomData;\n\nuse itertools::Itertools as _;\nuse regex::Regex;\n\nuse super::{Key as _, Value as _, bincode};\nuse crate::raft;\nuse crate::sql;\nuse crate::storage::mvcc;\n\n/// Formats encoded keys and values.\npub trait Formatter {\n /// Formats a key.\n fn key(key: &[u8]) -> String;\n\n /// Formats a value. Also takes the key to determine the kind of value.\n fn value(key: &[u8], value: &[u8]) -> String;\n\n /// Formats a key/value pair.\n fn key_value(key: &[u8], value: &[u8]) -> String {\n Self::key_maybe_value(key, Some(value))\n }\n\n /// Formats a key/value pair, where the value may not exist.\n fn key_maybe_value(key: &[u8], value: Option<&[u8]>) -> String {\n let fmtkey = Self::key(key);\n let fmtvalue = value.map_or(\"None\".to_string(), |v| Self::value(key, v));\n format!(\"{fmtkey} → {fmtvalue}\")\n }\n}\n\n/// Formats raw byte slices without any decoding.\npub struct Raw;\n\nimpl Raw {\n /// Formats raw bytes as escaped ASCII strings.\n pub fn bytes(bytes: &[u8]) -> String {\n let escaped = bytes.iter().copied().flat_map(std::ascii::escape_default).collect_vec();\n format!(\"\\\"{}\\\"\", String::from_utf8_lossy(&escaped))\n }\n}\n\nimpl Formatter for Raw {\n fn key(key: &[u8]) -> String {\n Self::bytes(key)\n }\n\n fn value(_key: &[u8], value: &[u8]) -> String {\n Self::bytes(value)\n }\n}\n\n/// Formats Raft log entries. Dispatches to F to format each Raft command.\npub struct Raft(PhantomData);\n\nimpl Raft {\n /// Formats a Raft entry.\n pub fn entry(entry: &raft::Entry) -> String {\n let fmtcommand = entry.command.as_deref().map_or(\"None\".to_string(), |c| F::value(&[], c));\n format!(\"{}@{} {fmtcommand}\", entry.index, entry.term)\n }\n}\n\nimpl Formatter for Raft {\n fn key(key: &[u8]) -> String {\n let Ok(key) = raft::Key::decode(key) else {\n return Raw::key(key); // invalid key\n };\n format!(\"raft:{key:?}\")\n }\n\n fn value(key: &[u8], value: &[u8]) -> String {\n let Ok(key) = raft::Key::decode(key) else {\n return Raw::value(key, value); // invalid key\n };\n match key {\n raft::Key::CommitIndex => {\n match bincode::deserialize::<(raft::Index, raft::Term)>(value) {\n Ok((index, term)) => format!(\"{index}@{term}\"),\n Err(_) => Raw::bytes(value),\n }\n }\n raft::Key::TermVote => {\n match bincode::deserialize::<(raft::Term, Option)>(value) {\n Ok((term, vote)) => format!(\n \"term={term} vote={}\",\n vote.map_or(\"None\".to_string(), |v| v.to_string()),\n ),\n Err(_) => Raw::bytes(value),\n }\n }\n raft::Key::Entry(_) => match bincode::deserialize::(value) {\n Ok(entry) => Self::entry(&entry),\n Err(_) => Raw::bytes(value),\n },\n }\n }\n}\n\n/// Formats MVCC keys/values. Dispatches to F to format the inner key/value.\npub struct MVCC(PhantomData);\n\nimpl Formatter for MVCC {\n fn key(key: &[u8]) -> String {\n let Ok(key) = mvcc::Key::decode(key) else {\n return Raw::key(key); // invalid key\n };\n match key {\n mvcc::Key::TxnWrite(version, innerkey) => {\n format!(\"mvcc:TxnWrite({version}, {})\", F::key(&innerkey))\n }\n mvcc::Key::Version(innerkey, version) => {\n format!(\"mvcc:Version({}, {version})\", F::key(&innerkey))\n }\n mvcc::Key::Unversioned(innerkey) => {\n format!(\"mvcc:Unversioned({})\", F::key(&innerkey))\n }\n mvcc::Key::NextVersion | mvcc::Key::TxnActive(_) | mvcc::Key::TxnActiveSnapshot(_) => {\n format!(\"mvcc:{key:?}\")\n }\n }\n }\n\n fn value(key: &[u8], value: &[u8]) -> String {\n let Ok(key) = mvcc::Key::decode(key) else {\n return Raw::bytes(value); // invalid key\n };\n match key {\n mvcc::Key::NextVersion => {\n let Ok(version) = bincode::deserialize::(value) else {\n return Raw::bytes(value);\n };\n version.to_string()\n }\n mvcc::Key::TxnActiveSnapshot(_) => {\n let Ok(active) = bincode::deserialize::>(value) else {\n return Raw::bytes(value);\n };\n format!(\"{{{}}}\", active.iter().join(\",\"))\n }\n mvcc::Key::TxnActive(_) | mvcc::Key::TxnWrite(_, _) => Raw::bytes(value),\n mvcc::Key::Version(userkey, _) => match bincode::deserialize(value) {\n Ok(Some(value)) => F::value(&userkey, value),\n Ok(None) => \"None\".to_string(),\n Err(_) => Raw::bytes(value),\n },\n mvcc::Key::Unversioned(userkey) => F::value(&userkey, value),\n }\n }\n}\n\n/// Formats SQL keys/values.\npub struct SQL;\n\nimpl SQL {\n /// Formats a list of SQL values.\n fn values(values: impl IntoIterator) -> String {\n values.into_iter().join(\",\")\n }\n\n /// Formats a table schema.\n fn schema(table: sql::types::Table) -> String {\n // Put it all on a single line.\n let re = Regex::new(r#\"\\n\\s*\"#).expect(\"invalid regex\");\n re.replace_all(&table.to_string(), \" \").into_owned()\n }\n}\n\nimpl Formatter for SQL {\n fn key(key: &[u8]) -> String {\n // Special-case the Raft applied index key.\n if key == sql::engine::Raft::APPLIED_INDEX_KEY {\n return String::from_utf8_lossy(key).into_owned();\n }\n let Ok(key) = sql::engine::Key::decode(key) else {\n return Raw::key(key); // invalid key\n };\n match key {\n sql::engine::Key::Table(name) => format!(\"sql:Table({name})\"),\n sql::engine::Key::Index(table, column, value) => {\n format!(\"sql:Index({table}.{column}, {value})\")\n }\n sql::engine::Key::Row(table, id) => {\n format!(\"sql:Row({table}, {id})\")\n }\n }\n }\n\n fn value(key: &[u8], value: &[u8]) -> String {\n // Special-case the applied_index key.\n if key == sql::engine::Raft::APPLIED_INDEX_KEY\n && let Ok(applied_index) = bincode::deserialize::(value)\n {\n return applied_index.to_string();\n }\n\n let Ok(key) = sql::engine::Key::decode(key) else {\n return Raw::key(value);\n };\n match key {\n sql::engine::Key::Table(_) => {\n let Ok(table) = bincode::deserialize(value) else {\n return Raw::bytes(value);\n };\n Self::schema(table)\n }\n sql::engine::Key::Row(_, _) => {\n let Ok(row) = bincode::deserialize::(value) else {\n return Raw::bytes(value);\n };\n Self::values(row)\n }\n sql::engine::Key::Index(_, _, _) => {\n let Ok(index) = bincode::deserialize::>(value) else {\n return Raw::bytes(value);\n };\n Self::values(index)\n }\n }\n }\n}\n\n/// Formats SQL Raft write commands, from the Raft log.\npub struct SQLCommand;\n\nimpl Formatter for SQLCommand {\n fn key(_key: &[u8]) -> String {\n // There is no key, since these are wrapped in a Raft log entry.\n panic!(\"SQL commands don't have a key\");\n }\n\n fn value(_key: &[u8], value: &[u8]) -> String {\n let Ok(write) = sql::engine::Write::decode(value) else {\n return Raw::bytes(value);\n };\n\n let txn = match &write {\n sql::engine::Write::Begin => None,\n sql::engine::Write::Commit(txn)\n | sql::engine::Write::Rollback(txn)\n | sql::engine::Write::Delete { txn, .. }\n | sql::engine::Write::Insert { txn, .. }\n | sql::engine::Write::Update { txn, .. }\n | sql::engine::Write::CreateTable { txn, .. }\n | sql::engine::Write::DropTable { txn, .. } => Some(txn),\n };\n let fmttxn =\n txn.filter(|t| !t.read_only).map_or(\"\".to_string(), |t| format!(\"t{} \", t.version));\n\n let fmtcommand = match write {\n sql::engine::Write::Begin => \"BEGIN\".to_string(),\n sql::engine::Write::Commit(_) => \"COMMIT\".to_string(),\n sql::engine::Write::Rollback(_) => \"ROLLBACK\".to_string(),\n sql::engine::Write::Delete { table, ids, .. } => {\n format!(\"DELETE {table} {}\", ids.iter().map(|id| id.to_string()).join(\",\"))\n }\n sql::engine::Write::Insert { table, rows, .. } => {\n format!(\n \"INSERT {table} {}\",\n rows.into_iter().map(|row| format!(\"({})\", SQL::values(row))).join(\" \")\n )\n }\n sql::engine::Write::Update { table, rows, .. } => format!(\n \"UPDATE {table} {}\",\n rows.into_iter().map(|(id, row)| format!(\"{id}→({})\", SQL::values(row))).join(\" \")\n ),\n sql::engine::Write::CreateTable { schema, .. } => SQL::schema(schema),\n sql::engine::Write::DropTable { table, .. } => format!(\"DROP TABLE {table}\"),\n };\n format!(\"{fmttxn}{fmtcommand}\")\n }\n}\n" }, { "path": "src/encoding/keycode.rs", "content": "//! Keycode is a lexicographical order-preserving binary encoding for use with\n//! keys in key/value stores. It is designed for simplicity, not efficiency\n//! (i.e. it does not use varints or other compression methods).\n//!\n//! Ordering is important because it allows limited scans across specific parts\n//! of the keyspace, e.g. scanning an individual table or using an index range\n//! predicate like `WHERE id < 100`. It also avoids sorting in some cases where\n//! the keys are already in the desired order, e.g. in the Raft log.\n//!\n//! The encoding is not self-describing: the caller must provide a concrete type\n//! to decode into, and the binary key must conform to its structure.\n//!\n//! Keycode supports a subset of primitive data types, encoded as follows:\n//!\n//! * [`bool`]: `0x00` for `false`, `0x01` for `true`.\n//! * [`u64`]: big-endian binary representation.\n//! * [`i64`]: big-endian binary, sign bit flipped.\n//! * [`f64`]: big-endian binary, sign bit flipped, all flipped if negative.\n//! * [`Vec`]: `0x00` escaped as `0x00ff`, terminated with `0x0000`.\n//! * [`String`]: like [`Vec`].\n//! * Sequences: concatenation of contained elements, with no other structure.\n//! * Enum: the variant's index as [`u8`], then the content sequence.\n//! * [`crate::sql::types::Value`]: like any other enum.\n//!\n//! The canonical key representation is an enum. For example:\n//!\n//! ```\n//! #[derive(Debug, Deserialize, Serialize)]\n//! enum Key {\n//! Foo,\n//! Bar(String),\n//! Baz(bool, u64, #[serde(with = \"serde_bytes\")] Vec),\n//! }\n//! ```\n//!\n//! Unfortunately, byte strings such as `Vec` must be wrapped with\n//! [`serde_bytes::ByteBuf`] or use the `#[serde(with=\"serde_bytes\")]`\n//! attribute. See .\n\nuse std::ops::Bound;\n\nuse itertools::Either;\nuse serde::de::{\n Deserialize, DeserializeSeed, EnumAccess, IntoDeserializer as _, SeqAccess, VariantAccess,\n Visitor,\n};\nuse serde::ser::{Impossible, Serialize, SerializeSeq, SerializeTuple, SerializeTupleVariant};\n\nuse crate::errdata;\nuse crate::error::{Error, Result};\n\n/// Serializes a key to a binary Keycode representation.\n///\n/// In the common case, the encoded key is borrowed for a storage engine call\n/// and then thrown away. We could avoid a bunch of allocations by taking a\n/// reusable byte vector to encode into and return a reference to it, but we\n/// keep it simple.\npub fn serialize(key: &T) -> Vec {\n let mut serializer = Serializer { output: Vec::new() };\n // Panic on failure, as this is a problem with the data structure.\n key.serialize(&mut serializer).expect(\"key must be serializable\");\n serializer.output\n}\n\n/// Deserializes a key from a binary Keycode representation.\npub fn deserialize<'a, T: Deserialize<'a>>(input: &'a [u8]) -> Result {\n let mut deserializer = Deserializer::from_bytes(input);\n let t = T::deserialize(&mut deserializer)?;\n if !deserializer.input.is_empty() {\n return errdata!(\n \"unexpected trailing bytes {:x?} at end of key {input:x?}\",\n deserializer.input,\n );\n }\n Ok(t)\n}\n\n/// Generates a key range for a key prefix, used e.g. for prefix scans.\n///\n/// The exclusive end bound is generated by adding 1 to the value of the last\n/// byte. If the last byte(s) is 0xff (so adding 1 would overflow), we instead\n/// find the latest non-0xff byte, increment that, and truncate the rest. If all\n/// bytes are 0xff, we scan to the end of the range, since there can't be other\n/// prefixes after it.\npub fn prefix_range(prefix: &[u8]) -> (Bound>, Bound>) {\n let start = Bound::Included(prefix.to_vec());\n let end = match prefix.iter().rposition(|&b| b != 0xff) {\n Some(i) => Bound::Excluded(\n prefix.iter().take(i).copied().chain(std::iter::once(prefix[i] + 1)).collect(),\n ),\n None => Bound::Unbounded,\n };\n (start, end)\n}\n\n/// Serializes keys as binary byte vectors.\nstruct Serializer {\n output: Vec,\n}\n\nimpl serde::ser::Serializer for &mut Serializer {\n type Ok = ();\n type Error = Error;\n\n type SerializeSeq = Self;\n type SerializeTuple = Self;\n type SerializeTupleVariant = Self;\n type SerializeTupleStruct = Impossible<(), Error>;\n type SerializeMap = Impossible<(), Error>;\n type SerializeStruct = Impossible<(), Error>;\n type SerializeStructVariant = Impossible<(), Error>;\n\n /// bool simply uses 1 for true and 0 for false.\n fn serialize_bool(self, v: bool) -> Result<()> {\n self.output.push(if v { 1 } else { 0 });\n Ok(())\n }\n\n fn serialize_i8(self, _: i8) -> Result<()> {\n unimplemented!()\n }\n\n fn serialize_i16(self, _: i16) -> Result<()> {\n unimplemented!()\n }\n\n fn serialize_i32(self, _: i32) -> Result<()> {\n unimplemented!()\n }\n\n /// i64 uses the big-endian two's complement encoding, but flips the\n /// left-most sign bit such that negative numbers are ordered before\n /// positive numbers.\n ///\n /// The relative ordering of the remaining bits is already correct: -1, the\n /// largest negative integer, is encoded as 01111111...11111111, ordered\n /// after all other negative integers but before positive integers.\n fn serialize_i64(self, v: i64) -> Result<()> {\n let mut bytes = v.to_be_bytes();\n bytes[0] ^= 1 << 7; // flip sign bit\n self.output.extend(bytes);\n Ok(())\n }\n\n fn serialize_u8(self, _: u8) -> Result<()> {\n unimplemented!()\n }\n\n fn serialize_u16(self, _: u16) -> Result<()> {\n unimplemented!()\n }\n\n fn serialize_u32(self, _: u32) -> Result<()> {\n unimplemented!()\n }\n\n /// u64 simply uses the big-endian encoding.\n fn serialize_u64(self, v: u64) -> Result<()> {\n self.output.extend(v.to_be_bytes());\n Ok(())\n }\n\n fn serialize_f32(self, _: f32) -> Result<()> {\n unimplemented!()\n }\n\n /// f64 is encoded in big-endian IEEE 754 form, but it flips the sign bit to\n /// order positive numbers after negative numbers, and also flips all other\n /// bits for negative numbers to order them from smallest to largest. NaN is\n /// ordered at the end.\n fn serialize_f64(self, v: f64) -> Result<()> {\n let mut bytes = v.to_be_bytes();\n match v.is_sign_negative() {\n false => bytes[0] ^= 1 << 7, // positive, flip sign bit\n true => bytes.iter_mut().for_each(|b| *b = !*b), // negative, flip all bits\n }\n self.output.extend(bytes);\n Ok(())\n }\n\n fn serialize_char(self, _: char) -> Result<()> {\n unimplemented!()\n }\n\n // Strings are encoded like bytes.\n fn serialize_str(self, v: &str) -> Result<()> {\n self.serialize_bytes(v.as_bytes())\n }\n\n // Byte slices are terminated by 0x0000, escaping 0x00 as 0x00ff. This\n // ensures that we can detect the end, and that for two overlapping slices,\n // the shorter one orders before the longer one.\n //\n // We can't use e.g. length prefix encoding, since it doesn't sort correctly.\n fn serialize_bytes(self, v: &[u8]) -> Result<()> {\n let bytes = v\n .iter()\n .flat_map(|&byte| match byte {\n 0x00 => Either::Left([0x00, 0xff].into_iter()),\n byte => Either::Right([byte].into_iter()),\n })\n .chain([0x00, 0x00]);\n self.output.extend(bytes);\n Ok(())\n }\n\n fn serialize_none(self) -> Result<()> {\n unimplemented!()\n }\n\n fn serialize_some(self, _: &T) -> Result<()> {\n unimplemented!()\n }\n\n fn serialize_unit(self) -> Result<()> {\n unimplemented!()\n }\n\n fn serialize_unit_struct(self, _: &'static str) -> Result<()> {\n unimplemented!()\n }\n\n /// Enum variants are serialized using their index, as a single byte.\n fn serialize_unit_variant(self, _: &'static str, index: u32, _: &'static str) -> Result<()> {\n self.output.push(index.try_into()?);\n Ok(())\n }\n\n fn serialize_newtype_struct(self, _: &'static str, _: &T) -> Result<()> {\n unimplemented!()\n }\n\n /// Newtype variants are serialized using the variant index and inner type.\n fn serialize_newtype_variant(\n self,\n name: &'static str,\n index: u32,\n variant: &'static str,\n value: &T,\n ) -> Result<()> {\n self.serialize_unit_variant(name, index, variant)?;\n value.serialize(self)\n }\n\n /// Sequences are serialized as the concatenation of the serialized elements.\n fn serialize_seq(self, _: Option) -> Result {\n Ok(self)\n }\n\n /// Tuples are serialized as the concatenation of the serialized elements.\n fn serialize_tuple(self, _: usize) -> Result {\n Ok(self)\n }\n\n fn serialize_tuple_struct(\n self,\n _: &'static str,\n _: usize,\n ) -> Result {\n unimplemented!()\n }\n\n /// Tuple variants are serialized using the variant index and the\n /// concatenation of the serialized elements.\n fn serialize_tuple_variant(\n self,\n name: &'static str,\n index: u32,\n variant: &'static str,\n _: usize,\n ) -> Result {\n self.serialize_unit_variant(name, index, variant)?;\n Ok(self)\n }\n\n fn serialize_map(self, _: Option) -> Result {\n unimplemented!()\n }\n\n fn serialize_struct(self, _: &'static str, _: usize) -> Result {\n unimplemented!()\n }\n\n fn serialize_struct_variant(\n self,\n _: &'static str,\n _: u32,\n _: &'static str,\n _: usize,\n ) -> Result {\n unimplemented!()\n }\n}\n\n/// Sequences simply concatenate the serialized elements, with no external structure.\nimpl SerializeSeq for &mut Serializer {\n type Ok = ();\n type Error = Error;\n\n fn serialize_element(&mut self, value: &T) -> Result<()> {\n value.serialize(&mut **self)\n }\n\n fn end(self) -> Result<()> {\n Ok(())\n }\n}\n\n/// Tuples, like sequences, simply concatenate the serialized elements.\nimpl SerializeTuple for &mut Serializer {\n type Ok = ();\n type Error = Error;\n\n fn serialize_element(&mut self, value: &T) -> Result<()> {\n value.serialize(&mut **self)\n }\n\n fn end(self) -> Result<()> {\n Ok(())\n }\n}\n\n/// Tuples, like sequences, simply concatenate the serialized elements.\nimpl SerializeTupleVariant for &mut Serializer {\n type Ok = ();\n type Error = Error;\n\n fn serialize_field(&mut self, value: &T) -> Result<()> {\n value.serialize(&mut **self)\n }\n\n fn end(self) -> Result<()> {\n Ok(())\n }\n}\n\n/// Deserializes keys from byte slices into a given type. The format is not\n/// self-describing, so the caller must provide a concrete type to deserialize\n/// into.\npub struct Deserializer<'de> {\n input: &'de [u8],\n}\n\nimpl<'de> Deserializer<'de> {\n /// Creates a deserializer for a byte slice.\n pub fn from_bytes(input: &'de [u8]) -> Self {\n Deserializer { input }\n }\n\n /// Chops off and returns the next len bytes of the byte slice, or errors if\n /// there aren't enough bytes left.\n fn take_bytes(&mut self, len: usize) -> Result<&[u8]> {\n if self.input.len() < len {\n return errdata!(\"insufficient bytes, expected {len} bytes for {:x?}\", self.input);\n }\n let bytes = &self.input[..len];\n self.input = &self.input[len..];\n Ok(bytes)\n }\n\n /// Decodes and chops off the next encoded byte slice.\n fn decode_next_bytes(&mut self) -> Result> {\n let mut decoded = Vec::new();\n let mut iter = self.input.iter().enumerate();\n let taken = loop {\n match iter.next() {\n Some((_, 0x00)) => match iter.next() {\n Some((i, 0x00)) => break i + 1, // terminator\n Some((_, 0xff)) => decoded.push(0x00), // escaped 0x00\n _ => return errdata!(\"invalid escape sequence\"),\n },\n Some((_, b)) => decoded.push(*b),\n None => return errdata!(\"unexpected end of input\"),\n }\n };\n self.input = &self.input[taken..];\n Ok(decoded)\n }\n}\n\n/// For details on serialization formats, see Serializer.\nimpl<'de> serde::de::Deserializer<'de> for &mut Deserializer<'de> {\n type Error = Error;\n\n fn deserialize_any>(self, _: V) -> Result {\n panic!(\"must provide type, Keycode is not self-describing\")\n }\n\n fn deserialize_bool>(self, visitor: V) -> Result {\n visitor.visit_bool(match self.take_bytes(1)?[0] {\n 0x00 => false,\n 0x01 => true,\n b => return errdata!(\"invalid boolean value {b}\"),\n })\n }\n\n fn deserialize_i8>(self, _: V) -> Result {\n unimplemented!()\n }\n\n fn deserialize_i16>(self, _: V) -> Result {\n unimplemented!()\n }\n\n fn deserialize_i32>(self, _: V) -> Result {\n unimplemented!()\n }\n\n fn deserialize_i64>(self, visitor: V) -> Result {\n let mut bytes = self.take_bytes(8)?.to_vec();\n bytes[0] ^= 1 << 7; // flip sign bit\n visitor.visit_i64(i64::from_be_bytes(bytes.as_slice().try_into()?))\n }\n\n fn deserialize_u8>(self, _: V) -> Result {\n unimplemented!()\n }\n\n fn deserialize_u16>(self, _: V) -> Result {\n unimplemented!()\n }\n\n fn deserialize_u32>(self, _: V) -> Result {\n unimplemented!()\n }\n\n fn deserialize_u64>(self, visitor: V) -> Result {\n visitor.visit_u64(u64::from_be_bytes(self.take_bytes(8)?.try_into()?))\n }\n\n fn deserialize_f32>(self, _: V) -> Result {\n unimplemented!()\n }\n\n fn deserialize_f64>(self, visitor: V) -> Result {\n let mut bytes = self.take_bytes(8)?.to_vec();\n match bytes[0] >> 7 {\n 0 => bytes.iter_mut().for_each(|b| *b = !*b), // negative, flip all bits\n 1 => bytes[0] ^= 1 << 7, // positive, flip sign bit\n _ => panic!(\"bits can only be 0 or 1\"),\n }\n visitor.visit_f64(f64::from_be_bytes(bytes.as_slice().try_into()?))\n }\n\n fn deserialize_char>(self, _: V) -> Result {\n unimplemented!()\n }\n\n fn deserialize_str>(self, visitor: V) -> Result {\n let bytes = self.decode_next_bytes()?;\n visitor.visit_str(&String::from_utf8(bytes)?)\n }\n\n fn deserialize_string>(self, visitor: V) -> Result {\n let bytes = self.decode_next_bytes()?;\n visitor.visit_string(String::from_utf8(bytes)?)\n }\n\n fn deserialize_bytes>(self, visitor: V) -> Result {\n let bytes = self.decode_next_bytes()?;\n visitor.visit_bytes(&bytes)\n }\n\n fn deserialize_byte_buf>(self, visitor: V) -> Result {\n let bytes = self.decode_next_bytes()?;\n visitor.visit_byte_buf(bytes)\n }\n\n fn deserialize_option>(self, _: V) -> Result {\n unimplemented!()\n }\n\n fn deserialize_unit>(self, _: V) -> Result {\n unimplemented!()\n }\n\n fn deserialize_unit_struct>(self, _: &'static str, _: V) -> Result {\n unimplemented!()\n }\n\n fn deserialize_newtype_struct>(\n self,\n _: &'static str,\n _: V,\n ) -> Result {\n unimplemented!()\n }\n\n fn deserialize_seq>(self, visitor: V) -> Result {\n visitor.visit_seq(self)\n }\n\n fn deserialize_tuple>(self, _: usize, visitor: V) -> Result {\n visitor.visit_seq(self)\n }\n\n fn deserialize_tuple_struct>(\n self,\n _: &'static str,\n _: usize,\n _: V,\n ) -> Result {\n unimplemented!()\n }\n\n fn deserialize_map>(self, _: V) -> Result {\n unimplemented!()\n }\n\n fn deserialize_struct>(\n self,\n _: &'static str,\n _: &'static [&'static str],\n _: V,\n ) -> Result {\n unimplemented!()\n }\n\n fn deserialize_enum>(\n self,\n _: &'static str,\n _: &'static [&'static str],\n visitor: V,\n ) -> Result {\n visitor.visit_enum(self)\n }\n\n fn deserialize_identifier>(self, _: V) -> Result {\n unimplemented!()\n }\n\n fn deserialize_ignored_any>(self, _: V) -> Result {\n unimplemented!()\n }\n}\n\n/// Sequences are simply deserialized until the byte slice is exhausted.\nimpl<'de> SeqAccess<'de> for Deserializer<'de> {\n type Error = Error;\n\n fn next_element_seed>(&mut self, seed: T) -> Result> {\n if self.input.is_empty() {\n return Ok(None);\n }\n seed.deserialize(self).map(Some)\n }\n}\n\n/// Enum variants are deserialized by their index.\nimpl<'de> EnumAccess<'de> for &mut Deserializer<'de> {\n type Error = Error;\n type Variant = Self;\n\n fn variant_seed>(self, seed: V) -> Result<(V::Value, Self::Variant)> {\n let index = self.take_bytes(1)?[0] as u32;\n let value: Result<_> = seed.deserialize(index.into_deserializer());\n Ok((value?, self))\n }\n}\n\n/// Enum variant contents are deserialized as sequences.\nimpl<'de> VariantAccess<'de> for &mut Deserializer<'de> {\n type Error = Error;\n\n fn unit_variant(self) -> Result<()> {\n Ok(())\n }\n\n fn newtype_variant_seed>(self, seed: T) -> Result {\n seed.deserialize(&mut *self)\n }\n\n fn tuple_variant>(self, _: usize, visitor: V) -> Result {\n visitor.visit_seq(self)\n }\n\n fn struct_variant>(self, _: &'static [&'static str], _: V) -> Result {\n unimplemented!()\n }\n}\n\n#[cfg(test)]\nmod tests {\n use std::borrow::Cow;\n use std::f64::consts::PI;\n\n use paste::paste;\n use serde::{Deserialize, Serialize};\n use serde_bytes::ByteBuf;\n\n use super::*;\n use crate::sql::types::Value;\n\n #[derive(Debug, Deserialize, Serialize, PartialEq)]\n enum Key<'a> {\n Unit,\n NewType(String),\n Tuple(bool, #[serde(with = \"serde_bytes\")] Vec, u64),\n Cow(\n #[serde(with = \"serde_bytes\")]\n #[serde(borrow)]\n Cow<'a, [u8]>,\n bool,\n #[serde(borrow)] Cow<'a, str>,\n ),\n }\n\n /// Assert that serializing a value yields the expected byte sequence (as a\n /// hex-encoded string), and that deserializing it yields the original value.\n macro_rules! test_serialize_deserialize {\n ( $( $name:ident: $input:expr => $expect:literal, )* ) => {\n $(\n #[test]\n fn $name() -> Result<()> {\n let mut input = $input;\n let expect = $expect;\n let output = serialize(&input);\n assert_eq!(hex::encode(&output), expect, \"encode failed\");\n\n let expect = input;\n input = deserialize(&output)?; // reuse input variable for proper type\n assert_eq!(input, expect, \"decode failed\");\n Ok(())\n }\n )*\n };\n }\n\n /// Assert that deserializing invalid inputs results in errors. Takes byte\n /// slices (as hex-encoded strings) and the type to deserialize into.\n macro_rules! test_deserialize_error {\n ( $( $name:ident: $input:literal as $type:ty, )* ) => {\n paste! {\n $(\n #[test]\n #[should_panic]\n fn [< $name _deserialize_error >]() {\n let bytes = hex::decode($input).unwrap();\n deserialize::<$type>(&bytes).unwrap();\n }\n )*\n }\n };\n }\n\n // Assert that serializing a value results in an error.\n macro_rules! test_serialize_error {\n ( $( $name:ident: $input:expr, )* ) => {\n paste! {\n $(\n #[test]\n #[should_panic]\n fn [< $name _serialize_error >]() {\n let input = $input;\n serialize(&input);\n }\n )*\n }\n };\n }\n\n test_serialize_deserialize! {\n bool_false: false => \"00\",\n bool_true: true => \"01\",\n\n f64_min: f64::MIN => \"0010000000000000\",\n f64_neg_inf: f64::NEG_INFINITY => \"000fffffffffffff\",\n f64_neg_pi: -PI => \"3ff6de04abbbd2e7\",\n f64_neg_zero: -0f64 => \"7fffffffffffffff\",\n f64_zero: 0f64 => \"8000000000000000\",\n f64_pi: PI => \"c00921fb54442d18\",\n f64_max: f64::MAX => \"ffefffffffffffff\",\n f64_inf: f64::INFINITY => \"fff0000000000000\",\n // We don't test NAN here, since NAN != NAN.\n\n i64_min: i64::MIN => \"0000000000000000\",\n i64_neg_65535: -65535i64 => \"7fffffffffff0001\",\n i64_neg_1: -1i64 => \"7fffffffffffffff\",\n i64_0: 0i64 => \"8000000000000000\",\n i64_1: 1i64 => \"8000000000000001\",\n i64_65535: 65535i64 => \"800000000000ffff\",\n i64_max: i64::MAX => \"ffffffffffffffff\",\n\n u64_min: u64::MIN => \"0000000000000000\",\n u64_1: 1_u64 => \"0000000000000001\",\n u64_65535: 65535_u64 => \"000000000000ffff\",\n u64_max: u64::MAX => \"ffffffffffffffff\",\n\n bytes: ByteBuf::from(vec![0x01, 0xff]) => \"01ff0000\",\n bytes_empty: ByteBuf::new() => \"0000\",\n bytes_escape: ByteBuf::from(vec![0x00, 0x01, 0x02]) => \"00ff01020000\",\n\n string: \"foo\".to_string() => \"666f6f0000\",\n string_empty: \"\".to_string() => \"0000\",\n string_escape: \"foo\\x00bar\".to_string() => \"666f6f00ff6261720000\",\n string_utf8: \"👋\".to_string() => \"f09f918b0000\",\n\n tuple: (true, u64::MAX, ByteBuf::from(vec![0x00, 0x01])) => \"01ffffffffffffffff00ff010000\",\n array_bool: [false, true, false] => \"000100\",\n vec_bool: vec![false, true, false] => \"000100\",\n vec_u64: vec![u64::MIN, u64::MAX, 65535_u64] => \"0000000000000000ffffffffffffffff000000000000ffff\",\n\n enum_unit: Key::Unit => \"00\",\n enum_newtype: Key::NewType(\"foo\".to_string()) => \"01666f6f0000\",\n enum_tuple: Key::Tuple(false, vec![0x00, 0x01], u64::MAX) => \"020000ff010000ffffffffffffffff\",\n enum_cow: Key::Cow(vec![0x00, 0x01].into(), false, String::from(\"foo\").into()) => \"0300ff01000000666f6f0000\",\n enum_cow_borrow: Key::Cow([0x00, 0x01].as_slice().into(), false, \"foo\".into()) => \"0300ff01000000666f6f0000\",\n\n value_null: Value::Null => \"00\",\n value_bool: Value::Boolean(true) => \"0101\",\n value_int: Value::Integer(-1) => \"027fffffffffffffff\",\n value_float: Value::Float(PI) => \"03c00921fb54442d18\",\n value_string: Value::String(\"foo\".to_string()) => \"04666f6f0000\",\n }\n\n test_serialize_error! {\n char: 'a',\n f32: 0f32,\n i8: 0i8,\n i16: 0i16,\n i32: 0i32,\n i128: 0i128,\n u8: 0u8,\n u16: 0u16,\n u32: 0u32,\n u128: 0u128,\n some: Some(true),\n none: Option::::None,\n vec_u8: vec![0u8],\n }\n\n test_deserialize_error! {\n bool_empty: \"\" as bool,\n bool_2: \"02\" as bool,\n char: \"61\" as char,\n f32: \"00000000\" as f32,\n i8: \"00\" as i8,\n i16: \"0000\" as i16,\n i32: \"00000000\" as i32,\n i128: \"00000000000000000000000000000000\" as i128,\n u16: \"0000\" as u16,\n u32: \"00000000\" as u32,\n u64_partial: \"0000\" as u64,\n u128: \"00000000000000000000000000000000\" as u128,\n option: \"00\" as Option::,\n string_utf8_invalid: \"c0\" as String,\n tuple_partial: \"0001\" as (bool, bool, bool),\n vec_u8: \"0000\" as Vec,\n }\n}\n" }, { "path": "src/encoding/mod.rs", "content": "//! Binary data encodings.\n//!\n//! * keycode: used for keys in the key/value store.\n//! * bincode: used for values in the key/value store and network protocols.\n\npub mod bincode;\npub mod format;\npub mod keycode;\n\nuse std::cmp::{Eq, Ord};\nuse std::collections::{BTreeSet, HashSet};\nuse std::hash::Hash;\nuse std::io::{Read, Write};\n\nuse serde::de::DeserializeOwned;\nuse serde::{Deserialize, Serialize};\n\nuse crate::error::Result;\n\n/// Adds automatic Keycode encode/decode methods to key enums. These are used\n/// as keys in the key/value store.\npub trait Key<'de>: Serialize + Deserialize<'de> {\n /// Decodes a key from a byte slice using Keycode.\n fn decode(bytes: &'de [u8]) -> Result {\n keycode::deserialize(bytes)\n }\n\n /// Encodes a key to a byte vector using Keycode.\n ///\n /// In the common case, the encoded key is borrowed for a storage engine\n /// call and then thrown away. We could avoid a bunch of allocations by\n /// taking a reusable byte vector to encode into and return a reference to\n /// it, but we keep it simple.\n fn encode(&self) -> Vec {\n keycode::serialize(self)\n }\n}\n\n/// Adds automatic Bincode encode/decode methods to value types. These are used\n/// for values in key/value storage engines, and also for e.g. network protocol\n/// messages and other values.\npub trait Value: Serialize + DeserializeOwned {\n /// Decodes a value from a byte slice using Bincode.\n fn decode(bytes: &[u8]) -> Result {\n bincode::deserialize(bytes)\n }\n\n /// Decodes a value from a reader using Bincode.\n fn decode_from(reader: R) -> Result {\n bincode::deserialize_from(reader)\n }\n\n /// Decodes a value from a reader using Bincode, or returns None if the\n /// reader is closed.\n fn maybe_decode_from(reader: R) -> Result> {\n bincode::maybe_deserialize_from(reader)\n }\n\n /// Encodes a value to a byte vector using Bincode.\n fn encode(&self) -> Vec {\n bincode::serialize(self)\n }\n\n /// Encodes a value into a writer using Bincode.\n fn encode_into(&self, writer: W) -> Result<()> {\n bincode::serialize_into(writer, self)\n }\n}\n\n/// Blanket implementations for various types wrapping a value type.\nimpl Value for Option {}\nimpl Value for Result {}\nimpl Value for Vec {}\nimpl Value for (V1, V2) {}\nimpl Value for HashSet {}\nimpl Value for BTreeSet {}\n" }, { "path": "src/error.rs", "content": "use std::fmt::Display;\n\nuse serde::{Deserialize, Serialize};\n\n/// toyDB errors.\n#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]\npub enum Error {\n /// The operation was aborted and must be retried. This typically happens\n /// with e.g. Raft leader changes. This is used instead of implementing\n /// complex retry logic and replay protection in Raft.\n Abort,\n /// Invalid data, typically decoding errors or unexpected internal values.\n InvalidData(String),\n /// Invalid user input, typically parser or query errors.\n InvalidInput(String),\n /// An IO error.\n IO(String),\n /// A write was attempted in a read-only transaction.\n ReadOnly,\n /// A write transaction conflicted with a different writer and lost. The\n /// transaction must be retried.\n Serialization,\n}\n\nimpl std::error::Error for Error {}\n\nimpl Display for Error {\n fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {\n match self {\n Error::Abort => write!(f, \"operation aborted\"),\n Error::InvalidData(msg) => write!(f, \"invalid data: {msg}\"),\n Error::InvalidInput(msg) => write!(f, \"invalid input: {msg}\"),\n Error::IO(msg) => write!(f, \"io error: {msg}\"),\n Error::ReadOnly => write!(f, \"read-only transaction\"),\n Error::Serialization => write!(f, \"serialization failure, retry transaction\"),\n }\n }\n}\n\nimpl Error {\n /// Returns whether the error is considered deterministic. Raft state\n /// machine application needs to know whether a command failure is\n /// deterministic on the input command -- if it is, the command can be\n /// considered applied and the error returned to the client, but otherwise\n /// the state machine must panic to prevent node divergence.\n pub fn is_deterministic(&self) -> bool {\n match self {\n // Aborts don't happen during application, only leader changes. But\n // we consider them non-deterministic in case an abort should happen\n // unexpectedly below Raft.\n Error::Abort => false,\n // Possible data corruption local to this node.\n Error::InvalidData(_) => false,\n // Input errors are (likely) deterministic. They might not be in\n // case data was corrupted in flight, but we ignore this case.\n Error::InvalidInput(_) => true,\n // IO errors are typically local to the node (e.g. faulty disk).\n Error::IO(_) => false,\n // Write commands in read-only transactions are deterministic.\n Error::ReadOnly => true,\n // Write conflicts are determinstic.\n Error::Serialization => true,\n }\n }\n}\n\n/// Constructs an Error::InvalidData for the given format string.\n#[macro_export]\nmacro_rules! errdata {\n ($($args:tt)*) => { $crate::error::Error::InvalidData(format!($($args)*)).into() };\n}\n\n/// Constructs an Error::InvalidInput for the given format string.\n#[macro_export]\nmacro_rules! errinput {\n ($($args:tt)*) => { $crate::error::Error::InvalidInput(format!($($args)*)).into() };\n}\n\n/// A toyDB Result returning Error.\npub type Result = std::result::Result;\n\nimpl From for Result {\n fn from(error: Error) -> Self {\n Err(error)\n }\n}\n\nimpl serde::de::Error for Error {\n fn custom(msg: T) -> Self {\n Error::InvalidData(msg.to_string())\n }\n}\n\nimpl serde::ser::Error for Error {\n fn custom(msg: T) -> Self {\n Error::InvalidData(msg.to_string())\n }\n}\n\nimpl From for Error {\n fn from(err: bincode::error::DecodeError) -> Self {\n Error::InvalidData(err.to_string())\n }\n}\n\nimpl From for Error {\n fn from(err: bincode::error::EncodeError) -> Self {\n Error::InvalidData(err.to_string())\n }\n}\n\nimpl From for Error {\n fn from(err: config::ConfigError) -> Self {\n Error::InvalidInput(err.to_string())\n }\n}\n\nimpl From for Error {\n fn from(err: crossbeam::channel::RecvError) -> Self {\n Error::IO(err.to_string())\n }\n}\n\nimpl From> for Error {\n fn from(err: crossbeam::channel::SendError) -> Self {\n Error::IO(err.to_string())\n }\n}\n\nimpl From for Error {\n fn from(err: crossbeam::channel::TryRecvError) -> Self {\n Error::IO(err.to_string())\n }\n}\n\nimpl From> for Error {\n fn from(err: crossbeam::channel::TrySendError) -> Self {\n Error::IO(err.to_string())\n }\n}\n\nimpl From for Error {\n fn from(err: hdrhistogram::CreationError) -> Self {\n panic!(\"{err}\") // faulty code\n }\n}\n\nimpl From for Error {\n fn from(err: hdrhistogram::RecordError) -> Self {\n Error::InvalidInput(err.to_string())\n }\n}\n\nimpl From for Error {\n fn from(err: log::ParseLevelError) -> Self {\n Error::InvalidInput(err.to_string())\n }\n}\n\nimpl From for Error {\n fn from(err: log::SetLoggerError) -> Self {\n panic!(\"{err}\") // faulty code\n }\n}\n\nimpl From for Error {\n fn from(err: rand::distr::uniform::Error) -> Self {\n Error::InvalidInput(err.to_string())\n }\n}\n\nimpl From for Error {\n fn from(err: regex::Error) -> Self {\n panic!(\"{err}\") // faulty code\n }\n}\n\nimpl From for Error {\n fn from(err: rustyline::error::ReadlineError) -> Self {\n Error::IO(err.to_string())\n }\n}\n\nimpl From for Error {\n fn from(err: std::array::TryFromSliceError) -> Self {\n Error::InvalidData(err.to_string())\n }\n}\n\nimpl From for Error {\n fn from(err: std::io::Error) -> Self {\n Error::IO(err.to_string())\n }\n}\n\nimpl From for Error {\n fn from(err: std::num::ParseFloatError) -> Self {\n Error::InvalidInput(err.to_string())\n }\n}\n\nimpl From for Error {\n fn from(err: std::num::ParseIntError) -> Self {\n Error::InvalidInput(err.to_string())\n }\n}\n\nimpl From for Error {\n fn from(err: std::num::TryFromIntError) -> Self {\n Error::InvalidData(err.to_string())\n }\n}\n\nimpl From for Error {\n fn from(err: std::string::FromUtf8Error) -> Self {\n Error::InvalidData(err.to_string())\n }\n}\n\nimpl From> for Error {\n fn from(err: std::sync::PoisonError) -> Self {\n // This only happens when a different thread panics while holding a\n // mutex. This should be fatal, so we panic here too.\n panic!(\"{err}\")\n }\n}\n" }, { "path": "src/lib.rs", "content": "#![warn(clippy::all)]\n#![allow(clippy::large_enum_variant)]\n#![allow(clippy::module_inception)]\n#![allow(clippy::type_complexity)]\n\npub mod client;\npub mod encoding;\npub mod error;\npub mod raft;\npub mod server;\npub mod sql;\npub mod storage;\n\npub use client::Client;\npub use server::Server;\npub use sql::execution::StatementResult;\n" }, { "path": "src/raft/log.rs", "content": "use std::ops::{Bound, RangeBounds};\n\nuse serde::{Deserialize, Serialize};\n\nuse super::{NodeID, Term};\nuse crate::encoding::{self, Key as _, Value as _, bincode};\nuse crate::error::Result;\nuse crate::storage;\n\n/// A log index (entry position). Starts at 1. 0 indicates no index.\npub type Index = u64;\n\n/// A log entry containing a state machine command.\n#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]\npub struct Entry {\n /// The entry index.\n ///\n /// We could omit the index in the encoded value, since it's also stored in\n /// the key, but we keep it simple.\n pub index: Index,\n /// The term in which the entry was added.\n pub term: Term,\n /// The state machine command. None (noop) commands are used during leader\n /// election to commit old entries, see section 5.4.2 in the Raft paper.\n pub command: Option>,\n}\n\nimpl encoding::Value for Entry {}\n\n/// A log storage key.\n#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]\npub enum Key {\n /// A log entry, storing the term and command.\n Entry(Index),\n /// Stores the current term and vote (if any).\n TermVote,\n /// Stores the current commit index (if any).\n CommitIndex,\n}\n\nimpl encoding::Key<'_> for Key {}\n\n/// The Raft log stores a sequence of arbitrary commands (typically writes) that\n/// are replicated across nodes and applied sequentially to the local state\n/// machine. Each entry contains an index, command, and the term in which the\n/// leader proposed it. Commands may be noops (None), which are added when a\n/// leader is elected (see section 5.4.2 in the Raft paper). For example:\n///\n/// Index | Term | Command\n/// ------|------|------------------------------------------------------\n/// 1 | 1 | None\n/// 2 | 1 | CREATE TABLE table (id INT PRIMARY KEY, value STRING)\n/// 3 | 1 | INSERT INTO table VALUES (1, 'foo')\n/// 4 | 2 | None\n/// 5 | 2 | UPDATE table SET value = 'bar' WHERE id = 1\n/// 6 | 2 | DELETE FROM table WHERE id = 1\n///\n/// Note that this is for illustration only, and the actual toyDB Raft commands\n/// are not SQL statements but lower-level write operations.\n///\n/// A key/value store is used to store the log entries on disk, keyed by index,\n/// along with a few other metadata keys (e.g. who we voted for in this term).\n///\n/// In the steady state, the log is append-only: when a client submits a\n/// command, the leader appends it to its own log (via [`Log::append`]) and\n/// replicates it to followers who append it to their logs (via\n/// [`Log::splice`]). When an index has been replicated to a majority of nodes\n/// it becomes committed, making the log immutable up to that index and\n/// guaranteeing that all nodes will eventually contain it. Nodes keep track of\n/// the commit index via [`Log::commit`] and apply committed commands to the\n/// state machine.\n///\n/// However, uncommitted entries can be replaced or removed. A leader may append\n/// entries to its log, but then be unable to reach consensus on them (e.g.\n/// because it is unable to communicate with a majority of nodes). If a\n/// different leader is elected and writes different commands to those same\n/// indexes, then the uncommitted entries will be replaced with entries from the\n/// new leader once the old leader (or a follower) discovers it.\n///\n/// The Raft log has the following invariants:\n///\n/// * Entry indexes are contiguous starting at 1 (no index gaps).\n/// * Entry terms never decrease from the previous entry.\n/// * Entry terms are at or below the current term.\n/// * Appended entries are durable (flushed to disk).\n/// * Appended entries use the current term.\n/// * Committed entries are never changed or removed (no log truncation).\n/// * Committed entries will eventually be replicated to all nodes.\n/// * Entries with the same index/term contain the same command.\n/// * If two logs contain a matching index/term, all previous entries\n/// are identical (see section 5.3 in the Raft paper).\npub struct Log {\n /// The underlying storage engine. Uses a trait object instead of generics,\n /// to allow runtime selection of the engine and avoid propagating the\n /// generic type parameters throughout Raft.\n pub engine: Box,\n /// The current term.\n term: Term,\n /// Our leader vote in the current term, if any.\n vote: Option,\n /// The index of the last stored entry.\n last_index: Index,\n /// The term of the last stored entry.\n last_term: Term,\n /// The index of the last committed entry.\n commit_index: Index,\n /// The term of the last committed entry.\n commit_term: Term,\n /// If true, fsync entries to disk when appended. This is mandated by Raft,\n /// but comes with a hefty performance penalty (especially since we don't\n /// optimize for it by batching entries before fsyncing). Disabling it will\n /// yield much better write performance, but may lose data on crashes, which\n /// in some scenarios can cause log entries to become \"uncommitted\" and\n /// state machines diverging.\n fsync: bool,\n}\n\nimpl Log {\n /// Initializes a log using the given storage engine.\n pub fn new(mut engine: Box) -> Result {\n // Load some initial in-memory state from disk.\n let (term, vote) = engine\n .get(&Key::TermVote.encode())?\n .map(|v| bincode::deserialize(&v))\n .transpose()?\n .unwrap_or((0, None));\n let (last_index, last_term) = engine\n .scan_dyn((\n Bound::Included(Key::Entry(0).encode()),\n Bound::Included(Key::Entry(u64::MAX).encode()),\n ))\n .last()\n .transpose()?\n .map(|(_, v)| Entry::decode(&v))\n .transpose()?\n .map(|e| (e.index, e.term))\n .unwrap_or((0, 0));\n let (commit_index, commit_term) = engine\n .get(&Key::CommitIndex.encode())?\n .map(|v| bincode::deserialize(&v))\n .transpose()?\n .unwrap_or((0, 0));\n\n let fsync = true; // fsync by default\n Ok(Self { engine, term, vote, last_index, last_term, commit_index, commit_term, fsync })\n }\n\n /// Controls whether to fsync writes. Disabling this may violate Raft\n /// guarantees, see comment on fsync attribute.\n pub fn enable_fsync(&mut self, fsync: bool) {\n self.fsync = fsync\n }\n\n /// Returns the commit index and term.\n pub fn get_commit_index(&self) -> (Index, Term) {\n (self.commit_index, self.commit_term)\n }\n\n /// Returns the last log index and term.\n pub fn get_last_index(&self) -> (Index, Term) {\n (self.last_index, self.last_term)\n }\n\n /// Returns the current term (0 if none) and vote.\n pub fn get_term_vote(&self) -> (Term, Option) {\n (self.term, self.vote)\n }\n\n /// Stores the current term and cast vote (if any). Enforces that the term\n /// does not regress, and that we only vote for one node in a term. append()\n /// will use this term, and splice() can't write entries beyond it.\n pub fn set_term_vote(&mut self, term: Term, vote: Option) -> Result<()> {\n assert!(term > 0, \"can't set term 0\");\n assert!(term >= self.term, \"term regression {} → {}\", self.term, term);\n assert!(term > self.term || self.vote.is_none() || vote == self.vote, \"can't change vote\");\n\n if term == self.term && vote == self.vote {\n return Ok(());\n }\n self.engine.set(&Key::TermVote.encode(), bincode::serialize(&(term, vote)))?;\n // Always fsync, even with Log::fsync = false. Term changes are rare, so\n // this doesn't materially affect performance, and double voting could\n // lead to multiple leaders and split brain which is really bad.\n self.engine.flush()?;\n self.term = term;\n self.vote = vote;\n Ok(())\n }\n\n /// Appends a command to the log at the current term, and flushes it to\n /// disk, returning its index. None implies a noop command, typically after\n /// Raft leader changes.\n pub fn append(&mut self, command: Option>) -> Result {\n assert!(self.term > 0, \"can't append entry in term 0\");\n let entry = Entry { index: self.last_index + 1, term: self.term, command };\n self.engine.set(&Key::Entry(entry.index).encode(), entry.encode())?;\n if self.fsync {\n self.engine.flush()?;\n }\n self.last_index = entry.index;\n self.last_term = entry.term;\n Ok(entry.index)\n }\n\n /// Commits entries up to and including the given index. The index must\n /// exist and be at or after the current commit index.\n pub fn commit(&mut self, index: Index) -> Result {\n let term = match self.get(index)? {\n Some(entry) if entry.index < self.commit_index => {\n panic!(\"commit index regression {} → {}\", self.commit_index, entry.index);\n }\n Some(entry) if entry.index == self.commit_index => return Ok(index),\n Some(entry) => entry.term,\n None => panic!(\"commit index {index} does not exist\"),\n };\n self.engine.set(&Key::CommitIndex.encode(), bincode::serialize(&(index, term)))?;\n // NB: the commit index doesn't need to be fsynced, since the entries\n // are fsynced and the commit index can be recovered from the quorum.\n self.commit_index = index;\n self.commit_term = term;\n Ok(index)\n }\n\n /// Fetches an entry at an index, or None if it does not exist.\n pub fn get(&mut self, index: Index) -> Result> {\n self.engine.get(&Key::Entry(index).encode())?.map(|v| Entry::decode(&v)).transpose()\n }\n\n /// Checks if the log contains an entry with the given index and term.\n pub fn has(&mut self, index: Index, term: Term) -> Result {\n // Fast path: check against last_index. This is the common case when\n // followers process appends or heartbeats.\n if index == 0 || index > self.last_index {\n return Ok(false);\n }\n if (index, term) == (self.last_index, self.last_term) {\n return Ok(true);\n }\n Ok(self.get(index)?.map(|e| e.term == term).unwrap_or(false))\n }\n\n /// Returns an iterator over log entries in the given index range.\n pub fn scan(&mut self, range: impl RangeBounds) -> Iterator<'_> {\n let from = match range.start_bound() {\n Bound::Excluded(&index) => Bound::Excluded(Key::Entry(index).encode()),\n Bound::Included(&index) => Bound::Included(Key::Entry(index).encode()),\n Bound::Unbounded => Bound::Included(Key::Entry(0).encode()),\n };\n let to = match range.end_bound() {\n Bound::Excluded(&index) => Bound::Excluded(Key::Entry(index).encode()),\n Bound::Included(&index) => Bound::Included(Key::Entry(index).encode()),\n Bound::Unbounded => Bound::Included(Key::Entry(Index::MAX).encode()),\n };\n Iterator::new(self.engine.scan_dyn((from, to)))\n }\n\n /// Returns an iterator over entries that are ready to apply, starting after\n /// the current applied index up to the commit index.\n pub fn scan_apply(&mut self, applied_index: Index) -> Iterator<'_> {\n // NB: we don't assert that commit_index >= applied_index, because the\n // local commit index is not flushed to durable storage -- if lost on\n // restart, it can be recovered from the logs of a quorum.\n if applied_index >= self.commit_index {\n return Iterator::new(Box::new(std::iter::empty()));\n }\n self.scan(applied_index + 1..=self.commit_index)\n }\n\n /// Splices a set of entries into the log and flushes it to disk. New\n /// indexes will be appended. Overlapping indexes with the same term must be\n /// equal and will be ignored. Overlapping indexes with different terms will\n /// truncate the existing log at the first conflict and then splice the new\n /// entries.\n ///\n /// The entries must have contiguous indexes and equal/increasing terms, and\n /// the first entry must be in the range [1,last_index+1] with a term at or\n /// above the previous (base) entry's term and at or below the current term.\n pub fn splice(&mut self, entries: Vec) -> Result {\n let (Some(first), Some(last)) = (entries.first(), entries.last()) else {\n return Ok(self.last_index); // empty input is noop\n };\n\n // Check that the entries are well-formed.\n assert!(first.index > 0 && first.term > 0, \"spliced entry has index or term 0\",);\n assert!(\n entries.windows(2).all(|w| w[0].index + 1 == w[1].index),\n \"spliced entries are not contiguous\"\n );\n assert!(\n entries.windows(2).all(|w| w[0].term <= w[1].term),\n \"spliced entries have term regression\",\n );\n\n // Check that the entries connect to the existing log (if any), and that the\n // term doesn't regress.\n assert!(last.term <= self.term, \"splice term {} beyond current {}\", last.term, self.term);\n match self.get(first.index - 1)? {\n Some(base) if first.term < base.term => {\n panic!(\"splice term regression {} → {}\", base.term, first.term)\n }\n Some(_) => {}\n None if first.index == 1 => {}\n None => panic!(\"first index {} must touch existing log\", first.index),\n }\n\n // Skip entries that are already in the log.\n let mut entries = entries.as_slice();\n let mut scan = self.scan(first.index..=last.index);\n while let Some(entry) = scan.next().transpose()? {\n // [0] is ok, because the scan has the same size as entries.\n assert!(entry.index == entries[0].index, \"index mismatch at {entry:?}\");\n if entry.term != entries[0].term {\n break;\n }\n assert!(entry.command == entries[0].command, \"command mismatch at {entry:?}\");\n entries = &entries[1..];\n }\n drop(scan);\n\n // If all entries already exist then we're done.\n let Some(first) = entries.first() else {\n return Ok(self.last_index);\n };\n\n // Write the entries that weren't already in the log, and remove the\n // tail of the old log if any. We can't write below the commit index,\n // since these entries must be immutable.\n assert!(first.index > self.commit_index, \"spliced entries below commit index\");\n\n for entry in entries {\n self.engine.set(&Key::Entry(entry.index).encode(), entry.encode())?;\n }\n for index in last.index + 1..=self.last_index {\n self.engine.delete(&Key::Entry(index).encode())?;\n }\n if self.fsync {\n self.engine.flush()?;\n }\n\n self.last_index = last.index;\n self.last_term = last.term;\n Ok(self.last_index)\n }\n\n /// Returns log engine status.\n pub fn status(&mut self) -> Result {\n self.engine.status()\n }\n}\n\n/// A log entry iterator.\npub struct Iterator<'a> {\n inner: Box,\n}\n\nimpl<'a> Iterator<'a> {\n fn new(inner: Box) -> Self {\n Self { inner }\n }\n}\n\nimpl std::iter::Iterator for Iterator<'_> {\n type Item = Result;\n\n fn next(&mut self) -> Option {\n self.inner.next().map(|r| r.and_then(|(_, v)| Entry::decode(&v)))\n }\n}\n\n/// Most Raft tests are Goldenscripts under src/raft/testscripts.\n#[cfg(test)]\nmod tests {\n use std::error::Error;\n use std::fmt::Write as _;\n use std::result::Result;\n\n use crossbeam::channel::Receiver;\n use itertools::Itertools as _;\n use regex::Regex;\n use tempfile::TempDir;\n use test_each_file::test_each_path;\n\n use super::*;\n use crate::encoding::format::{self, Formatter as _};\n use crate::storage::engine::test as testengine;\n\n // Run goldenscript tests in src/raft/testscripts/log.\n test_each_path! { in \"src/raft/testscripts/log\" as scripts => test_goldenscript }\n\n fn test_goldenscript(path: &std::path::Path) {\n goldenscript::run(&mut TestRunner::new(), path).expect(\"goldenscript failed\")\n }\n\n /// Runs Raft log goldenscript tests. For available commands, see run().\n struct TestRunner {\n log: Log,\n op_rx: Receiver,\n #[allow(dead_code)]\n tempdir: TempDir,\n }\n\n impl TestRunner {\n fn new() -> Self {\n // Use both a BitCask and a Memory engine, and mirror operations\n // across them. Emit write events to op_tx.\n let (op_tx, op_rx) = crossbeam::channel::unbounded();\n let tempdir = TempDir::with_prefix(\"toydb\").expect(\"tempdir failed\");\n let bitcask =\n storage::BitCask::new(tempdir.path().join(\"bitcask\")).expect(\"bitcask failed\");\n let memory = storage::Memory::new();\n let engine = testengine::Emit::new(testengine::Mirror::new(bitcask, memory), op_tx);\n let log = Log::new(Box::new(engine)).expect(\"log failed\");\n Self { log, op_rx, tempdir }\n }\n\n /// Parses an index@term pair.\n fn parse_index_term(s: &str) -> Result<(Index, Term), Box> {\n let re = Regex::new(r\"^(\\d+)@(\\d+)$\").expect(\"invalid regex\");\n let groups = re.captures(s).ok_or_else(|| format!(\"invalid index/term {s}\"))?;\n let index = groups.get(1).unwrap().as_str().parse()?;\n let term = groups.get(2).unwrap().as_str().parse()?;\n Ok((index, term))\n }\n\n /// Parses an index range, in Rust range syntax.\n fn parse_index_range(s: &str) -> Result, Box> {\n use std::ops::Bound;\n let mut bound = (Bound::::Unbounded, Bound::::Unbounded);\n let re = Regex::new(r\"^(\\d+)?\\.\\.(=)?(\\d+)?\").expect(\"invalid regex\");\n let groups = re.captures(s).ok_or_else(|| format!(\"invalid range {s}\"))?;\n if let Some(start) = groups.get(1) {\n bound.0 = Bound::Included(start.as_str().parse()?);\n }\n if let Some(end) = groups.get(3) {\n let end = end.as_str().parse()?;\n if groups.get(2).is_some() {\n bound.1 = Bound::Included(end)\n } else {\n bound.1 = Bound::Excluded(end)\n }\n }\n Ok(bound)\n }\n }\n\n impl goldenscript::Runner for TestRunner {\n fn run(&mut self, command: &goldenscript::Command) -> Result> {\n let mut output = String::new();\n let mut tags = command.tags.clone();\n\n match command.name.as_str() {\n // append [COMMAND]\n \"append\" => {\n let mut args = command.consume_args();\n let command = args.next_pos().map(|a| a.value.as_bytes().to_vec());\n args.reject_rest()?;\n let index = self.log.append(command)?;\n let entry = self.log.get(index)?.expect(\"entry not found\");\n let fmtentry = format::Raft::::entry(&entry);\n writeln!(output, \"append → {fmtentry}\")?;\n }\n\n // commit INDEX\n \"commit\" => {\n let mut args = command.consume_args();\n let index = args.next_pos().ok_or(\"index not given\")?.parse()?;\n args.reject_rest()?;\n let index = self.log.commit(index)?;\n let entry = self.log.get(index)?.expect(\"entry not found\");\n let fmtentry = format::Raft::::entry(&entry);\n writeln!(output, \"commit → {fmtentry}\")?;\n }\n\n // dump\n \"dump\" => {\n command.consume_args().reject_rest()?;\n let range = (std::ops::Bound::Unbounded, std::ops::Bound::Unbounded);\n let mut scan = self.log.engine.scan_dyn(range);\n while let Some((key, value)) = scan.next().transpose()? {\n let fmtkv = format::Raft::::key_value(&key, &value);\n let rawkv = format::Raw::key_value(&key, &value);\n writeln!(output, \"{fmtkv} [{rawkv}]\")?;\n }\n }\n\n // get INDEX...\n \"get\" => {\n let mut args = command.consume_args();\n let indexes: Vec =\n args.rest_pos().iter().map(|a| a.parse()).try_collect()?;\n args.reject_rest()?;\n for index in indexes {\n let entry = self.log.get(index)?;\n let fmtentry = entry\n .as_ref()\n .map(format::Raft::::entry)\n .unwrap_or(\"None\".to_string());\n writeln!(output, \"{fmtentry}\")?;\n }\n }\n\n // get_term\n \"get_term\" => {\n command.consume_args().reject_rest()?;\n let (term, vote) = self.log.get_term_vote();\n let vote = vote.map(|v| v.to_string()).unwrap_or(\"None\".to_string());\n writeln!(output, \"term={term} vote={vote}\")?;\n }\n\n // has INDEX@TERM...\n \"has\" => {\n let mut args = command.consume_args();\n let indexes: Vec<(Index, Term)> = args\n .rest_pos()\n .iter()\n .map(|a| Self::parse_index_term(&a.value))\n .try_collect()?;\n args.reject_rest()?;\n for (index, term) in indexes {\n let has = self.log.has(index, term)?;\n writeln!(output, \"{has}\")?;\n }\n }\n\n // reload\n \"reload\" => {\n command.consume_args().reject_rest()?;\n // To get owned access to the inner engine, temporarily\n // replace it with an empty memory engine.\n let engine =\n std::mem::replace(&mut self.log.engine, Box::new(storage::Memory::new()));\n self.log = Log::new(engine)?;\n }\n\n // scan [RANGE]\n \"scan\" => {\n let mut args = command.consume_args();\n let range = Self::parse_index_range(\n args.next_pos().map_or(\"..\", |a| a.value.as_str()),\n )?;\n args.reject_rest()?;\n let mut scan = self.log.scan(range);\n while let Some(entry) = scan.next().transpose()? {\n let fmtentry = format::Raft::::entry(&entry);\n writeln!(output, \"{fmtentry}\")?;\n }\n }\n\n // scan_apply APPLIED_INDEX\n \"scan_apply\" => {\n let mut args = command.consume_args();\n let applied_index =\n args.next_pos().ok_or(\"applied index not given\")?.parse()?;\n args.reject_rest()?;\n let mut scan = self.log.scan_apply(applied_index);\n while let Some(entry) = scan.next().transpose()? {\n let fmtentry = format::Raft::::entry(&entry);\n writeln!(output, \"{fmtentry}\")?;\n }\n }\n\n // set_term TERM [VOTE]\n \"set_term\" => {\n let mut args = command.consume_args();\n let term = args.next_pos().ok_or(\"term not given\")?.parse()?;\n let vote = args.next_pos().map(|a| a.parse()).transpose()?;\n args.reject_rest()?;\n self.log.set_term_vote(term, vote)?;\n }\n\n // splice [INDEX@TERM=COMMAND...]\n \"splice\" => {\n let mut args = command.consume_args();\n let mut entries = Vec::new();\n for arg in args.rest_key() {\n let (index, term) = Self::parse_index_term(arg.key.as_deref().unwrap())?;\n let command = match arg.value.as_str() {\n \"\" => None,\n value => Some(value.as_bytes().to_vec()),\n };\n entries.push(Entry { index, term, command });\n }\n args.reject_rest()?;\n let index = self.log.splice(entries)?;\n let entry = self.log.get(index)?.expect(\"entry not found\");\n let fmtentry = format::Raft::::entry(&entry);\n writeln!(output, \"splice → {fmtentry}\")?;\n }\n\n // status [engine=BOOL]\n \"status\" => {\n let mut args = command.consume_args();\n let engine = args.lookup_parse(\"engine\")?.unwrap_or(false);\n args.reject_rest()?;\n let (term, vote) = self.log.get_term_vote();\n let (last_index, last_term) = self.log.get_last_index();\n let (commit_index, commit_term) = self.log.get_commit_index();\n let vote = vote.map(|id| id.to_string()).unwrap_or(\"None\".to_string());\n write!(\n output,\n \"term={term} last={last_index}@{last_term} commit={commit_index}@{commit_term} vote={vote}\",\n )?;\n if engine {\n write!(output, \" engine={:#?}\", self.log.status()?)?;\n }\n writeln!(output)?;\n }\n\n name => return Err(format!(\"unknown command {name}\").into()),\n }\n\n // If requested, output engine operations.\n if tags.remove(\"ops\") {\n while let Ok(op) = self.op_rx.try_recv() {\n match op {\n testengine::Operation::Delete { key } => {\n let fmtkey = format::Raft::::key(&key);\n let rawkey = format::Raw::key(&key);\n writeln!(output, \"engine delete {fmtkey} [{rawkey}]\")?\n }\n testengine::Operation::Flush => writeln!(output, \"engine flush\")?,\n testengine::Operation::Set { key, value } => {\n let fmtkv = format::Raft::::key_value(&key, &value);\n let rawkv = format::Raw::key_value(&key, &value);\n writeln!(output, \"engine set {fmtkv} [{rawkv}]\")?\n }\n }\n }\n }\n\n if let Some(tag) = tags.iter().next() {\n return Err(format!(\"unknown tag {tag}\").into());\n }\n\n Ok(output)\n }\n\n /// If requested via [ops] tag, output engine operations for the command.\n fn end_command(&mut self, _: &goldenscript::Command) -> Result> {\n // Drain any remaining engine operations.\n while self.op_rx.try_recv().is_ok() {}\n Ok(String::new())\n }\n }\n}\n" }, { "path": "src/raft/message.rs", "content": "use std::collections::BTreeMap;\n\nuse serde::{Deserialize, Serialize};\n\nuse super::{Entry, Index, NodeID, Term};\nuse crate::encoding;\nuse crate::error::Result;\nuse crate::storage;\n\n/// A message envelope specifying the sender and receiver.\n#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]\npub struct Envelope {\n /// The sender.\n pub from: NodeID,\n /// The sender's current term.\n pub term: Term,\n /// The recipient.\n pub to: NodeID,\n /// The message.\n pub message: Message,\n}\n\nimpl encoding::Value for Envelope {}\n\n/// A message sent between Raft nodes. Messages are sent asynchronously (i.e.\n/// they are not request/response) and may be dropped or reordered.\n///\n/// In practice, they are sent across a TCP connection and crossbeam channel\n/// which ensures messages are not dropped or reordered as long as the\n/// connection remains intact. A message and its response are sent across\n/// separate TCP connections (outbound from the respective sender).\n#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]\npub enum Message {\n /// Candidates campaign for leadership by soliciting votes from peers.\n /// Votes will only be granted if the candidate's log is at least as\n /// up-to-date as the voter.\n Campaign {\n /// The index of the candidate's last log entry.\n last_index: Index,\n /// The term of the candidate's last log entry.\n last_term: Term,\n },\n\n /// Followers may vote for a single candidate per term, but only if the\n /// candidate's log is at least as up-to-date as the follower. Candidates\n /// implicitly vote for themselves.\n CampaignResponse {\n /// If true, the follower granted the candidate a vote. A false response\n /// isn't necessary, but is emitted for clarity.\n vote: bool,\n },\n\n /// Leaders send periodic heartbeats. This serves several purposes:\n ///\n /// * Inform nodes about the leader, and prevent elections.\n /// * Detect lost appends and reads, as a retry mechanism.\n /// * Advance followers' commit indexes, so they can apply entries.\n ///\n /// The Raft paper does not have a distinct heartbeat message, and instead\n /// uses an empty AppendEntries RPC, but we choose to add one for better\n /// separation of concerns.\n Heartbeat {\n /// The index of the leader's last log entry. The term is the leader's\n /// current term, since it appends a noop entry on election win. The\n /// follower compares this to its own log to determine if it's\n /// up-to-date.\n last_index: Index,\n /// The index of the leader's last committed log entry. Followers use\n /// this to advance their commit index and apply entries. It's only safe\n /// to commit this if the local log matches last_index, such that the\n /// follower's log is identical to the leader at the commit index.\n commit_index: Index,\n /// The leader's latest read sequence number in this term.\n read_seq: ReadSequence,\n },\n\n /// Followers respond to leader heartbeats if they still consider it leader.\n HeartbeatResponse {\n /// If non-zero, the heartbeat's last_index which was matched in the\n /// follower's log. Otherwise, the follower is either divergent or\n /// lagging behind the leader.\n match_index: Index,\n /// The heartbeat's read sequence number.\n read_seq: ReadSequence,\n },\n\n /// Leaders replicate log entries to followers by appending to their logs\n /// after the given base entry.\n ///\n /// If the base entry matches the follower's log then their logs are\n /// identical up to it (see section 5.3 in the Raft paper), and the entries\n /// can be appended -- possibly replacing conflicting entries. Otherwise,\n /// the append is rejected and the leader must retry an earlier base index\n /// until a common base is found.\n ///\n /// Empty appends messages (no entries) are used to probe follower logs for\n /// a common match index in the case of divergent logs, restarted nodes, or\n /// dropped messages. This is typically done by sending probes with a\n /// decrementing base index until a match is found, at which point the\n /// subsequent entries can be sent.\n Append {\n /// The index of the log entry to append after.\n base_index: Index,\n /// The term of the base entry.\n base_term: Term,\n /// Log entries to append. Must start at base_index + 1.\n entries: Vec,\n },\n\n /// Followers accept or reject appends from the leader depending on whether\n /// the base entry matches their log.\n AppendResponse {\n /// If non-zero, the follower appended entries up to this index. The\n /// entire log up to this index is consistent with the leader. If no\n /// entries were sent (a probe), this will be the matching base index.\n match_index: Index,\n /// If non-zero, the follower rejected an append at this base index\n /// because the base index/term did not match its log. If the follower's\n /// log is shorter than the base index, the reject index will be lowered\n /// to the index after its last local index, to avoid probing each\n /// missing index.\n reject_index: Index,\n },\n\n /// Leaders need to confirm they are still the leader before serving reads,\n /// to guarantee linearizability in case a different leader has been\n /// estalished elsewhere. Read requests are served once the sequence number\n /// has been confirmed by a quorum.\n Read { seq: ReadSequence },\n\n /// Followers confirm leadership at the read sequence numbers.\n ReadResponse { seq: ReadSequence },\n\n /// A client request. This can be submitted to the leader, or to a follower\n /// which will forward it to its leader. If there is no leader, or the\n /// leader or term changes, the request is aborted with an Error::Abort\n /// ClientResponse and the client must retry.\n ClientRequest {\n /// The request ID. Must be globally unique for the request duration.\n id: RequestID,\n /// The request itself.\n request: Request,\n },\n\n /// A client response.\n ClientResponse {\n /// The ID of the original ClientRequest.\n id: RequestID,\n /// The response, or an error.\n response: Result,\n },\n}\n\n/// A client request ID. Must be globally unique while in flight.\n///\n/// For simplicity, a random UUIDv4 is used. We could incorporate the\n/// node/process/MAC ID and timestamp for better collision avoidance (e.g. via\n/// UUIDv6) but it doesn't matter at this scale.\npub type RequestID = uuid::Uuid;\n\n/// A read sequence number, used to confirm leadership for linearizable reads.\npub type ReadSequence = u64;\n\n/// A client request, typically passed through to the state machine.\n#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]\npub enum Request {\n /// A state machine read command, executed via `State::read`. This is not\n /// replicated, and only evaluated on the leader.\n Read(Vec),\n /// A state machine write command, executed via `State::apply`. This is\n /// replicated across all nodes, and must produce a deterministic result.\n Write(Vec),\n /// Requests Raft cluster status from the leader.\n Status,\n}\n\nimpl encoding::Value for Request {}\n\n/// A client response. This will be wrapped in a Result for error handling.\n#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]\npub enum Response {\n /// A state machine read result.\n Read(Vec),\n /// A state machine write result.\n Write(Vec),\n /// The current Raft leader status.\n Status(Status),\n}\n\nimpl encoding::Value for Response {}\n\n/// Raft cluster status. Generated by the leader.\n#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]\npub struct Status {\n /// The current Raft leader, which generated this status.\n pub leader: NodeID,\n /// The current Raft term.\n pub term: Term,\n /// The match indexes of all nodes, indicating replication progress. Uses a\n /// BTreeMap for test determinism.\n pub match_index: BTreeMap,\n /// The current commit index.\n pub commit_index: Index,\n /// The current applied index.\n pub applied_index: Index,\n /// The log storage engine status.\n pub storage: storage::Status,\n}\n" }, { "path": "src/raft/mod.rs", "content": "//! Implements the Raft distributed consensus protocol.\n//!\n//! For details, see Diego Ongaro's original writings:\n//!\n//! * Raft paper: \n//! * Raft thesis: \n//! * Raft website: \n//!\n//! Raft is a protocol for a group of computers to agree on some data -- or more\n//! simply, to replicate the data. It is broadly equivalent to [Paxos] and\n//! [Viewstamped Replication], but more prescriptive and simpler to understand.\n//!\n//! Raft has three main properties:\n//!\n//! * Fault tolerance: the system tolerates node failures as long as a majority\n//! of nodes (>50%) remain operational.\n//!\n//! * Linearizability (aka strong consistency): once a client write has been\n//! accepted, it is visible to all clients -- they never see outdated data.\n//!\n//! * Durability: a write is never lost as long as a majority of nodes remain.\n//!\n//! It does this by electing a single leader node which serves client requests\n//! and replicates writes to other nodes. Requests are executed once they have\n//! been confirmed by a strict majority of nodes (a quorum). If a leader fails,\n//! a new leader is elected. Clusters have 3 or more nodes, since a two-node\n//! cluster can't tolerate failures (1/2 is not a majority and would lead to\n//! split brain).\n//!\n//! Notably, Raft does not provide horizontal scalability. Client requests are\n//! processed by a single leader node which can quickly become a bottleneck, and\n//! each node stores a complete copy of the entire dataset. Systems often handle\n//! this by sharding the data into multiple Raft clusters and using a\n//! distributed transaction protocol across them, but this is out of scope here.\n//!\n//! toyDB follows the Raft paper fairly closely, but, like most implementations,\n//! takes some minor artistic liberties.\n//!\n//! [Paxos]: https://www.microsoft.com/en-us/research/uploads/prod/2016/12/paxos-simple-Copy.pdf\n//! [Viewstamped Replication]: https://pmg.csail.mit.edu/papers/vr-revisited.pdf\n//!\n//! RAFT LOG AND STATE MACHINE\n//! ==========================\n//!\n//! Raft maintains an ordered command log containing arbitrary write commands\n//! submitted by clients. It attempts to reach consensus on this log by\n//! replicating it to a majority of nodes. If successful, the log is considered\n//! committed and immutable up to that point.\n//!\n//! Once committed, the commands in the log are applied sequentially to a local\n//! state machine on each node. Raft itself doesn't care what the state machine\n//! and commands are -- in toyDB's case it's a SQL database, but it could be\n//! anything. Raft simply passes opaque commands to an opaque state machine.\n//!\n//! Each log entry contains an index, the leader's term (see next section), and\n//! the command. For example, a naïve illustration of a toyDB Raft log might be:\n//!\n//! Index | Term | Command\n//! ------|------|------------------------------------------------------\n//! 1 | 1 | CREATE TABLE table (id INT PRIMARY KEY, value STRING)\n//! 2 | 1 | INSERT INTO table VALUES (1, 'foo')\n//! 3 | 2 | UPDATE table SET value = 'bar' WHERE id = 1\n//! 4 | 2 | DELETE FROM table WHERE id = 1\n//!\n//! The state machine must be deterministic, such that all nodes will reach the\n//! same identical state. Raft will apply the same commands in the same order\n//! independently on all nodes, but if the commands have non-deterministic\n//! behavior such as random number generation or communication with external\n//! systems it can lead to state divergence causing different results.\n//!\n//! In toyDB, the Raft log is managed by `Log` and stored locally in a\n//! `storage::Engine`. The state machine interface is the `State` trait. See\n//! their documentation for more details.\n//!\n//! LEADER ELECTION\n//! ===============\n//!\n//! Raft nodes can be in one of three states (or roles): follower, candidate,\n//! and leader. toyDB models these as `Node::Follower`, `Node::Candidate`, and\n//! `Node::Leader`.\n//!\n//! * Follower: replicates log entries from a leader. May not know a leader yet.\n//! * Candidate: campaigns for leadership in an election.\n//! * Leader: processes client requests and replicates writes to followers.\n//!\n//! Raft fundamentally relies on a single guarantee: there can be at most one\n//! _valid_ leader at any point in time (old, since-replaced leaders may think\n//! they're still a leader, e.g. during a network partition, but they won't be\n//! able to do anything). It enforces this through the leader election protocol.\n//!\n//! Raft divides time into terms, which are monotonically increasing numbers.\n//! Higher terms always take priority over lower terms. There can be at most one\n//! leader in a term, and it can't change. Nodes keep track of their last known\n//! term and store it on disk (see `Log.set_term()`). Messages between nodes are\n//! tagged with the current term (as `Envelope.term`) -- old terms are ignored,\n//! and future terms cause the node to become a follower in that term.\n//!\n//! Nodes start out as leaderless followers. If they receive a message from a\n//! leader (in a current or future term), they follow it. Otherwise, they wait\n//! out the election timeout (a few seconds), become candidates, and hold a\n//! leader election.\n//!\n//! Candidates increase their term by 1 and send `Message::Campaign` to all\n//! nodes, requesting their vote. Nodes respond with `Message::CampaignResponse`\n//! saying whether a vote was granted. A node can only grant a single vote in a\n//! term (stored to disk via `Log.set_term()`), on a first-come first-serve\n//! basis, and candidates implicitly vote for themselves.\n//!\n//! When a candidate receives a majority of votes (>50%), it becomes leader. It\n//! sends a `Message::Heartbeat` to all nodes asserting its leadership, and all\n//! nodes become followers when they receive it (regardless of who they voted\n//! for). Leaders continue to send periodic heartbeats every second or so. The\n//! new leader also appends an empty entry to its log in order to safely commit\n//! all entries from previous terms (Raft paper section 5.4.2).\n//!\n//! The new leader must have all committed entries in its log (or the cluster\n//! would lose data). To ensure this, there is one additional condition for\n//! granting a vote: the candidate's log must be at least as up-to-date as the\n//! voter. Because an entry must be replicated to a majority before being\n//! committed, this ensures a candidate can only win a majority of votes if its\n//! log is up-to-date with all committed entries (Raft paper section 5.4.1).\n//!\n//! It's possible that no candidate wins an election, for example due to a tie\n//! or a majority of nodes being offline. After an election timeout passes,\n//! candidates will again bump their term and start a new election, until a\n//! leader can be established. To avoid frequent ties, nodes use different,\n//! randomized election timeouts (Raft paper section 5.2).\n//!\n//! Similarly, if a follower doesn't hear from a leader in an election timeout\n//! interval, it will become candidate and hold another election. The periodic\n//! leader heartbeats prevent this as long as the leader is running and\n//! connected. A node that becomes disconnected from the leader will continually\n//! hold new elections by itself until the network heals, at which point a new\n//! election will be held in its term (disrupting the current leader).\n//!\n//! REPLICATION AND CONSENSUS\n//! =========================\n//!\n//! When the leader receives a client write request, it appends the command to\n//! its local log via `Log.append()`, and sends the log entry to all peers in\n//! a `Message::Append`. Followers will attempt to durably append the entry to\n//! their local logs and respond with `Message::AppendResponse`.\n//!\n//! Once a majority have acknowledged the append, the leader commits the entry\n//! via `Log.commit()` and applies it to its local state machine, returning the\n//! result to the client. It will inform followers about the commit in the next\n//! heartbeat as `Message::Heartbeat.commit_index` so they can apply it too, but\n//! this is not necessary for correctness (they will commit and apply it if they\n//! become leader, otherwise they have no need for applying it).\n//!\n//! Followers may not be able to append the entry to their log -- they may be\n//! unreachable, lag behind the leader, or have divergent logs (see Raft paper\n//! section 5.3). The `Append` contains the index and term of the log entry\n//! immediately before the replicated entry as `base_index` and `base_term`. An\n//! index/term pair uniquely identifies a command, and if two logs have the same\n//! index/term pair then the logs are identical up to and including that entry\n//! (Raft paper section 5.3). If the base index/term matches the follower's log,\n//! it appends the entry (potentially replacing any conflicting entries),\n//! otherwise it rejects it.\n//!\n//! When a follower rejects an append, the leader must try to find a common log\n//! entry that exists in both its and the follower's log where it can resume\n//! replication. It does this by sending `Message::Append` probes only\n//! containing a base index/term but no entries -- it will continue to probe\n//! decreasing indexes one by one until the follower responds with a match, then\n//! send an `Append` with the missing entries (Raft paper section 5.3). It keeps\n//! track of each follower's `match_index` and `next_index` in a `Progress`\n//! struct to manage this.\n//!\n//! In case `Append` messages or responses are lost, leaders also send their\n//! `last_index` and term in each `Heartbeat`. If followers don't have that\n//! index/term pair in their log, they'll say so in the `HeartbeatResponse` and\n//! the leader can begin probing their logs as with append rejections.\n//!\n//! CLIENT REQUESTS\n//! ===============\n//!\n//! Client requests are submitted as `Message::ClientRequest` to the local Raft\n//! node. They are only processed on the leader, but followers will proxy them\n//! to the leader (Raft thesis section 6.2). To avoid complications with message\n//! replays (Raft thesis section 6.3), requests are not retried internally, and\n//! are explicitly aborted with `Error::Abort` on leader/term changes as well as\n//! elections.\n//!\n//! Write requests, `Request::Write`, are appended to the Raft log and\n//! replicated. The leader keeps track of the request and its log index in a\n//! `Write` struct. Once the command is committed and applied to the local state\n//! machine, the leader looks up the write request by its log index and sends\n//! the result to the client. Deterministic errors (e.g. foreign key violations)\n//! are also returned to the client, but non-deterministic errors (e.g. IO\n//! errors) must panic the node to avoid state divergence.\n//!\n//! Read requests, `Request::Read`, are only executed on the leader and don't\n//! need to be replicated via the Raft log. However, to ensure linearizability,\n//! the leader has to confirm with a quorum that it's actually still the leader.\n//! Otherwise, it's possible that a new leader has been elected elsewhere and\n//! executed writes without us knowing about it. It does this by assigning an\n//! incrementing sequence number to each read, keeping track of the request in a\n//! `Read` struct, and immediately sending a `Read` message with the latest\n//! sequence number. Followers respond with the sequence number, and once a\n//! quorum have confirmed a sequence number the read is executed and the result\n//! returned to the client.\n//!\n//! IMPLEMENTATION CAVEATS\n//! ======================\n//!\n//! For simplicity, toyDB implements the bare minimum for a functional and\n//! correct Raft protocol, and omits several advanced mechanisms that would be\n//! needed for a real production system. In particular:\n//!\n//! * No leases: for linearizability, every read request requires the leader to\n//! confirm with followers that it's still the leader. This could be avoided\n//! with a leader lease for a predefined time interval (Raft paper section 8,\n//! Raft thesis section 6.3).\n//!\n//! * No cluster membership changes: to add or remove nodes, the entire cluster\n//! must be stopped and restarted with the new configuration, otherwise it\n//! risks multiple leaders (Raft paper section 6).\n//!\n//! * No snapshots: new or lagging nodes must be caught up by replicating and\n//! replaying the entire log, instead of sending a state machine snapshot\n//! (Raft paper section 7).\n//!\n//! * No log truncation: because snapshots aren't supported, the entire Raft\n//! log must be retained forever in order to catch up new/lagging nodes,\n//! leading to excessive storage use (Raft paper section 7).\n//!\n//! * No pre-vote or check-quorum: a node that's partially partitioned (can\n//! reach some but not all nodes) can cause persistent unavailability with\n//! spurious elections or heartbeats. A node rejoining after a partition can\n//! also temporarily disrupt a leader. This requires additional pre-vote and\n//! check-quorum protocol extensions (Raft thesis section 4.2.3 and 9.6).\n//!\n//! * No request retries: client requests will not be retried on leader changes\n//! or message loss, and will be aggressively aborted, to ignore problems\n//! related to message replay (Raft thesis section 6.3).\n//!\n//! * No reject hints: if a follower has a divergent log, the leader will probe\n//! entries one by one until a match is found. The replication protocol could\n//! instead be extended with rejection hints (Raft paper section 5.3).\n\nmod log;\nmod message;\nmod node;\nmod state;\n\nuse std::ops::Range;\nuse std::time::Duration;\n\npub use log::{Entry, Index, Key, Log};\npub use message::{Envelope, Message, ReadSequence, Request, RequestID, Response, Status};\npub use node::{Node, NodeID, Options, Term, Ticks};\npub use state::State;\n\n/// The interval between Raft ticks, the Raft unit of time.\npub const TICK_INTERVAL: Duration = Duration::from_millis(100);\n\n/// The interval between leader heartbeats in ticks.\nconst HEARTBEAT_INTERVAL: Ticks = 4;\n\n/// The default election timeout range in ticks. To avoid election ties, a node\n/// chooses a random value in this interval.\nconst ELECTION_TIMEOUT_RANGE: Range = 10..20;\n\n/// The maximum number of log entries to send in a single append message.\nconst MAX_APPEND_ENTRIES: usize = 100;\n" }, { "path": "src/raft/node.rs", "content": "use std::cmp::{max, min};\nuse std::collections::{HashMap, HashSet, VecDeque};\nuse std::ops::Range;\n\nuse crossbeam::channel::Sender;\nuse itertools::Itertools as _;\nuse log::{debug, info};\nuse rand::RngExt as _;\n\nuse super::log::{Index, Log};\nuse super::message::{Envelope, Message, ReadSequence, Request, RequestID, Response, Status};\nuse super::state::State;\nuse super::{ELECTION_TIMEOUT_RANGE, HEARTBEAT_INTERVAL, MAX_APPEND_ENTRIES};\nuse crate::errinput;\nuse crate::error::{Error, Result};\n\n/// A node ID, unique within a cluster. Assigned manually when started.\npub type NodeID = u8;\n\n/// A leader term number. Increases monotonically on elections.\npub type Term = u64;\n\n/// A logical clock interval as number of ticks.\npub type Ticks = u8;\n\n/// Raft node options.\n#[derive(Clone, Debug, PartialEq)]\npub struct Options {\n /// The number of ticks between leader heartbeats.\n pub heartbeat_interval: Ticks,\n /// The range of randomized election timeouts for followers and candidates.\n pub election_timeout_range: Range,\n /// Maximum number of entries to send in a single Append message.\n pub max_append_entries: usize,\n}\n\nimpl Default for Options {\n fn default() -> Self {\n Self {\n heartbeat_interval: HEARTBEAT_INTERVAL,\n election_timeout_range: ELECTION_TIMEOUT_RANGE,\n max_append_entries: MAX_APPEND_ENTRIES,\n }\n }\n}\n\n/// A Raft node with a dynamic role. This implements the Raft distributed\n/// consensus protocol, see the `raft` module documentation for more info.\n///\n/// The node is driven synchronously by processing inbound messages via `step()`\n/// and by advancing time via `tick()`. These methods consume the node and\n/// return a new one with a possibly different role. Outbound messages are sent\n/// via the given `tx` channel, and must be delivered to peers or clients.\n///\n/// This enum is the public interface to the node, with a closed set of roles.\n/// It wraps the `RawNode` types, which implement the actual node logic.\n/// The enum allows ergonomic use across role transitions since it can represent\n/// all roles, e.g.: `node = node.step()?`.\npub enum Node {\n /// A candidate campaigns for leadership.\n Candidate(RawNode),\n /// A follower replicates entries from a leader.\n Follower(RawNode),\n /// A leader processes client requests and replicates entries to followers.\n Leader(RawNode),\n}\n\nimpl Node {\n /// Creates a new Raft node. It starts as a leaderless follower, waiting to\n /// hear from a leader or otherwise transitioning to candidate and\n /// campaigning for leadership. In the case of a single-node cluster (no\n /// peers), the node immediately transitions to leader when created.\n pub fn new(\n id: NodeID,\n peers: HashSet,\n log: Log,\n state: Box,\n tx: Sender,\n opts: Options,\n ) -> Result {\n let node = RawNode::new(id, peers, log, state, tx, opts)?;\n // If this is a single-node cluster, become leader immediately.\n if node.cluster_size() == 1 {\n return Ok(node.into_candidate()?.into_leader()?.into());\n }\n Ok(node.into())\n }\n\n /// Returns the node's ID.\n pub fn id(&self) -> NodeID {\n match self {\n Self::Candidate(node) => node.id,\n Self::Follower(node) => node.id,\n Self::Leader(node) => node.id,\n }\n }\n\n /// Returns the node's term.\n pub fn term(&self) -> Term {\n match self {\n Self::Candidate(node) => node.term(),\n Self::Follower(node) => node.term(),\n Self::Leader(node) => node.term(),\n }\n }\n\n /// Processes an inbound message.\n pub fn step(self, msg: Envelope) -> Result {\n let peers = match &self {\n Self::Candidate(node) => &node.peers,\n Self::Follower(node) => &node.peers,\n Self::Leader(node) => &node.peers,\n };\n assert_eq!(msg.to, self.id(), \"message to other node: {msg:?}\");\n assert!(peers.contains(&msg.from) || msg.from == self.id(), \"unknown sender: {msg:?}\");\n debug!(\"Stepping {msg:?}\");\n\n match self {\n Self::Candidate(node) => node.step(msg),\n Self::Follower(node) => node.step(msg),\n Self::Leader(node) => node.step(msg),\n }\n }\n\n /// Advances time by a tick.\n pub fn tick(self) -> Result {\n match self {\n Self::Candidate(node) => node.tick(),\n Self::Follower(node) => node.tick(),\n Self::Leader(node) => node.tick(),\n }\n }\n}\n\nimpl From> for Node {\n fn from(node: RawNode) -> Self {\n Node::Candidate(node)\n }\n}\n\nimpl From> for Node {\n fn from(node: RawNode) -> Self {\n Node::Follower(node)\n }\n}\n\nimpl From> for Node {\n fn from(node: RawNode) -> Self {\n Node::Leader(node)\n }\n}\n\n/// Marker trait for a Raft role: leader, follower, or candidate.\npub trait Role {}\n\n/// A Raft node with role R.\n///\n/// This implements the typestate pattern, where individual node states (roles)\n/// are encoded as RawNode. See http://cliffle.com/blog/rust-typestate/.\npub struct RawNode {\n /// The node ID. Must be unique in the cluster.\n id: NodeID,\n /// The IDs of the other nodes in the cluster. Does not change while\n /// running. Can change on restart, but all nodes must have the same set of\n /// nodes, otherwise it can result in multiple leaders (split brain).\n peers: HashSet,\n /// The Raft log, which stores client commands to be executed.\n log: Log,\n /// The Raft state machine, which executes client commands from the log.\n state: Box,\n /// Channel for sending outbound messages to other nodes.\n tx: Sender,\n /// Node options.\n opts: Options,\n /// Role-specific state.\n role: R,\n}\n\nimpl RawNode {\n /// Helper for role transitions.\n fn into_role(self, role: T) -> RawNode {\n RawNode {\n id: self.id,\n peers: self.peers,\n log: self.log,\n state: self.state,\n tx: self.tx,\n opts: self.opts,\n role,\n }\n }\n\n /// Returns the node's current term.\n fn term(&self) -> Term {\n self.log.get_term_vote().0\n }\n\n /// Returns the cluster size as number of nodes.\n fn cluster_size(&self) -> usize {\n self.peers.len() + 1\n }\n\n /// Returns the cluster quorum size (strict majority).\n fn quorum_size(&self) -> usize {\n self.cluster_size() / 2 + 1\n }\n\n /// Returns the quorum value (i.e. median) of the given unsorted vector. It\n /// must have the same length as the cluster size.\n fn quorum_value(&self, mut values: Vec) -> T {\n assert_eq!(values.len(), self.cluster_size(), \"vector size must match cluster size\");\n *values.select_nth_unstable_by(self.quorum_size() - 1, |a, b| a.cmp(b).reverse()).1\n }\n\n /// Generates a random election timeout.\n fn random_election_timeout(&self) -> Ticks {\n rand::rng().random_range(self.opts.election_timeout_range.clone())\n }\n\n /// Sends a message to the given recipient.\n fn send(&self, to: NodeID, message: Message) -> Result<()> {\n Self::send_via(&self.tx, Envelope { from: self.id, to, term: self.term(), message })\n }\n\n /// Sends a message via the given channel. This avoid borrowing self, to\n /// allow sending while holding partial borrows of self.\n fn send_via(tx: &Sender, msg: Envelope) -> Result<()> {\n debug!(\"Sending {msg:?}\");\n Ok(tx.send(msg)?)\n }\n\n /// Broadcasts a message to all peers.\n fn broadcast(&self, message: Message) -> Result<()> {\n // Send in increasing ID order for test determinism.\n for id in self.peers.iter().copied().sorted() {\n self.send(id, message.clone())?;\n }\n Ok(())\n }\n}\n\n/// A follower replicates log entries from a leader and forwards client requests\n/// to it. Nodes start as leaderless followers, until they either discover a\n/// leader or hold an election.\npub struct Follower {\n /// The leader, or None if we're a leaderless follower.\n leader: Option,\n /// The number of ticks since the last message from the leader.\n leader_seen: Ticks,\n /// The leader_seen timeout before triggering an election.\n election_timeout: Ticks,\n // Local client requests that have been forwarded to the leader. These are\n // aborted on leader/term changes.\n forwarded: HashSet,\n}\n\nimpl Follower {\n /// Creates a new follower role.\n fn new(leader: Option, election_timeout: Ticks) -> Self {\n Self { leader, leader_seen: 0, election_timeout, forwarded: HashSet::new() }\n }\n}\n\nimpl Role for Follower {}\n\nimpl RawNode {\n /// Creates a new node as a leaderless follower.\n fn new(\n id: NodeID,\n peers: HashSet,\n log: Log,\n state: Box,\n tx: Sender,\n opts: Options,\n ) -> Result {\n if peers.contains(&id) {\n return errinput!(\"node ID {id} can't be in peers\");\n }\n let role = Follower::new(None, 0);\n let mut node = Self { id, peers, log, state, tx, opts, role };\n node.role.election_timeout = node.random_election_timeout();\n\n // Apply any pending entries following restart. State machine writes are\n // not flushed to durable storage, so a tail of writes may be lost if\n // the host crashes or restarts. The Raft log is durable, so we can\n // always recover the state from it. We reapply any missing entries here\n // if that should happen.\n node.maybe_apply()?;\n Ok(node)\n }\n\n /// Transitions the follower into a candidate, by campaigning for\n /// leadership in a new term.\n fn into_candidate(mut self) -> Result> {\n // Abort any forwarded requests. These must be retried with new leader.\n self.abort_forwarded()?;\n\n // Apply any pending log entries, so that we're caught up if we win.\n self.maybe_apply()?;\n\n // Become candidate and campaign.\n let election_timeout = self.random_election_timeout();\n let mut node = self.into_role(Candidate::new(election_timeout));\n node.campaign()?;\n\n let (term, vote) = node.log.get_term_vote();\n assert!(node.role.votes.contains(&node.id), \"candidate did not vote for self\");\n assert_ne!(term, 0, \"candidate can't have term 0\");\n assert_eq!(vote, Some(node.id), \"log vote does not match self\");\n\n Ok(node)\n }\n\n /// Transitions the follower into either a leaderless follower in a new term\n /// (e.g. if someone holds a new election) or a follower of a current leader.\n fn into_follower(mut self, term: Term, leader: Option) -> Result> {\n assert_ne!(term, 0, \"can't become follower in term 0\");\n\n // Abort any forwarded requests. These must be retried with new leader.\n self.abort_forwarded()?;\n\n if let Some(leader) = leader {\n // We found a leader in the current term.\n assert!(self.peers.contains(&leader), \"leader is not a peer\");\n assert_eq!(self.role.leader, None, \"already have leader in term\");\n assert_eq!(term, self.term(), \"can't follow leader in different term\");\n info!(\"Following leader {leader} in term {term}\");\n self.role = Follower::new(Some(leader), self.role.election_timeout);\n } else {\n // We found a new term, but we don't know who the leader is yet.\n // We'll find out if we step a message from it.\n assert_ne!(term, self.term(), \"can't become leaderless follower in current term\");\n info!(\"Discovered new term {term}\");\n self.log.set_term_vote(term, None)?;\n self.role = Follower::new(None, self.random_election_timeout());\n }\n Ok(self)\n }\n\n /// Processes an inbound message.\n fn step(mut self, msg: Envelope) -> Result {\n // Past term: outdated peer, drop the message.\n if msg.term < self.term() {\n debug!(\"Dropping message from past term: {msg:?}\");\n return Ok(self.into());\n }\n // Future term: newer leader or candidate, become leaderless follower\n // and step the message.\n if msg.term > self.term() {\n return self.into_follower(msg.term, None)?.step(msg);\n }\n\n // Record when we last saw a message from the leader (if any).\n if Some(msg.from) == self.role.leader {\n self.role.leader_seen = 0\n }\n\n match msg.message {\n // The leader sends periodic heartbeats. If we don't have a leader\n // yet, follow it. If the commit_index advances, apply commands.\n Message::Heartbeat { last_index, commit_index, read_seq } => {\n assert!(commit_index <= last_index, \"commit_index after last_index\");\n\n // Make sure the heartbeat is from our leader, or follow it.\n match self.role.leader {\n Some(leader) => assert_eq!(msg.from, leader, \"multiple leaders in term\"),\n None => self = self.into_follower(msg.term, Some(msg.from))?,\n }\n\n // Check if our log matches the leader's log up to last_index,\n // and respond to the heartbeat. last_index always has the\n // leader's term, since it only appends entries in its term.\n let match_index = if self.log.has(last_index, msg.term)? { last_index } else { 0 };\n self.send(msg.from, Message::HeartbeatResponse { match_index, read_seq })?;\n\n // Advance the commit index and apply entries. We can only do\n // this if we matched the leader's last_index, which implies\n // that the logs are identical up to match_index. This also\n // implies that the commit_index is present in our log.\n if match_index != 0 && commit_index > self.log.get_commit_index().0 {\n self.log.commit(commit_index)?;\n self.maybe_apply()?;\n }\n }\n\n // Append log entries from the leader to the local log.\n Message::Append { base_index, base_term, entries } => {\n if let Some(first) = entries.first() {\n assert_eq!(base_index, first.index - 1, \"base index mismatch\");\n }\n\n // Make sure the append is from our leader, or follow it.\n match self.role.leader {\n Some(leader) => assert_eq!(msg.from, leader, \"multiple leaders in term\"),\n None => self = self.into_follower(msg.term, Some(msg.from))?,\n }\n\n // If the base entry matches our log, append the entries.\n if base_index == 0 || self.log.has(base_index, base_term)? {\n let match_index = entries.last().map(|e| e.index).unwrap_or(base_index);\n self.log.splice(entries)?;\n self.send(msg.from, Message::AppendResponse { match_index, reject_index: 0 })?;\n } else {\n // Otherwise, reject the base index. If the local log is\n // shorter than the base index, lower the reject index to\n // skip all missing entries.\n let reject_index = min(base_index, self.log.get_last_index().0 + 1);\n self.send(msg.from, Message::AppendResponse { reject_index, match_index: 0 })?;\n }\n }\n\n // Confirm the leader's read sequence number.\n Message::Read { seq } => {\n // Make sure the read is from our leader, or follow it.\n match self.role.leader {\n Some(leader) => assert_eq!(msg.from, leader, \"multiple leaders in term\"),\n None => self = self.into_follower(msg.term, Some(msg.from))?,\n }\n\n // Confirm the read.\n self.send(msg.from, Message::ReadResponse { seq })?;\n }\n\n // A candidate is requesting our vote. We only grant one per term.\n Message::Campaign { last_index, last_term } => {\n // Don't vote if we already voted for someone else in this term.\n // We can repeat our vote for the same node though.\n if let (_, Some(vote)) = self.log.get_term_vote()\n && msg.from != vote\n {\n self.send(msg.from, Message::CampaignResponse { vote: false })?;\n return Ok(self.into());\n }\n\n // Only vote if the candidate's log is at least as long as ours.\n // At least one node in any quorum must have all committed\n // entries, and this ensures we'll only elect a leader that has\n // all committed entries. See section 5.4.1 in the Raft paper.\n let (log_index, log_term) = self.log.get_last_index();\n if log_term > last_term || log_term == last_term && log_index > last_index {\n self.send(msg.from, Message::CampaignResponse { vote: false })?;\n return Ok(self.into());\n }\n\n // Grant the vote.\n info!(\"Voting for {} in term {} election\", msg.from, msg.term);\n self.log.set_term_vote(msg.term, Some(msg.from))?;\n self.send(msg.from, Message::CampaignResponse { vote: true })?;\n }\n\n // Forward client requests to the leader, or abort them if there is\n // none. These will not be retried, the client should use timeouts\n // instead. Local client requests use our node ID as the sender.\n Message::ClientRequest { id, request: _ } => {\n assert_eq!(msg.from, self.id, \"client request from other node\");\n\n if let Some(leader) = self.role.leader {\n debug!(\"Forwarding request to leader {leader}: {msg:?}\");\n self.role.forwarded.insert(id);\n self.send(leader, msg.message)?\n } else {\n let response = Err(Error::Abort);\n self.send(msg.from, Message::ClientResponse { id, response })?\n }\n }\n\n // Client responses from the leader are passed on to the client.\n Message::ClientResponse { id, response } => {\n assert_eq!(Some(msg.from), self.role.leader, \"client response from non-leader\");\n\n if self.role.forwarded.remove(&id) {\n self.send(self.id, Message::ClientResponse { id, response })?;\n }\n }\n\n // We may receive a vote after we lost an election, ignore it.\n Message::CampaignResponse { .. } => {}\n\n // We're not leader this term, so we shouldn't see these.\n Message::HeartbeatResponse { .. }\n | Message::AppendResponse { .. }\n | Message::ReadResponse { .. } => {\n panic!(\"follower received unexpected message {msg:?}\")\n }\n };\n Ok(self.into())\n }\n\n /// Processes a logical clock tick.\n fn tick(mut self) -> Result {\n // Campaign if we haven't heard from the leader in a while.\n self.role.leader_seen += 1;\n if self.role.leader_seen >= self.role.election_timeout {\n return Ok(self.into_candidate()?.into());\n }\n Ok(self.into())\n }\n\n /// Aborts all forwarded requests (e.g. on term/leader changes).\n fn abort_forwarded(&mut self) -> Result<()> {\n // Sort by ID for test determinism.\n for id in std::mem::take(&mut self.role.forwarded).into_iter().sorted() {\n debug!(\"Aborting forwarded request {id}\");\n self.send(self.id, Message::ClientResponse { id, response: Err(Error::Abort) })?;\n }\n Ok(())\n }\n\n /// Applies any pending log entries.\n fn maybe_apply(&mut self) -> Result<()> {\n let mut iter = self.log.scan_apply(self.state.get_applied_index());\n while let Some(entry) = iter.next().transpose()? {\n debug!(\"Applying {entry:?}\");\n // Throw away the result, since only the leader responds to clients.\n // This includes errors -- any non-deterministic errors (e.g. IO\n // errors) must panic instead to avoid node divergence.\n _ = self.state.apply(entry);\n }\n Ok(())\n }\n}\n\n/// A candidate is campaigning to become a leader.\npub struct Candidate {\n /// Votes received (including our own).\n votes: HashSet,\n /// Ticks elapsed since election start.\n election_duration: Ticks,\n /// Election timeout, in ticks.\n election_timeout: Ticks,\n}\n\nimpl Candidate {\n /// Creates a new candidate role.\n fn new(election_timeout: Ticks) -> Self {\n Self { votes: HashSet::new(), election_duration: 0, election_timeout }\n }\n}\n\nimpl Role for Candidate {}\n\nimpl RawNode {\n /// Transitions the candidate to a follower. We either lost the election and\n /// follow the winner, or we discovered a new term and step into it as a\n /// leaderless follower.\n fn into_follower(mut self, term: Term, leader: Option) -> Result> {\n let election_timeout = self.random_election_timeout();\n if let Some(leader) = leader {\n // We lost the election, follow the winner.\n assert_eq!(term, self.term(), \"can't follow leader in different term\");\n info!(\"Lost election, following leader {leader} in term {term}\");\n Ok(self.into_role(Follower::new(Some(leader), election_timeout)))\n } else {\n // We found a new term, but we don't necessarily know who the leader\n // is yet. We'll find out when we step a message from it.\n assert_ne!(term, self.term(), \"can't become leaderless follower in current term\");\n info!(\"Discovered new term {term}\");\n self.log.set_term_vote(term, None)?;\n Ok(self.into_role(Follower::new(None, election_timeout)))\n }\n }\n\n /// Transitions the candidate to a leader. We won the election.\n fn into_leader(self) -> Result> {\n let (term, vote) = self.log.get_term_vote();\n assert_ne!(term, 0, \"leaders can't have term 0\");\n assert_eq!(vote, Some(self.id), \"leader did not vote for self\");\n\n info!(\"Won election for term {term}, becoming leader\");\n let peers = self.peers.clone();\n let (last_index, _) = self.log.get_last_index();\n let mut node = self.into_role(Leader::new(peers, last_index));\n\n // Propose an empty command when assuming leadership, to disambiguate\n // previous entries in the log. See section 5.4.2 in the Raft paper.\n // We do this prior to the heartbeat, to avoid a wasted replication\n // roundtrip if the heartbeat response indicates the peer is behind.\n node.propose(None)?;\n node.maybe_commit_and_apply()?;\n node.heartbeat()?;\n\n Ok(node)\n }\n\n /// Processes an inbound message.\n fn step(mut self, msg: Envelope) -> Result {\n // Past term: outdated peer, drop the message.\n if msg.term < self.term() {\n debug!(\"Dropping message from past term: {msg:?}\");\n return Ok(self.into());\n }\n // Future term: newer leader or candidate, become leaderless follower\n // and step the message.\n if msg.term > self.term() {\n return self.into_follower(msg.term, None)?.step(msg);\n }\n\n match msg.message {\n // If we received a vote, record it. If the vote gives us quorum,\n // assume leadership.\n Message::CampaignResponse { vote: true } => {\n self.role.votes.insert(msg.from);\n if self.role.votes.len() >= self.quorum_size() {\n return Ok(self.into_leader()?.into());\n }\n }\n\n // We didn't get the vote. :(\n Message::CampaignResponse { vote: false } => {}\n\n // Don't grant votes for other candidates.\n Message::Campaign { .. } => {\n self.send(msg.from, Message::CampaignResponse { vote: false })?\n }\n\n // If we hear from a leader in this term, we lost the election.\n // Follow it and step the message.\n Message::Heartbeat { .. } | Message::Append { .. } | Message::Read { .. } => {\n return self.into_follower(msg.term, Some(msg.from))?.step(msg);\n }\n\n // Abort client requests while campaigning. The client must retry.\n Message::ClientRequest { id, request: _ } => {\n self.send(msg.from, Message::ClientResponse { id, response: Err(Error::Abort) })?;\n }\n\n // We're not a leader in this term, nor are we forwarding requests,\n // so we shouldn't see these.\n Message::HeartbeatResponse { .. }\n | Message::AppendResponse { .. }\n | Message::ReadResponse { .. }\n | Message::ClientResponse { .. } => panic!(\"unexpected message {msg:?}\"),\n }\n Ok(self.into())\n }\n\n /// Processes a logical clock tick.\n fn tick(mut self) -> Result {\n // If noone won this election, start a new one after a while.\n self.role.election_duration += 1;\n if self.role.election_duration >= self.role.election_timeout {\n self.campaign()?;\n }\n Ok(self.into())\n }\n\n /// Hold a new election by increasing the term, voting for ourself, and\n /// soliciting votes from all peers.\n fn campaign(&mut self) -> Result<()> {\n let term = self.term() + 1;\n info!(\"Starting new election for term {term}\");\n self.role = Candidate::new(self.random_election_timeout());\n self.role.votes.insert(self.id); // vote for ourself\n self.log.set_term_vote(term, Some(self.id))?;\n\n let (last_index, last_term) = self.log.get_last_index();\n self.broadcast(Message::Campaign { last_index, last_term })\n }\n}\n\n/// A leader serves client requests and replicates the log to followers.\n/// If the leader loses leadership, all client requests are aborted.\npub struct Leader {\n /// Follower replication progress.\n progress: HashMap,\n /// Tracks pending write requests by log index. Added when the write is\n /// proposed and appended to the leader's log, and removed when the command\n /// is applied to the state machine, returning the result to the client.\n writes: HashMap,\n /// Tracks pending read requests. For linearizability, read requests are\n /// assigned a sequence number and only executed once a quorum of nodes have\n /// confirmed that we're still the leader. Otherwise, an old leader could\n /// serve stale reads if a new leader has been elected elsewhere.\n reads: VecDeque,\n /// The read sequence number used for the last read. Initialized to 0 in\n /// this term, and incremented for every read command.\n read_seq: ReadSequence,\n /// Number of ticks since last heartbeat.\n since_heartbeat: Ticks,\n}\n\n/// Per-follower replication progress (in this term).\nstruct Progress {\n /// The highest index where the follower's log is known to match the leader.\n /// Initialized to 0, increases monotonically.\n match_index: Index,\n /// The next index to replicate to the follower. Initialized to\n /// last_index+1, decreased when probing log mismatches. Always in\n /// the range [match_index+1, last_index+1].\n ///\n /// Entries not yet sent are in the range [next_index, last_index].\n /// Entries not acknowledged are in the range [match_index+1, next_index).\n next_index: Index,\n /// The last read sequence number confirmed by this follower. To avoid stale\n /// reads on leader changes, a read is only served once its sequence number\n /// is confirmed by a quorum.\n read_seq: ReadSequence,\n}\n\nimpl Progress {\n /// Attempts to advance a follower's match index, returning true if it did.\n /// If next_index is below it, it is advanced to the following index.\n fn advance(&mut self, match_index: Index) -> bool {\n if match_index <= self.match_index {\n return false;\n }\n self.match_index = match_index;\n self.next_index = max(self.next_index, match_index + 1);\n true\n }\n\n /// Attempts to advance a follower's read_seq, returning true if it did.\n fn advance_read(&mut self, read_seq: ReadSequence) -> bool {\n if read_seq <= self.read_seq {\n return false;\n }\n self.read_seq = read_seq;\n true\n }\n\n /// Attempts to regress a follower's next index to the given index, returning\n /// true if it did. Won't regress below match_index + 1.\n fn regress_next(&mut self, next_index: Index) -> bool {\n if next_index >= self.next_index || self.next_index <= self.match_index + 1 {\n return false;\n }\n self.next_index = max(next_index, self.match_index + 1);\n true\n }\n}\n\n/// A pending client write request.\nstruct Write {\n /// The node which submitted the write.\n from: NodeID,\n /// The write request ID.\n id: RequestID,\n}\n\n/// A pending client read request.\nstruct Read {\n /// The sequence number of this read.\n seq: ReadSequence,\n /// The node which submitted the read.\n from: NodeID,\n /// The read request ID.\n id: RequestID,\n /// The read command.\n command: Vec,\n}\n\nimpl Leader {\n /// Creates a new leader role.\n fn new(peers: HashSet, last_index: Index) -> Self {\n let next_index = last_index + 1;\n let progress = peers\n .into_iter()\n .map(|p| (p, Progress { next_index, match_index: 0, read_seq: 0 }))\n .collect();\n Self {\n progress,\n writes: HashMap::new(),\n reads: VecDeque::new(),\n read_seq: 0,\n since_heartbeat: 0,\n }\n }\n}\n\nimpl Role for Leader {}\n\nimpl RawNode {\n /// Transitions the leader into a follower. This can only happen if we\n /// discover a new term, so we become a leaderless follower. Stepping the\n /// received message may then follow a new leader, if there is one.\n fn into_follower(mut self, term: Term) -> Result> {\n assert!(term > self.term(), \"leader can only become follower in later term\");\n info!(\"Discovered new term {term}\");\n\n // Abort in-flight requests. The client must retry. Sort the requests\n // by ID for test determinism.\n for write in std::mem::take(&mut self.role.writes).into_values().sorted_by_key(|w| w.id) {\n let response = Err(Error::Abort);\n self.send(write.from, Message::ClientResponse { id: write.id, response })?;\n }\n for read in std::mem::take(&mut self.role.reads).into_iter().sorted_by_key(|r| r.id) {\n let response = Err(Error::Abort);\n self.send(read.from, Message::ClientResponse { id: read.id, response })?;\n }\n\n self.log.set_term_vote(term, None)?;\n let election_timeout = self.random_election_timeout();\n Ok(self.into_role(Follower::new(None, election_timeout)))\n }\n\n /// Processes an inbound message.\n fn step(mut self, msg: Envelope) -> Result {\n // Past term: outdated peer, drop the message.\n if msg.term < self.term() {\n debug!(\"Dropping message from past term: {msg:?}\");\n return Ok(self.into());\n }\n // Future term: become leaderless follower and step the message.\n if msg.term > self.term() {\n return self.into_follower(msg.term)?.step(msg);\n }\n\n match msg.message {\n // A follower received our heartbeat and confirms our leadership.\n // We may be able to execute new reads, and we may find that the\n // follower's log is lagging and requires us to catch it up.\n Message::HeartbeatResponse { match_index, read_seq } => {\n let (last_index, _) = self.log.get_last_index();\n assert!(match_index <= last_index, \"future match index\");\n assert!(read_seq <= self.role.read_seq, \"future read sequence number\");\n\n // If the read sequence number advances, try to execute reads.\n if self.progress(msg.from).advance_read(read_seq) {\n self.maybe_read()?;\n }\n\n // If the follower didn't match our last index, an append to it\n // must have failed (or it's catching up). Probe it to discover\n // a matching entry and start replicating. Move next_index back\n // to last_index since the follower just told us it doesn't have\n // it (or a previous last_index).\n if match_index == 0 {\n self.progress(msg.from).regress_next(last_index);\n self.maybe_send_append(msg.from, true)?;\n }\n\n // If the follower's match index advances, an append response\n // got lost. Try to commit and apply.\n //\n // We don't need to eagerly send any pending entries, since any\n // proposals made after this heartbeat was sent should have been\n // eagerly replicated in steady state. If not, the next\n // heartbeat will trigger a probe above.\n if self.progress(msg.from).advance(match_index) {\n self.maybe_commit_and_apply()?;\n }\n }\n\n // A follower appended our log entries (or a probe found a match).\n // Record its progress and attempt to commit and apply.\n Message::AppendResponse { match_index, reject_index: 0 } if match_index > 0 => {\n let (last_index, _) = self.log.get_last_index();\n assert!(match_index <= last_index, \"future match index\");\n\n if self.progress(msg.from).advance(match_index) {\n self.maybe_commit_and_apply()?;\n }\n\n // Eagerly send any further pending entries. This may be a\n // successful probe response, or the peer may be lagging and\n // we're catching it up one MAX_APPEND_ENTRIES batch at a time.\n self.maybe_send_append(msg.from, false)?;\n }\n\n // A follower confirmed our read sequence number. If it advances,\n // try to execute reads.\n Message::ReadResponse { seq } => {\n if self.progress(msg.from).advance_read(seq) {\n self.maybe_read()?;\n }\n }\n\n // A follower rejected an append because the base entry in\n // reject_index did not match its log. Probe the previous entry by\n // sending an empty append until we find a common base.\n //\n // This linear probing can be slow with long divergent logs, but we\n // keep it simple. See also section 5.3 in the Raft paper.\n Message::AppendResponse { reject_index, match_index: 0 } if reject_index > 0 => {\n let (last_index, _) = self.log.get_last_index();\n assert!(reject_index <= last_index, \"future reject index\");\n\n // If the rejected base index is at or below the match index,\n // the rejection is stale and can be ignored.\n if reject_index <= self.progress(msg.from).match_index {\n return Ok(self.into());\n }\n\n // Probe below the reject index, if we haven't already moved\n // next_index below it. This avoids sending duplicate probes\n // (heartbeats will trigger retries if they're lost).\n if self.progress(msg.from).regress_next(reject_index) {\n self.maybe_send_append(msg.from, true)?;\n }\n }\n\n // AppendResponses must set either match_index or reject_index.\n Message::AppendResponse { .. } => panic!(\"invalid message {msg:?}\"),\n\n // A client submitted a write request. Propose it, and wait until\n // it's replicated and applied to the state machine before returning\n // the response to the client.\n Message::ClientRequest { id, request: Request::Write(command) } => {\n let index = self.propose(Some(command))?;\n self.role.writes.insert(index, Write { from: msg.from, id });\n if self.cluster_size() == 1 {\n self.maybe_commit_and_apply()?;\n }\n }\n\n // A client submitted a read request. To ensure linearizability, we\n // must confirm that we are still the leader by sending the read's\n // sequence number and wait for quorum confirmation.\n Message::ClientRequest { id, request: Request::Read(command) } => {\n self.role.read_seq += 1;\n let read = Read { seq: self.role.read_seq, from: msg.from, id, command };\n self.role.reads.push_back(read);\n self.broadcast(Message::Read { seq: self.role.read_seq })?;\n if self.cluster_size() == 1 {\n self.maybe_read()?;\n }\n }\n\n // A client submitted a status command.\n Message::ClientRequest { id, request: Request::Status } => {\n let response = self.status().map(Response::Status);\n self.send(msg.from, Message::ClientResponse { id, response })?;\n }\n\n // Don't grant any votes (we've already voted for ourself).\n Message::Campaign { .. } => {\n self.send(msg.from, Message::CampaignResponse { vote: false })?\n }\n\n // Votes can come in after we won the election, ignore them.\n Message::CampaignResponse { .. } => {}\n\n // There can't be another leader in this term.\n Message::Heartbeat { .. } | Message::Append { .. } | Message::Read { .. } => {\n panic!(\"saw other leader {} in term {}\", msg.from, msg.term);\n }\n\n // Leaders don't proxy client requests.\n Message::ClientResponse { .. } => panic!(\"unexpected message {msg:?}\"),\n }\n\n Ok(self.into())\n }\n\n /// Processes a logical clock tick.\n fn tick(mut self) -> Result {\n // Send periodic heartbeats.\n self.role.since_heartbeat += 1;\n if self.role.since_heartbeat >= self.opts.heartbeat_interval {\n self.heartbeat()?;\n }\n Ok(self.into())\n }\n\n /// Broadcasts a heartbeat to all peers.\n fn heartbeat(&mut self) -> Result<()> {\n let (last_index, last_term) = self.log.get_last_index();\n let (commit_index, _) = self.log.get_commit_index();\n let read_seq = self.role.read_seq;\n assert_eq!(last_term, self.term(), \"leader's last_term not in current term\");\n\n self.role.since_heartbeat = 0;\n self.broadcast(Message::Heartbeat { last_index, commit_index, read_seq })\n }\n\n /// Proposes a command for consensus by appending it to our log and\n /// replicating it to peers. If successful, it will eventually be committed\n /// and applied to the state machine.\n fn propose(&mut self, command: Option>) -> Result {\n let index = self.log.append(command)?;\n for peer in self.peers.iter().copied().sorted() {\n // Eagerly send the entry to the peer if it's in steady state and\n // we've sent all previous entries. Otherwise, the peer is lagging\n // and we're probing past entries for a match.\n if index == self.progress(peer).next_index {\n self.maybe_send_append(peer, false)?;\n }\n }\n Ok(index)\n }\n\n /// Commits new entries that have been replicated to a quorum and applies\n /// them to the state machine, returning results to clients.\n fn maybe_commit_and_apply(&mut self) -> Result {\n // Determine the new commit index by quorum.\n let (last_index, _) = self.log.get_last_index();\n let commit_index = self.quorum_value(\n self.role.progress.values().map(|p| p.match_index).chain([last_index]).collect(),\n );\n\n // If the commit index doesn't advance, do nothing. We don't assert on\n // this, since the quorum value may regress e.g. following a restart or\n // leader change where followers are initialized with match index 0.\n let (old_index, old_term) = self.log.get_commit_index();\n if commit_index <= old_index {\n return Ok(old_index);\n }\n\n // We can only safely commit an entry from our own term (see section\n // 5.4.2 in Raft paper).\n match self.log.get(commit_index)? {\n Some(entry) if entry.term == self.term() => {}\n Some(_) => return Ok(old_index),\n None => panic!(\"commit index {commit_index} missing\"),\n }\n\n // Commit entries.\n self.log.commit(commit_index)?;\n\n // Apply entries and respond to clients.\n let term = self.term();\n let mut iter = self.log.scan_apply(self.state.get_applied_index());\n while let Some(entry) = iter.next().transpose()? {\n debug!(\"Applying {entry:?}\");\n let write = self.role.writes.remove(&entry.index);\n let result = self.state.apply(entry);\n\n if let Some(Write { id, from: to }) = write {\n let message = Message::ClientResponse { id, response: result.map(Response::Write) };\n Self::send_via(&self.tx, Envelope { from: self.id, term, to, message })?;\n }\n }\n drop(iter);\n\n // If the commit term changed, there may be pending reads waiting for us\n // to commit and apply an entry from our own term. Execute them.\n if old_term != self.term() {\n self.maybe_read()?;\n }\n\n Ok(commit_index)\n }\n\n /// Executes any ready read requests, where a quorum have confirmed that\n /// we're still the leader for the read sequences.\n fn maybe_read(&mut self) -> Result<()> {\n if self.role.reads.is_empty() {\n return Ok(());\n }\n\n // It's only safe to read if we've committed and applied an entry from\n // our own term (the leader appends an entry when elected). Otherwise we\n // may be behind on application and serve stale reads.\n let (commit_index, commit_term) = self.log.get_commit_index();\n let applied_index = self.state.get_applied_index();\n if commit_term < self.term() || applied_index < commit_index {\n return Ok(());\n }\n\n // Determine the maximum read sequence confirmed by quorum.\n let quorum_read_seq = self.quorum_value(\n self.role.progress.values().map(|p| p.read_seq).chain([self.role.read_seq]).collect(),\n );\n\n // Execute ready reads. The VecDeque is ordered by read_seq, so we\n // can keep pulling until we hit quorum_read_seq.\n while let Some(read) = self.role.reads.front() {\n if read.seq > quorum_read_seq {\n break;\n }\n let read = self.role.reads.pop_front().unwrap();\n let response = self.state.read(read.command).map(Response::Read);\n self.send(read.from, Message::ClientResponse { id: read.id, response })?;\n }\n Ok(())\n }\n\n /// Sends a batch of pending log entries to a follower, in the\n /// [next_index,last_index] range. Limited by max_append_entries.\n ///\n /// If probe is true, we're trying to find a log index on the follower where\n /// it matches our log. To do this, we send an empty append probe with\n /// base_index of next_index-1. If the follower confirms the base_index\n /// matches its log, the actual entries are sent next -- otherwise,\n /// next_index is decremented and another probe is sent until a match is\n /// found. See section 5.3 in the Raft paper.\n ///\n /// The probe is skipped if the follower is up-to-date (according to\n /// match_index and last_index). If the probe's base_index has already been\n /// confirmed via match_index, an actual append is sent instead.\n fn maybe_send_append(&mut self, peer: NodeID, mut probe: bool) -> Result<()> {\n let (last_index, _) = self.log.get_last_index();\n let progress = self.role.progress.get_mut(&peer).expect(\"unknown node\");\n assert_ne!(progress.next_index, 0, \"invalid next_index\");\n assert!(progress.next_index > progress.match_index, \"invalid next_index <= match_index\");\n assert!(progress.match_index <= last_index, \"invalid match_index > last_index\");\n assert!(progress.next_index <= last_index + 1, \"invalid next_index > last_index + 1\");\n\n // If the peer is caught up, there's no point sending an append.\n if progress.match_index == last_index {\n return Ok(());\n }\n\n // If a probe was requested, but the base_index has already been\n // confirmed via match_index, there is no point in probing. Just send\n // the entries instead.\n probe = probe && progress.next_index > progress.match_index + 1;\n\n // If there are no pending entries, and this is not a probe, there's\n // nothing more to send until we get a response from the follower.\n if progress.next_index > last_index && !probe {\n return Ok(());\n }\n\n // Fetch the base and entries.\n let (base_index, base_term) = match progress.next_index {\n 0 => panic!(\"next_index=0 for node {peer}\"),\n 1 => (0, 0), // first entry, there is no base\n next => self.log.get(next - 1)?.map(|e| (e.index, e.term)).expect(\"missing base entry\"),\n };\n let entries = match probe {\n false => self\n .log\n .scan(progress.next_index..)\n .take(self.opts.max_append_entries)\n .try_collect()?,\n true => Vec::new(),\n };\n\n // Optimistically assume the entries will be accepted by the follower,\n // and bump next_index to avoid resending them until a response.\n if let Some(last) = entries.last() {\n progress.next_index = last.index + 1;\n }\n\n debug!(\"Replicating {} entries with base {base_index} to {peer}\", entries.len());\n self.send(peer, Message::Append { base_index, base_term, entries })\n }\n\n /// Generates cluster status.\n fn status(&mut self) -> Result {\n Ok(Status {\n leader: self.id,\n term: self.term(),\n match_index: self\n .role\n .progress\n .iter()\n .map(|(id, p)| (*id, p.match_index))\n .chain(std::iter::once((self.id, self.log.get_last_index().0)))\n .collect(),\n commit_index: self.log.get_commit_index().0,\n applied_index: self.state.get_applied_index(),\n storage: self.log.status()?,\n })\n }\n\n /// Returns a mutable borrow of a node's progress. Convenience method.\n fn progress(&mut self, id: NodeID) -> &mut Progress {\n self.role.progress.get_mut(&id).expect(\"unknown node\")\n }\n}\n\n/// Most Raft tests are Goldenscripts under src/raft/testscripts.\n#[cfg(test)]\nmod tests {\n use std::borrow::Borrow;\n use std::error::Error;\n use std::fmt::Write as _;\n use std::path::Path;\n use std::result::Result;\n\n use crossbeam::channel::Receiver;\n use tempfile::TempDir;\n use test_case::test_case;\n use test_each_file::test_each_path;\n use uuid::Uuid;\n\n use super::*;\n use crate::encoding::{Key as _, Value as _, bincode};\n use crate::raft::Entry;\n use crate::raft::state::test::{self as teststate, KVCommand, KVResponse};\n use crate::storage;\n use crate::storage::engine::test as testengine;\n\n // Run goldenscript tests in src/raft/testscripts/node.\n test_each_path! { in \"src/raft/testscripts/node\" as scripts => test_goldenscript }\n\n fn test_goldenscript(path: &Path) {\n goldenscript::run(&mut TestRunner::new(), path).expect(\"goldenscript failed\")\n }\n\n /// Tests RawNode.quorum_size() and cluster_size().\n #[test_case(1 => 1)]\n #[test_case(2 => 2)]\n #[test_case(3 => 2)]\n #[test_case(4 => 3)]\n #[test_case(5 => 3)]\n #[test_case(6 => 4)]\n #[test_case(7 => 4)]\n #[test_case(8 => 5)]\n fn quorum_size(size: usize) -> usize {\n let node = RawNode::new_noop(1, (2..=size as NodeID).collect());\n assert_eq!(node.cluster_size(), size);\n node.quorum_size()\n }\n\n /// Tests RawNode.quorum_value().\n #[test_case(vec![1] => 1)]\n #[test_case(vec![1,3,2] => 2)]\n #[test_case(vec![4,1,3,2] => 2)]\n #[test_case(vec![1,1,1,2,2] => 1)]\n #[test_case(vec![1,1,2,2,2] => 2)]\n fn quorum_value(values: Vec) -> i8 {\n let size = values.len();\n let node = RawNode::new_noop(1, (2..=size as NodeID).collect());\n assert_eq!(node.cluster_size(), size);\n node.quorum_value(values)\n }\n\n /// Test helpers for RawNode.\n impl RawNode {\n /// Creates a noop node, with a noop state machine and transport.\n fn new_noop(id: NodeID, peers: HashSet) -> Self {\n let log = Log::new(Box::new(storage::Memory::new())).expect(\"log failed\");\n let state = teststate::Noop::new();\n let (tx, _) = crossbeam::channel::unbounded();\n RawNode::new(id, peers, log, state, tx, Options::default()).expect(\"node failed\")\n }\n }\n\n /// Helper macro which calls a closure on the inner RawNode.\n macro_rules! with_rawnode {\n // Node is moved.\n ($node:expr, $closure:expr) => {{\n fn with(node: RawNode, f: impl FnOnce(RawNode) -> T) -> T {\n f(node)\n }\n match $node {\n Node::Candidate(node) => with(node, $closure),\n Node::Follower(node) => with(node, $closure),\n Node::Leader(node) => with(node, $closure),\n }\n }};\n // Node is borrowed (ref).\n (ref $node:expr, $closure:expr) => {{\n fn with(node: &RawNode, f: impl FnOnce(&RawNode) -> T) -> T {\n f(node)\n }\n match $node {\n &Node::Candidate(ref node) => with(node, $closure),\n &Node::Follower(ref node) => with(node, $closure),\n &Node::Leader(ref node) => with(node, $closure),\n }\n }};\n // Node is mutably borrowed (ref mut).\n (ref mut $node:expr, $closure:expr) => {{\n fn with(node: &mut RawNode, f: impl FnOnce(&mut RawNode) -> T) -> T {\n f(node)\n }\n match $node {\n &mut Node::Candidate(ref mut node) => with(node, $closure),\n &mut Node::Follower(ref mut node) => with(node, $closure),\n &mut Node::Leader(ref mut node) => with(node, $closure),\n }\n }};\n }\n\n /// Test helpers for Node.\n impl Node {\n fn dismantle(self) -> (Log, Box) {\n with_rawnode!(self, |n| (n.log, n.state))\n }\n\n fn get_applied_index(&self) -> Index {\n with_rawnode!(ref self, |n| n.state.get_applied_index())\n }\n\n fn get_commit_index(&self) -> (Index, Term) {\n with_rawnode!(ref self, |n| n.log.get_commit_index())\n }\n\n fn get_last_index(&self) -> (Index, Term) {\n with_rawnode!(ref self, |n| n.log.get_last_index())\n }\n\n fn get_term_vote(&self) -> (Term, Option) {\n with_rawnode!(ref self, |n| n.log.get_term_vote())\n }\n\n fn options(&self) -> Options {\n with_rawnode!(ref self, |n| n.opts.clone())\n }\n\n fn peers(&self) -> HashSet {\n with_rawnode!(ref self, |n| n.peers.clone())\n }\n\n fn read(&self, command: Vec) -> crate::error::Result> {\n with_rawnode!(ref self, |n| n.state.read(command))\n }\n\n fn scan_log(&mut self) -> crate::error::Result> {\n with_rawnode!(ref mut self, |n| n.log.scan(..).collect())\n }\n }\n\n /// Runs Raft goldenscript tests. See run() for available commands.\n struct TestRunner {\n /// IDs of all cluster nodes, in order.\n ids: Vec,\n /// The cluster nodes, keyed by node ID.\n nodes: HashMap,\n /// Outbound send queues from each node.\n nodes_rx: HashMap>,\n /// Inbound receive queues to each node, to be stepped.\n nodes_pending: HashMap>,\n /// Applied log entries for each node, after state machine application.\n applied_rx: HashMap>,\n /// Network partitions (sender → receivers). A symmetric (bidirectional)\n /// partition needs an entry from each side.\n disconnected: HashMap>,\n /// In-flight client requests.\n requests: HashMap,\n /// The request ID to use for the next client request.\n next_request_id: u64,\n /// Temporary directory (deleted when dropped).\n tempdir: TempDir,\n }\n\n impl goldenscript::Runner for TestRunner {\n /// Runs a goldenscript command.\n fn run(&mut self, command: &goldenscript::Command) -> Result> {\n let mut output = String::new();\n match command.name.as_str() {\n // campaign [ID...]\n // Transition the given nodes to candidates and campaign.\n \"campaign\" => {\n let ids = self.parse_ids_or_all(&command.args)?;\n self.campaign(&ids, &mut output)?;\n }\n\n // cluster nodes=N [leader=ID] [heartbeat_interval=N] [election_timeout=N] [max_append_entries=N]\n // Creates a new Raft cluster.\n \"cluster\" => {\n let mut opts = Options::default();\n let mut args = command.consume_args();\n let nodes = args.lookup_parse(\"nodes\")?.unwrap_or(0);\n let leader = args.lookup_parse(\"leader\")?;\n if let Some(heartbeat_interval) = args.lookup_parse(\"heartbeat_interval\")? {\n opts.heartbeat_interval = heartbeat_interval;\n };\n if let Some(election_timeout) = args.lookup_parse(\"election_timeout\")? {\n opts.election_timeout_range = election_timeout..election_timeout + 1;\n }\n if let Some(max_append_entries) = args.lookup_parse(\"max_append_entries\")? {\n opts.max_append_entries = max_append_entries;\n }\n args.reject_rest()?;\n self.cluster(nodes, leader, opts, &mut output)?;\n }\n\n // deliver [from=ID] [ID...]\n // Delivers (steps) pending messages to the given nodes. If from\n // is given, only messages from the given node is delivered, the\n // others are left pending.\n \"deliver\" => {\n let mut args = command.consume_args();\n let from = args.lookup_parse(\"from\")?;\n let ids = self.parse_ids_or_all(&args.rest())?;\n self.deliver(&ids, from, &mut output)?;\n }\n\n // get ID KEY\n // Sends a client request to the given node to read the given\n // key from the state machine (key/value store).\n \"get\" => {\n let mut args = command.consume_args();\n let id = args.next_pos().ok_or(\"must specify node ID\")?.parse()?;\n let key = args.next_pos().ok_or(\"must specify key\")?.value.clone();\n args.reject_rest()?;\n let request = Request::Read(KVCommand::Get { key }.encode());\n self.request(id, request, &mut output)?;\n }\n\n // heal [ID...]\n // Heals all network partitions for the given nodes.\n \"heal\" => {\n let ids = self.parse_ids_or_all(&command.args)?;\n self.heal(&ids, &mut output)?;\n }\n\n // heartbeat ID...\n // Sends a heartbeat from the given leader nodes.\n \"heartbeat\" => {\n let ids = self.parse_ids_or_all(&command.args)?;\n self.heartbeat(&ids, &mut output)?;\n }\n\n // log [ID...]\n // Outputs the current Raft log for the given nodes.\n \"log\" => {\n let ids = self.parse_ids_or_all(&command.args)?;\n self.log(&ids, &mut output)?;\n }\n\n // partition ID...\n // Partitions the given nodes away from the rest of the cluster.\n // They can still communicate with each other, unless they were\n // previously partitioned.\n \"partition\" => {\n let ids = self.parse_ids_or_error(&command.args)?;\n self.partition(&ids, &mut output)?;\n }\n\n // put ID KEY=VALUE\n // Sends a client request to the given node to write a key/value\n // pair to the state machine (key/value store).\n \"put\" => {\n let mut args = command.consume_args();\n let id = args.next_pos().ok_or(\"must specify node ID\")?.parse()?;\n let kv = args.next_key().ok_or(\"must specify key/value pair\")?.clone();\n let (key, value) = (kv.key.unwrap(), kv.value);\n args.reject_rest()?;\n let request = Request::Write(KVCommand::Put { key, value }.encode());\n self.request(id, request, &mut output)?;\n }\n\n // restart [commit_index=INDEX] [applied_index=INDEX] [ID...]\n // Restarts the given nodes (or all nodes). They retain their\n // log and state, unless applied_index is given (which reverts\n // the state machine to the given index, or 0 if empty).\n // commit_index may be given to regress the commit index (it\n // is not flushed to durable storage).\n \"restart\" => {\n let mut args = command.consume_args();\n let applied_index = args.lookup_parse(\"applied_index\")?;\n let commit_index = args.lookup_parse(\"commit_index\")?;\n let ids = self.parse_ids_or_all(&args.rest())?;\n self.restart(&ids, commit_index, applied_index, &mut output)?;\n }\n\n // stabilize [heartbeat=BOOL] [ID...]\n // Stabilizes the given nodes by repeatedly delivering messages\n // until no more messages are pending. If heartbeat is true, also\n // emits a heartbeat from the leader and restabilizes, e.g. to\n // propagate the commit index.\n \"stabilize\" => {\n let mut args = command.consume_args();\n let heartbeat = args.lookup_parse(\"heartbeat\")?.unwrap_or(false);\n let ids = self.parse_ids_or_all(&args.rest())?;\n self.stabilize(&ids, heartbeat, &mut output)?;\n }\n\n // state [ID...]\n // Prints the current state machine contents on the given nodes.\n \"state\" => {\n let ids = self.parse_ids_or_all(&command.args)?;\n self.state(&ids, &mut output)?;\n }\n\n // status [request=BOOL] [ID...]\n // Prints the current node status of the given nodes. If request\n // is true, sends a status client request to a single node,\n // otherwise fetches status directly from each node.\n \"status\" => {\n let mut args = command.consume_args();\n let request = args.lookup_parse(\"request\")?.unwrap_or(false);\n let ids = self.parse_ids_or_all(&args.rest())?;\n if request {\n let [id] = *ids.as_slice() else {\n return Err(\"request=true requires 1 node ID\".into());\n };\n self.request(id, Request::Status, &mut output)?;\n } else {\n self.status(&ids, &mut output)?;\n }\n }\n\n // step ID JSON\n // Steps a manually generated JSON message on the given node.\n \"step\" => {\n let mut args = command.consume_args();\n let id = args.next_pos().ok_or(\"node ID not given\")?.parse()?;\n let raw = &args.next_pos().ok_or(\"message not given\")?.value;\n let msg = serde_json::from_str(raw)?;\n args.reject_rest()?;\n self.transition(id, |n| n.step(msg), &mut output)?;\n }\n\n // tick [ID...]\n // Ticks the given nodes.\n \"tick\" => {\n let ids = self.parse_ids_or_all(&command.args)?;\n for id in ids {\n self.transition(id, |n| n.tick(), &mut output)?;\n }\n }\n\n name => return Err(format!(\"unknown command {name}\").into()),\n }\n Ok(output)\n }\n }\n\n impl TestRunner {\n fn new() -> Self {\n Self {\n ids: Vec::new(),\n nodes: HashMap::new(),\n nodes_rx: HashMap::new(),\n nodes_pending: HashMap::new(),\n applied_rx: HashMap::new(),\n disconnected: HashMap::new(),\n requests: HashMap::new(),\n next_request_id: 1,\n tempdir: TempDir::with_prefix(\"toydb\").expect(\"tempdir failed\"),\n }\n }\n\n /// Creates a new empty node and inserts it.\n fn add_node(\n &mut self,\n id: NodeID,\n peers: HashSet,\n opts: Options,\n ) -> Result<(), Box> {\n // Use both a BitCask and a Memory engine, and mirror operations\n // across them, for added engine test coverage.\n let path = self.tempdir.path().join(format!(\"{id}.log\"));\n let bitcask = storage::BitCask::new(path).expect(\"bitcask failed\");\n let memory = storage::Memory::new();\n let engine = testengine::Mirror::new(bitcask, memory);\n let log = Log::new(Box::new(engine))?;\n let state = teststate::KV::new();\n self.add_node_with(id, peers, log, state, opts)\n }\n\n /// Creates a new node with the given log and state and inserts it.\n fn add_node_with(\n &mut self,\n id: NodeID,\n peers: HashSet,\n log: Log,\n state: Box,\n opts: Options,\n ) -> Result<(), Box> {\n let (node_tx, node_rx) = crossbeam::channel::unbounded();\n let (applied_tx, applied_rx) = crossbeam::channel::unbounded();\n let state = teststate::Emit::new(state, applied_tx);\n self.nodes.insert(id, Node::new(id, peers, log, state, node_tx, opts)?);\n self.nodes_rx.insert(id, node_rx);\n self.nodes_pending.insert(id, Vec::new());\n self.applied_rx.insert(id, applied_rx);\n self.disconnected.insert(id, HashSet::new());\n Ok(())\n }\n\n /// Transitions nodes to candidates and campaign in a new term.\n fn campaign(&mut self, ids: &[NodeID], output: &mut String) -> Result<(), Box> {\n let campaign = |node| match node {\n Node::Candidate(mut node) => {\n node.campaign()?;\n Ok(node.into())\n }\n Node::Follower(node) => Ok(node.into_candidate()?.into()),\n Node::Leader(node) => {\n let term = node.term();\n Ok(node.into_follower(term + 1)?.into_candidate()?.into())\n }\n };\n for id in ids.iter().copied() {\n self.transition(id, campaign, output)?;\n }\n Ok(())\n }\n\n /// Creates a Raft cluster.\n fn cluster(\n &mut self,\n nodes: u8,\n leader: Option,\n opts: Options,\n output: &mut String,\n ) -> Result<(), Box> {\n if !self.ids.is_empty() {\n return Err(\"cluster already exists\".into());\n }\n if nodes == 0 {\n return Err(\"cluster can't have 0 nodes\".into());\n }\n\n self.ids = (1..=nodes).collect();\n\n for id in self.ids.clone() {\n let peers = self.ids.iter().copied().filter(|i| i != &id).collect();\n self.add_node(id, peers, opts.clone())?;\n }\n\n // Promote leader if requested. Suppress output.\n if let Some(id) = leader {\n let quiet = &mut String::new();\n let Some(Node::Follower(node)) = self.nodes.remove(&id) else {\n return Err(format!(\"invalid leader {id}\").into());\n };\n self.nodes.insert(id, node.into_candidate()?.into_leader()?.into());\n self.receive(id, quiet)?;\n self.stabilize(&self.ids.clone(), true, quiet)?;\n }\n\n // Drain any initial applied entries.\n for applied_rx in self.applied_rx.values_mut() {\n while applied_rx.try_recv().is_ok() {}\n }\n\n // Output final cluster status.\n self.status(&self.ids, output)\n }\n\n /// Delivers pending messages to the given nodes. If from is given, only\n /// delivers messages from that node. Returns the number of delivered\n /// messages.\n fn deliver(\n &mut self,\n ids: &[NodeID],\n from: Option,\n output: &mut String,\n ) -> Result> {\n // Take a snapshot of the pending queues before delivering any\n // messages. This avoids outbound messages in response to delivery\n // being delivered to higher node IDs in the same loop, which can\n // give unintuitive results.\n let mut step = Vec::new();\n for id in ids.iter().copied() {\n let Some(pending) = self.nodes_pending.remove(&id) else {\n return Err(format!(\"unknown node {id}\").into());\n };\n let (deliver, requeue) =\n pending.into_iter().partition(|msg| from.is_none() || from == Some(msg.from));\n self.nodes_pending.insert(id, requeue);\n step.extend(deliver);\n }\n\n let delivered = step.len();\n for msg in step {\n self.transition(msg.to, |node| node.step(msg), output)?;\n }\n Ok(delivered)\n }\n\n /// Heals the given partitioned nodes, restoring connectivity with all\n /// other nodes.\n fn heal(&mut self, ids: &[NodeID], output: &mut String) -> Result<(), Box> {\n for id in ids.iter().copied() {\n self.disconnected.insert(id, HashSet::new());\n for peers in self.disconnected.values_mut() {\n peers.remove(&id);\n }\n }\n output.push_str(&Self::format_disconnected(&self.disconnected));\n Ok(())\n }\n\n /// Emits a heartbeat from the given leader nodes.\n fn heartbeat(&mut self, ids: &[NodeID], output: &mut String) -> Result<(), Box> {\n for id in ids.iter().copied() {\n let Some(Node::Leader(leader)) = self.nodes.get_mut(&id) else {\n return Err(format!(\"{id} is not a leader\").into());\n };\n leader.heartbeat()?;\n self.receive(id, output)?;\n }\n Ok(())\n }\n\n /// Outputs the current log contents for the given nodes.\n fn log(&mut self, ids: &[NodeID], output: &mut String) -> Result<(), Box> {\n for id in ids {\n let node = self.nodes.get_mut(id).ok_or(format!(\"unknown node {id}\"))?;\n let nodefmt = Self::format_node(node);\n let (last_index, last_term) = node.get_last_index();\n let (commit_index, commit_term) = node.get_commit_index();\n let (term, vote) = node.get_term_vote();\n writeln!(\n output,\n \"{nodefmt} term={term} last={last_index}@{last_term} commit={commit_index}@{commit_term} vote={vote:?}\",\n )?;\n for entry in node.scan_log()? {\n writeln!(output, \"{nodefmt} entry {}\", Self::format_entry(&entry))?;\n }\n }\n Ok(())\n }\n\n /// Partitions the given nodes from all other nodes in the cluster\n /// (bidirectionally). The given nodes can communicate with each other\n /// unless they were previously partitioned.\n fn partition(&mut self, ids: &[NodeID], output: &mut String) -> Result<(), Box> {\n let ids = HashSet::::from_iter(ids.iter().copied());\n for id in ids.iter().copied() {\n for peer in self.ids.iter().copied().filter(|p| !ids.contains(p)) {\n self.disconnected.entry(id).or_default().insert(peer);\n self.disconnected.entry(peer).or_default().insert(id);\n }\n }\n output.push_str(&Self::format_disconnected(&self.disconnected));\n Ok(())\n }\n\n /// Receives outbound messages from a node, prints them, and queues them\n /// for delivery. Returns the number of received messages.\n fn receive(&mut self, id: NodeID, output: &mut String) -> Result> {\n let rx = self.nodes_rx.get_mut(&id).ok_or(format!(\"unknown node {id}\"))?;\n let mut count = 0;\n for msg in rx.try_iter() {\n count += 1;\n let (from, term, to) = (msg.from, msg.term, msg.to); // simplify formatting\n let msgfmt = Self::format_message(&msg.message);\n\n // If the peer is disconnected, drop the message and output it.\n if self.disconnected[&msg.from].contains(&msg.to) {\n writeln!(\n output,\n \"n{from}@{term} ⇥ n{to} {}\",\n Self::format_strikethrough(&msgfmt),\n )?;\n continue;\n }\n\n // Intercept and output client responses.\n if msg.from == msg.to {\n let Message::ClientResponse { id, response } = &msg.message else {\n return Err(format!(\"invalid self-addressed message: {msg:?}\").into());\n };\n writeln!(output, \"n{from}@{term} → c{to} {msgfmt}\")?;\n let request = &self.requests.remove(id).ok_or(\"unknown request id\")?;\n writeln!(\n output,\n \"c{to}@{term} {} ⇒ {}\",\n Self::format_request(request),\n Self::format_response(response),\n )?;\n continue;\n }\n\n // Output the message and queue it for delivery.\n writeln!(output, \"n{from}@{term} → n{to} {msgfmt}\")?;\n self.nodes_pending.get_mut(&msg.to).ok_or(format!(\"unknown node {to}\"))?.push(msg);\n }\n Ok(count)\n }\n\n /// Submits a client request via the given node.\n fn request(\n &mut self,\n id: NodeID,\n request: Request,\n output: &mut String,\n ) -> Result<(), Box> {\n let request_id = Uuid::from_u64_pair(0, self.next_request_id);\n self.next_request_id += 1;\n self.requests.insert(request_id, request.clone());\n\n let term = self.nodes.get(&id).ok_or(format!(\"unknown node {id}\"))?.term();\n let msg = Envelope {\n from: id,\n to: id,\n term,\n message: Message::ClientRequest { id: request_id, request },\n };\n writeln!(output, \"c{id}@{term} → n{id} {}\", Self::format_message(&msg.message))?;\n self.transition(id, |n| n.step(msg), output)\n }\n\n /// Restarts the given nodes. If commit_index or applied_index are\n /// given, the log commit index or state machine will regress.\n fn restart(\n &mut self,\n ids: &[NodeID],\n commit_index: Option,\n applied_index: Option,\n output: &mut String,\n ) -> Result<(), Box> {\n for id in ids.iter().copied() {\n let node = self.nodes.remove(&id).ok_or(format!(\"unknown node {id}\"))?;\n let peers = node.peers();\n let opts = node.options();\n let (log, mut state) = node.dismantle();\n let mut log = Log::new(log.engine)?; // reset log\n\n // If requested, regress the commit index.\n if let Some(commit_index) = commit_index {\n if commit_index > log.get_commit_index().0 {\n return Err(format!(\"commit_index={commit_index} beyond current\").into());\n }\n let commit_term = match log.get(commit_index)? {\n Some(e) => e.term,\n None if commit_index == 0 => 0,\n None => return Err(format!(\"unknown commit_index={commit_index}\").into()),\n };\n log.engine.set(\n &crate::raft::log::Key::CommitIndex.encode(),\n bincode::serialize(&(commit_index, commit_term)),\n )?;\n // Reset the log again.\n log = Log::new(log.engine)?;\n }\n\n // If requested, wipe the state machine and reapply up to the\n // requested applied index.\n if let Some(applied_index) = applied_index {\n if applied_index > log.get_commit_index().0 {\n return Err(format!(\"applied_index={applied_index} beyond commit\").into());\n }\n state = teststate::KV::new();\n let mut scan = log.scan(..=applied_index);\n while let Some(entry) = scan.next().transpose()? {\n _ = state.apply(entry); // apply errors are returned to client\n }\n assert_eq!(state.get_applied_index(), applied_index, \"wrong applied index\");\n }\n\n // Add node, and run a noop transition to output applied entries.\n self.add_node_with(id, peers, log, state, opts)?;\n self.transition(id, Ok, output)?;\n }\n // Output restarted node status.\n self.status(ids, output)\n }\n\n /// Stabilizes the given nodes by repeatedly delivering pending messages\n /// until no new messages are generated. If heartbeat is true, leaders\n /// then emit a heartbeat and restabilize again, e.g. to propagate the\n /// commit index.\n fn stabilize(\n &mut self,\n ids: &[NodeID],\n heartbeat: bool,\n output: &mut String,\n ) -> Result<(), Box> {\n while self.deliver(ids, None, output)? > 0 {}\n // If requested, heartbeat the current leader (with the highest\n // term) and re-stabilize the nodes.\n if heartbeat {\n let leader = self\n .nodes\n .values()\n .sorted_by_key(|n| n.term())\n .rev()\n .find(|n| matches!(n, Node::Leader(_)));\n if let Some(leader) = leader {\n self.heartbeat(&[leader.id()], output)?;\n self.stabilize(ids, false, output)?;\n }\n }\n Ok(())\n }\n\n /// Outputs the current state machine for the given nodes.\n fn state(&mut self, ids: &[NodeID], output: &mut String) -> Result<(), Box> {\n for id in ids {\n let node = self.nodes.get_mut(id).ok_or(format!(\"unknown node {id}\"))?;\n let nodefmt = Self::format_node(node);\n let applied_index = node.get_applied_index();\n let raw = node.read(KVCommand::Scan.encode())?;\n let KVResponse::Scan(kvs) = KVResponse::decode(&raw)? else {\n return Err(\"unexpected scan response\".into());\n };\n writeln!(output, \"{nodefmt} applied={applied_index}\")?;\n for (key, value) in kvs {\n writeln!(output, \"{nodefmt} state {key}={value}\")?;\n }\n }\n Ok(())\n }\n\n /// Outputs status for the given nodes.\n fn status(&self, ids: &[NodeID], output: &mut String) -> Result<(), Box> {\n for id in ids {\n let node = self.nodes.get(id).ok_or(format!(\"unknown node {id}\"))?;\n let (last_index, last_term) = node.get_last_index();\n let (commit_index, commit_term) = node.get_commit_index();\n let applied_index = node.get_applied_index();\n write!(\n output,\n \"{node} last={last_index}@{last_term} commit={commit_index}@{commit_term} applied={applied_index}\",\n node = Self::format_node_role(node)\n )?;\n if let Node::Leader(leader) = node {\n let progress = leader\n .role\n .progress\n .iter()\n .sorted_by_key(|(id, _)| *id)\n .map(|(id, pr)| format!(\"{id}:{}→{}\", pr.match_index, pr.next_index))\n .join(\" \");\n write!(output, \" progress={{{progress}}}\")?\n }\n output.push('\\n');\n }\n Ok(())\n }\n\n /// Applies a node transition (typically a step or tick), and outputs\n /// relevant changes.\n fn transition(\n &mut self,\n id: NodeID,\n f: impl FnOnce(Node) -> crate::error::Result,\n output: &mut String,\n ) -> Result<(), Box> {\n let mut node = self.nodes.remove(&id).ok_or(format!(\"unknown node {id}\"))?;\n\n // Fetch pre-transition info.\n let old_noderole = Self::format_node_role(&node);\n let (old_commit_index, _) = node.get_commit_index();\n let mut old_entries = node.scan_log()?.into_iter();\n\n // Apply the transition.\n node = f(node)?;\n\n // Fetch post-transition info.\n let nodefmt = Self::format_node(&node);\n let noderole = Self::format_node_role(&node);\n let (commit_index, commit_term) = node.get_commit_index();\n\n let entries = node.scan_log()?.into_iter();\n let appended: Vec = entries\n .skip_while(|e| Some(e.term) == old_entries.next().map(|e| e.term))\n .collect();\n\n self.nodes.insert(id, node);\n\n // Output relevant changes.\n if old_noderole != noderole {\n writeln!(output, \"{old_noderole} ⇨ {noderole}\")?\n }\n for entry in appended {\n writeln!(output, \"{nodefmt} append {}\", Self::format_entry(&entry))?\n }\n if old_commit_index != commit_index {\n writeln!(output, \"{nodefmt} commit {commit_index}@{commit_term}\")?;\n }\n for entry in self.applied_rx[&id].try_iter() {\n writeln!(output, \"{nodefmt} apply {}\", Self::format_entry(&entry))?\n }\n\n // Receive any outbound messages.\n self.receive(id, output)?;\n Ok(())\n }\n\n /// Parses node IDs from the given argument values. Errors on key/value\n /// arguments. Can take both [Argument] and [&Argument].\n fn parse_ids(&self, args: &[A]) -> Result, Box>\n where\n A: Borrow,\n {\n let mut ids = Vec::new();\n for arg in args.iter().map(|a| a.borrow()) {\n if let Some(key) = &arg.key {\n return Err(format!(\"unknown argument '{key}'\").into());\n }\n let id = arg.parse()?;\n if !self.nodes.contains_key(&id) {\n return Err(format!(\"unknown node {id}\").into());\n }\n ids.push(id)\n }\n Ok(ids)\n }\n\n // Parses node IDs from the given argument values, or returns all node\n // IDs if none were given.\n fn parse_ids_or_all(&self, args: &[A]) -> Result, Box>\n where\n A: Borrow,\n {\n let ids = self.parse_ids(args)?;\n if ids.is_empty() {\n return Ok(self.ids.clone());\n }\n Ok(ids)\n }\n\n // Parses node IDs from the given argument values, or errors if none.\n fn parse_ids_or_error(&self, args: &[A]) -> Result, Box>\n where\n A: Borrow,\n {\n let ids = self.parse_ids(args)?;\n if ids.is_empty() {\n return Err(\"node ID not given\".into());\n }\n Ok(ids)\n }\n\n /// Formats network partitions.\n fn format_disconnected(disconnected: &HashMap>) -> String {\n // Return early if the cluster is fully connected.\n if disconnected.iter().all(|(_, peers)| peers.is_empty()) {\n return format!(\n \"{} fully connected\\n\",\n disconnected.keys().sorted().map(|id| format!(\"n{id}\")).join(\" \")\n );\n }\n\n let mut output = String::new();\n\n // Separate symmetric and asymmetric partitions.\n let mut symmetric: HashMap> = HashMap::new();\n let mut asymmetric: HashMap> = HashMap::new();\n for (id, peers) in disconnected {\n for peer in peers {\n if disconnected[peer].contains(id) {\n symmetric.entry(*id).or_default().insert(*peer);\n } else {\n asymmetric.entry(*id).or_default().insert(*peer);\n }\n }\n }\n\n // Anchor the symmetric partitions at the node with the largest number\n // of disconnects, otherwise the smallest (first) ID.\n for (id, peers) in &symmetric.clone() {\n for peer in peers {\n // Recompute the peer set sizes for each iteration, since we\n // modify the peer set below.\n let len = symmetric.get(id).map(|p| p.len()).unwrap_or(0);\n let peer_len = symmetric.get(peer).map(|p| p.len()).unwrap_or(0);\n // If this peer set is the smallest (or we're the higher ID),\n // remove the entry. We may no longer be in the map.\n if (len < peer_len || len == peer_len && id > peer)\n && let Some(peers) = symmetric.get_mut(id)\n {\n peers.remove(peer);\n if peers.is_empty() {\n symmetric.remove(id);\n }\n }\n }\n }\n\n // The values (HashSets) correspond to the RHS of a partition. Let's\n // group the LHS of the partition as well, from smallest to largest,\n // separately for symmetric and asymmetric partitions. The vector\n // contains (LHS, RHS, symmetric) groupings for each partition.\n let mut grouped: Vec<(HashSet, HashSet, bool)> = Vec::new();\n for (id, peers, symm) in symmetric\n .into_iter()\n .map(|(i, p)| (i, p, true))\n .chain(asymmetric.into_iter().map(|(i, p)| (i, p, false)))\n .sorted_by_key(|(id, _, symm)| (*id, !symm))\n {\n // Look for an existing LHS group with the same RHS, and insert\n // this node into it. Otherwise, create a new LHS group.\n match grouped.iter_mut().find(|(_, rhs, s)| peers == *rhs && symm == *s) {\n Some((lhs, _, _)) => _ = lhs.insert(id),\n None => grouped.push((HashSet::from([id]), peers, symm)),\n }\n }\n\n // Display the groups.\n for (lhs, rhs, symm) in grouped {\n let lhs = lhs.iter().sorted().map(|id| format!(\"n{id}\")).join(\" \");\n let sep = if symm { '⇹' } else { '⇥' };\n let rhs = rhs.iter().sorted().map(|id| format!(\"n{id}\")).join(\" \");\n writeln!(output, \"{lhs} {sep} {rhs}\").unwrap();\n }\n\n output\n }\n\n /// Formats an entry.\n fn format_entry(entry: &Entry) -> String {\n let command = match entry.command.as_ref() {\n Some(raw) => KVCommand::decode(raw).expect(\"invalid command\").to_string(),\n None => \"None\".to_string(),\n };\n format!(\"{index}@{term} {command}\", index = entry.index, term = entry.term)\n }\n\n /// Formats a message.\n fn format_message(msg: &Message) -> String {\n match msg {\n Message::Campaign { last_index, last_term } => {\n format!(\"Campaign last={last_index}@{last_term}\")\n }\n Message::CampaignResponse { vote } => {\n format!(\"CampaignResponse vote={vote}\")\n }\n Message::Heartbeat { last_index, commit_index, read_seq } => {\n format!(\n \"Heartbeat last_index={last_index} commit_index={commit_index} read_seq={read_seq}\"\n )\n }\n Message::HeartbeatResponse { match_index, read_seq } => {\n format!(\"HeartbeatResponse match_index={match_index} read_seq={read_seq}\")\n }\n Message::Append { base_index, base_term, entries } => {\n let ent = entries.iter().map(|e| format!(\"{}@{}\", e.index, e.term)).join(\" \");\n format!(\"Append base={base_index}@{base_term} [{ent}]\")\n }\n Message::AppendResponse { match_index, reject_index } => {\n match (match_index, reject_index) {\n (0, 0) => panic!(\"match_index and reject_index both 0\"),\n (match_index, 0) => format!(\"AppendResponse match_index={match_index}\"),\n (0, reject_index) => format!(\"AppendResponse reject_index={reject_index}\"),\n (_, _) => panic!(\"match_index and reject_index both set\"),\n }\n }\n Message::Read { seq } => {\n format!(\"Read seq={seq}\")\n }\n Message::ReadResponse { seq } => {\n format!(\"ReadResponse seq={seq}\")\n }\n Message::ClientRequest { id, request } => {\n format!(\n \"ClientRequest id=0x{} {}\",\n hex::encode(id).trim_start_matches(\"00\"),\n match request {\n Request::Read(v) => format!(\"read 0x{}\", hex::encode(v)),\n Request::Write(v) => format!(\"write 0x{}\", hex::encode(v)),\n Request::Status => \"status\".to_string(),\n }\n )\n }\n Message::ClientResponse { id, response } => {\n format!(\n \"ClientResponse id=0x{} {}\",\n hex::encode(id).trim_start_matches(\"00\"),\n match response {\n Ok(Response::Read(v)) => format!(\"read 0x{}\", hex::encode(v)),\n Ok(Response::Write(v)) => format!(\"write 0x{}\", hex::encode(v)),\n Ok(Response::Status(v)) => format!(\"status {v:?}\"),\n Err(error) => format!(\"Error::{error:#?}\"),\n }\n )\n }\n }\n }\n\n /// Formats a node identifier.\n fn format_node(node: &Node) -> String {\n format!(\"n{id}@{term}\", id = node.id(), term = node.term())\n }\n\n /// Formats a node identifier with role.\n fn format_node_role(node: &Node) -> String {\n let role = match node {\n Node::Candidate(_) => \"candidate\".to_string(),\n Node::Follower(node) => {\n let leader = node.role.leader.map(|id| format!(\"n{id}\")).unwrap_or_default();\n format!(\"follower({leader})\")\n }\n Node::Leader(_) => \"leader\".to_string(),\n };\n format!(\"{node} {role}\", node = Self::format_node(node))\n }\n\n /// Formats a request.\n fn format_request(request: &Request) -> String {\n match request {\n Request::Read(c) | Request::Write(c) => KVCommand::decode(c).unwrap().to_string(),\n Request::Status => \"status\".to_string(),\n }\n }\n\n /// Formats a response.\n fn format_response(response: &crate::error::Result) -> String {\n match response {\n Ok(Response::Read(r) | Response::Write(r)) => {\n KVResponse::decode(r).unwrap().to_string()\n }\n Ok(Response::Status(status)) => format!(\"{status:#?}\"),\n Err(error) => format!(\"Error::{error:?} ({error})\"),\n }\n }\n\n /// Strike-through formats the given string using a Unicode combining stroke.\n fn format_strikethrough(s: &str) -> String {\n s.chars().flat_map(|c| [c, '\\u{0336}']).collect()\n }\n }\n}\n" }, { "path": "src/raft/state.rs", "content": "use super::{Entry, Index};\nuse crate::error::Result;\n\n/// A Raft-managed state machine. Raft itself does not care what the state\n/// machine is, nor what the commands and results do -- it will simply apply\n/// arbitrary binary commands sequentially from the Raft log, returning an\n/// arbitrary binary result to the client.\n///\n/// Since commands are applied identically across all nodes, they must be\n/// deterministic and yield the same state and result across all nodes too.\n/// Otherwise, the nodes will diverge, such that different nodes will produce\n/// different results.\n///\n/// Write commands (`Request::Write`) are replicated and applied on all nodes\n/// via `State::apply`. The state machine must keep track of the last applied\n/// index and return it via `State::get_applied_index`. Read commands\n/// (`Request::Read`) are only executed on a single node via `State::read` and\n/// must not make any state changes.\npub trait State: Send {\n /// Returns the last applied log index from the state machine.\n ///\n /// This must correspond to the current state of the state machine, since it\n /// determines which command to apply next. In particular, a node crash may\n /// result in partial command application or data loss, which must be\n /// handled appropriately.\n fn get_applied_index(&self) -> Index;\n\n /// Applies a log entry to the state machine, returning a client result.\n /// Errors are considered applied and propagated back to the client.\n ///\n /// This is executed on all nodes, so the result must be deterministic: it\n /// must yield the same state and result on all nodes, even if the command\n /// is reapplied following a node crash.\n ///\n /// Any non-deterministic apply error (e.g. an IO error) must panic and\n /// crash the node -- if it instead returns an error to the client, the\n /// command is considered applied and node states will diverge. The state\n /// machine is responsible for panicing when appropriate.\n ///\n /// The entry may contain a noop command, which is committed by Raft during\n /// leader changes. This still needs to be applied to the state machine to\n /// properly update the applied index, and should return an empty result.\n fn apply(&mut self, entry: Entry) -> Result>;\n\n /// Executes a read command in the state machine, returning a client result.\n /// Errors are also propagated back to the client.\n ///\n /// This is only executed on a single node, so it must not result in any\n /// state changes (i.e. it must not write).\n fn read(&self, command: Vec) -> Result>;\n}\n\n/// Test helper state machines.\n#[cfg(test)]\npub mod test {\n use std::collections::BTreeMap;\n use std::fmt::Display;\n\n use crossbeam::channel::Sender;\n use itertools::Itertools as _;\n use serde::{Deserialize, Serialize};\n\n use super::*;\n use crate::encoding::{self, Value as _};\n\n /// Wraps a state machine and emits applied entries to the provided channel.\n pub struct Emit {\n inner: Box,\n tx: Sender,\n }\n\n impl Emit {\n pub fn new(inner: Box, tx: Sender) -> Box {\n Box::new(Self { inner, tx })\n }\n }\n\n impl State for Emit {\n fn get_applied_index(&self) -> Index {\n self.inner.get_applied_index()\n }\n\n fn apply(&mut self, entry: Entry) -> Result> {\n let response = self.inner.apply(entry.clone())?;\n self.tx.send(entry)?;\n Ok(response)\n }\n\n fn read(&self, command: Vec) -> Result> {\n self.inner.read(command)\n }\n }\n\n /// A simple string key/value store. Takes KVCommands.\n pub struct KV {\n applied_index: Index,\n data: BTreeMap,\n }\n\n impl KV {\n pub fn new() -> Box {\n Box::new(Self { applied_index: 0, data: BTreeMap::new() })\n }\n }\n\n impl State for KV {\n fn get_applied_index(&self) -> Index {\n self.applied_index\n }\n\n fn apply(&mut self, entry: Entry) -> Result> {\n let command = entry.command.as_deref().map(KVCommand::decode).transpose()?;\n let response = match command {\n Some(KVCommand::Put { key, value }) => {\n self.data.insert(key, value);\n KVResponse::Put(entry.index).encode()\n }\n Some(c @ (KVCommand::Get { .. } | KVCommand::Scan)) => {\n panic!(\"{c} submitted as write command\")\n }\n None => Vec::new(),\n };\n self.applied_index = entry.index;\n Ok(response)\n }\n\n fn read(&self, command: Vec) -> Result> {\n match KVCommand::decode(&command)? {\n KVCommand::Get { key } => {\n Ok(KVResponse::Get(self.data.get(&key).cloned()).encode())\n }\n KVCommand::Scan => Ok(KVResponse::Scan(self.data.clone()).encode()),\n c @ KVCommand::Put { .. } => panic!(\"{c} submitted as read command\"),\n }\n }\n }\n\n /// A KV command. Returns the corresponding KVResponse.\n #[derive(Serialize, Deserialize)]\n pub enum KVCommand {\n /// Fetches the value of the given key.\n Get { key: String },\n /// Stores the given key/value pair, returning the applied index.\n Put { key: String, value: String },\n /// Returns all key/value pairs.\n Scan,\n }\n\n impl encoding::Value for KVCommand {}\n\n impl Display for KVCommand {\n fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {\n match self {\n Self::Get { key } => write!(f, \"get {key}\"),\n Self::Put { key, value } => write!(f, \"put {key}={value}\"),\n Self::Scan => write!(f, \"scan\"),\n }\n }\n }\n\n /// A KVCommand response.\n #[derive(Serialize, Deserialize)]\n pub enum KVResponse {\n /// Get returns the key's value, or None if it does not exist.\n Get(Option),\n /// Put returns the applied index of the command.\n Put(Index),\n /// Scan returns the key/value pairs.\n Scan(BTreeMap),\n }\n\n impl encoding::Value for KVResponse {}\n\n impl Display for KVResponse {\n fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {\n match self {\n Self::Get(Some(value)) => write!(f, \"{value}\"),\n Self::Get(None) => write!(f, \"None\"),\n Self::Put(applied_index) => write!(f, \"{applied_index}\"),\n Self::Scan(kvs) => {\n write!(f, \"{}\", kvs.iter().map(|(k, v)| format!(\"{k}={v}\")).join(\",\"))\n }\n }\n }\n }\n\n /// A state machine which does nothing. All commands are ignored.\n pub struct Noop {\n applied_index: Index,\n }\n\n impl Noop {\n pub fn new() -> Box {\n Box::new(Self { applied_index: 0 })\n }\n }\n\n impl State for Noop {\n fn get_applied_index(&self) -> Index {\n self.applied_index\n }\n\n fn apply(&mut self, entry: Entry) -> Result> {\n self.applied_index = entry.index;\n Ok(Vec::new())\n }\n\n fn read(&self, _: Vec) -> Result> {\n Ok(Vec::new())\n }\n }\n}\n" }, { "path": "src/raft/testscripts/log/append", "content": "# Appending an entry with term 0 fails.\n!append foo\n---\nPanic: can't append entry in term 0\n\n# Appending to an empty log works. The term doesn't have to be 1. The entry is\n# written to the engine and flushed to durable storage.\nset_term 2\nappend foo [ops]\n---\nappend → 1@2 \"foo\"\nengine set raft:Entry(1) → 1@2 \"foo\" [\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x01\" → \"\\x01\\x02\\x01\\x03foo\"]\nengine flush\n\n# Appending a noop entry (no command) also works.\nappend [ops]\n---\nappend → 2@2 None\nengine set raft:Entry(2) → 2@2 None [\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\" → \"\\x02\\x02\\x00\"]\nengine flush\n\n# Check that the last index/term is updated (commit index isn't), and that\n# the engine contains the expected data, both in logical and raw form.\nstatus\nscan\ndump\n---\nterm=2 last=2@2 commit=0@0 vote=None\n1@2 \"foo\"\n2@2 None\nraft:Entry(1) → 1@2 \"foo\" [\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x01\" → \"\\x01\\x02\\x01\\x03foo\"]\nraft:Entry(2) → 2@2 None [\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\" → \"\\x02\\x02\\x00\"]\nraft:TermVote → term=2 vote=None [\"\\x01\" → \"\\x02\\x00\"]\n\n# Skipping a term then appending is allowed.\nset_term 3\nappend command\nset_term 5\nappend\n---\nappend → 3@3 \"command\"\nappend → 4@5 None\n\n# Dump the final status and data.\nstatus\nscan\ndump\n---\nterm=5 last=4@5 commit=0@0 vote=None\n1@2 \"foo\"\n2@2 None\n3@3 \"command\"\n4@5 None\nraft:Entry(1) → 1@2 \"foo\" [\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x01\" → \"\\x01\\x02\\x01\\x03foo\"]\nraft:Entry(2) → 2@2 None [\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\" → \"\\x02\\x02\\x00\"]\nraft:Entry(3) → 3@3 \"command\" [\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x03\" → \"\\x03\\x03\\x01\\x07command\"]\nraft:Entry(4) → 4@5 None [\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x04\" → \"\\x04\\x05\\x00\"]\nraft:TermVote → term=5 vote=None [\"\\x01\" → \"\\x05\\x00\"]\n" }, { "path": "src/raft/testscripts/log/commit", "content": "# Committing fails on an empty engine.\n!commit 1\n---\nPanic: commit index 1 does not exist\n\n# Add some entries.\nset_term 2\nsplice 1@1= 2@1=foo 3@2=bar\n---\nsplice → 3@2 \"bar\"\n\n# Committing entry 0 fails.\n!commit 0\n---\nPanic: commit index 0 does not exist\n\n# Committing entry 1 works, and updates the commit index.\n#\n# Show the engine operations too, and notice that the commit index isn't flushed\n# to durable storage (it can be recovered from the durable quorum logs).\ncommit 1 [ops]\nstatus\n---\ncommit → 1@1 None\nengine set raft:CommitIndex → 1@1 [\"\\x02\" → \"\\x01\\x01\"]\nterm=2 last=3@2 commit=1@1 vote=None\n\n# Dump the raw engine contents.\ndump\n---\nraft:Entry(1) → 1@1 None [\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x01\" → \"\\x01\\x01\\x00\"]\nraft:Entry(2) → 2@1 \"foo\" [\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\" → \"\\x02\\x01\\x01\\x03foo\"]\nraft:Entry(3) → 3@2 \"bar\" [\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x03\" → \"\\x03\\x02\\x01\\x03bar\"]\nraft:TermVote → term=2 vote=None [\"\\x01\" → \"\\x02\\x00\"]\nraft:CommitIndex → 1@1 [\"\\x02\" → \"\\x01\\x01\"]\n\n# Commits are idempotent, which doesn't incur an engine set.\ncommit 1 [ops]\nstatus\n---\ncommit → 1@1 None\nterm=2 last=3@2 commit=1@1 vote=None\n\n# Commits can skip an entry.\ncommit 3\nstatus\n---\ncommit → 3@2 \"bar\"\nterm=2 last=3@2 commit=3@2 vote=None\n\n# Commit regressions error.\n!commit 2\nstatus\n---\nPanic: commit index regression 3 → 2\nterm=2 last=3@2 commit=3@2 vote=None\n\n# Committing non-existant indexes error.\n!commit 4\nstatus\n---\nPanic: commit index 4 does not exist\nterm=2 last=3@2 commit=3@2 vote=None\n\n# Dump the raw values.\ndump\n---\nraft:Entry(1) → 1@1 None [\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x01\" → \"\\x01\\x01\\x00\"]\nraft:Entry(2) → 2@1 \"foo\" [\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\" → \"\\x02\\x01\\x01\\x03foo\"]\nraft:Entry(3) → 3@2 \"bar\" [\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x03\" → \"\\x03\\x02\\x01\\x03bar\"]\nraft:TermVote → term=2 vote=None [\"\\x01\" → \"\\x02\\x00\"]\nraft:CommitIndex → 3@2 [\"\\x02\" → \"\\x03\\x02\"]\n" }, { "path": "src/raft/testscripts/log/get", "content": "# get returns None on an empty engine.\nget 1\n---\nNone\n\n# Append a few entries.\nset_term 1\nappend\nappend foo\nset_term 2\nappend bar\n---\nappend → 1@1 None\nappend → 2@1 \"foo\"\nappend → 3@2 \"bar\"\n\n# get returns noop entries and regular entries.\nget 1 2\n---\n1@1 None\n2@1 \"foo\"\n\n# get returns None for missing entries, and for index 0.\nget 4 0\n---\nNone\nNone\n" }, { "path": "src/raft/testscripts/log/has", "content": "# has returns false on an empty engine.\nhas 1@1\n---\nfalse\n\n# Append a few entries.\nset_term 1\nappend\nappend foo\nset_term 2\nappend bar\n---\nappend → 1@1 None\nappend → 2@1 \"foo\"\nappend → 3@2 \"bar\"\n\n# has returns true both for noop entries and regular entries.\nhas 1@1 2@1\n---\ntrue\ntrue\n\n# has returns false for missing entries, including index 0.\nhas 4@2 0@0\n---\nfalse\nfalse\n\n# has returns false for term mismatches.\nhas 1@2 3@1 0@1\n---\nfalse\nfalse\nfalse\n" }, { "path": "src/raft/testscripts/log/init", "content": "# Tests that the log correctly initializes cached state when opened.\n\nset_term 1\n---\nok\n\nappend foo\nset_term 2 7\nappend bar\ncommit 1\n---\nappend → 1@1 \"foo\"\nappend → 2@2 \"bar\"\ncommit → 1@1 \"foo\"\n\nstatus\n---\nterm=2 last=2@2 commit=1@1 vote=7\n\nreload\n---\nok\n\nstatus\n---\nterm=2 last=2@2 commit=1@1 vote=7\n\nscan\n---\n1@1 \"foo\"\n2@2 \"bar\"\n" }, { "path": "src/raft/testscripts/log/scan", "content": "# scan works on an empty engine, even when given indexes.\nscan\nscan 3..7\n---\nok\n\n# Append a few entries.\nset_term 1\nappend\nappend foo\nset_term 2\nappend bar\n---\nappend → 1@1 None\nappend → 2@1 \"foo\"\nappend → 3@2 \"bar\"\n\n# Full scan.\nscan\n---\n1@1 None\n2@1 \"foo\"\n3@2 \"bar\"\n\n# Start bound.\nscan 2..\n---\n2@1 \"foo\"\n3@2 \"bar\"\n\nscan 4..\n---\nok\n\nscan 0..\n---\n1@1 None\n2@1 \"foo\"\n3@2 \"bar\"\n\n# End bound.\nscan \"..2\"\n---\n1@1 None\n\nscan \"..=2\"\n---\n1@1 None\n2@1 \"foo\"\n\nscan \"..7\"\n---\n1@1 None\n2@1 \"foo\"\n3@2 \"bar\"\n\nscan \"..1\"\n---\nok\n\nscan \"..0\"\n---\nok\n\n# Both bounds.\nscan 1..2\n---\n1@1 None\n\nscan \"1..=2\"\n---\n1@1 None\n2@1 \"foo\"\n\nscan 0..7\n---\n1@1 None\n2@1 \"foo\"\n3@2 \"bar\"\n\nscan 1..1\n---\nok\n\n# Bounds panics.\n!scan 1..0\n---\nPanic: range start is greater than range end in BTreeMap\n\n!scan 7..3\n---\nPanic: range start is greater than range end in BTreeMap\n" }, { "path": "src/raft/testscripts/log/scan_apply", "content": "# scan_apply works on an empty engine, even when given an applied index.\nscan_apply 0\nscan_apply 3\n---\nok\n\n# Append a few entries.\nset_term 1\nappend\nappend foo\nset_term 2\nappend bar\n---\nappend → 1@1 None\nappend → 2@1 \"foo\"\nappend → 3@2 \"bar\"\n\n# Nothing is committed, so scan_applied yields nothing.\nscan_apply 0\n---\nok\n\n# Commit the first two entries and apply them.\ncommit 2\nscan_apply 0\n---\ncommit → 2@1 \"foo\"\n1@1 None\n2@1 \"foo\"\n\n# Passing the commit index yields nothing.\nscan_apply 2\n---\nok\n\n# Passing an applied_index after the commit index is ok, and yields nothing.\nscan_apply 3\nscan_apply 10\n---\nok\n\n# Committing and applying the last entry works.\ncommit 3\nscan_apply 2\n---\ncommit → 3@2 \"bar\"\n3@2 \"bar\"\n\n# Scanning from a lower commit index again works.\nscan_apply 1\n---\n2@1 \"foo\"\n3@2 \"bar\"\n\nscan_apply 0\n---\n1@1 None\n2@1 \"foo\"\n3@2 \"bar\"\n" }, { "path": "src/raft/testscripts/log/splice", "content": "# Splicing at index 0 should fail.\n!splice 0@1=foo\n---\nPanic: spliced entry has index or term 0\n\n# Splicing without a term should fail.\n!splice 1@1=foo\n---\nPanic: splice term 1 beyond current 0\n\n\n\n# Splicing at index 2 should fail (creates gap).\nset_term 1\n!splice 2@1=foo\n---\nPanic: first index 2 must touch existing log\n\n# Splicing entries at start should work, both with and without commands, and\n# starting at a term after 1. They should be written to the engine and flushed\n# to durable storage. It should also update the state.\nset_term 2\nsplice 1@2= 2@2=command [ops]\nstatus\nscan\n---\nsplice → 2@2 \"command\"\nengine set raft:Entry(1) → 1@2 None [\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x01\" → \"\\x01\\x02\\x00\"]\nengine set raft:Entry(2) → 2@2 \"command\" [\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\" → \"\\x02\\x02\\x01\\x07command\"]\nengine flush\nterm=2 last=2@2 commit=0@0 vote=None\n1@2 None\n2@2 \"command\"\n\n# Splicing an empty list should work and be a noop.\nsplice [ops]\nstatus\nscan\n---\nsplice → 2@2 \"command\"\nterm=2 last=2@2 commit=0@0 vote=None\n1@2 None\n2@2 \"command\"\n\n# Splicing multiple duplicate entries should fail.\n!splice 3@2= 3@2=\n---\nPanic: spliced entries are not contiguous\n\n# Splicing entries with a gap should fail.\n!splice 3@2= 5@2=\n---\nPanic: spliced entries are not contiguous\n\n# Splicing entries with a term regression should fail.\n!splice 3@2= 4@1=\n---\nPanic: spliced entries have term regression\n\n# Splicing entries with a gap from the base should fail.\n!splice 4@2=\n---\nPanic: first index 4 must touch existing log\n\n# Splicing with a term regression from the base should fail.\n!splice 3@1=\n---\nPanic: splice term regression 2 → 1\n\n# Splicing with a term beyond the current term should fail.\n!splice 3@3=\n!splice 3@4=\n---\nPanic: splice term 3 beyond current 2\nPanic: splice term 4 beyond current 2\n\n# Fully overlapping entries is a noop.\nsplice 1@2= 2@2=command [ops]\nscan\n---\nsplice → 2@2 \"command\"\n1@2 None\n2@2 \"command\"\n\n# An overlapping prefix is a noop.\nsplice 1@2= [ops]\nscan\n---\nsplice → 2@2 \"command\"\n1@2 None\n2@2 \"command\"\n\n# An overlapping suffix is a noop.\nsplice 2@2=command [ops]\nscan\n---\nsplice → 2@2 \"command\"\n1@2 None\n2@2 \"command\"\n\n# Changing a command with the same term/index should fail.\n!splice 2@2=foo\nscan\n---\nPanic: command mismatch at Entry { index: 2, term: 2, command: Some([99, 111, 109, 109, 97, 110, 100]) }\n1@2 None\n2@2 \"command\"\n\n# Appending a new entry in the same term should work, as should\n# appending one in a new term.\nsplice 3@2=bar\nset_term 3\nsplice 4@3=\nscan\n---\nsplice → 3@2 \"bar\"\nsplice → 4@3 None\n1@2 None\n2@2 \"command\"\n3@2 \"bar\"\n4@3 None\n\n# Splicing with suffix overlap should work, and only write the new entries.\nsplice 3@2=bar 4@3= 5@3=foo 6@3=bar [ops]\nscan\n---\nsplice → 6@3 \"bar\"\nengine set raft:Entry(5) → 5@3 \"foo\" [\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x05\" → \"\\x05\\x03\\x01\\x03foo\"]\nengine set raft:Entry(6) → 6@3 \"bar\" [\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x06\" → \"\\x06\\x03\\x01\\x03bar\"]\nengine flush\n1@2 None\n2@2 \"command\"\n3@2 \"bar\"\n4@3 None\n5@3 \"foo\"\n6@3 \"bar\"\n\n# Splicing at an existing index with a new term should replace the tail.\nset_term 4\nsplice 4@4= [ops]\nstatus\nscan\n---\nsplice → 4@4 None\nengine set raft:Entry(4) → 4@4 None [\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x04\" → \"\\x04\\x04\\x00\"]\nengine delete raft:Entry(5) [\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x05\"]\nengine delete raft:Entry(6) [\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x06\"]\nengine flush\nterm=4 last=4@4 commit=0@0 vote=None\n1@2 None\n2@2 \"command\"\n3@2 \"bar\"\n4@4 None\n\n# This also holds at the start of the log.\nset_term 5\nsplice 1@5= 2@5=foo 3@5=bar [ops]\nstatus\nscan\n---\nsplice → 3@5 \"bar\"\nengine set raft:Entry(1) → 1@5 None [\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x01\" → \"\\x01\\x05\\x00\"]\nengine set raft:Entry(2) → 2@5 \"foo\" [\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\" → \"\\x02\\x05\\x01\\x03foo\"]\nengine set raft:Entry(3) → 3@5 \"bar\" [\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x03\" → \"\\x03\\x05\\x01\\x03bar\"]\nengine delete raft:Entry(4) [\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x04\"]\nengine flush\nterm=5 last=3@5 commit=0@0 vote=None\n1@5 None\n2@5 \"foo\"\n3@5 \"bar\"\n\n# Splicing across the commit index should work, as long as the entries match.\ncommit 2\nsplice 1@5= 2@5=foo 3@5=bar 4@5=\nstatus\nscan\n---\ncommit → 2@5 \"foo\"\nsplice → 4@5 None\nterm=5 last=4@5 commit=2@5 vote=None\n1@5 None\n2@5 \"foo\"\n3@5 \"bar\"\n4@5 None\n\n# Splicing across the commit index can replace a tail after the commit index.\nset_term 9\nsplice 3@6= 4@6=bar\nstatus\nscan\n---\nsplice → 4@6 \"bar\"\nterm=9 last=4@6 commit=2@5 vote=None\n1@5 None\n2@5 \"foo\"\n3@6 None\n4@6 \"bar\"\n\n# But replacing a tail at or before the commit index should fail.\n!splice 2@7=\n!splice 1@7=\n---\nPanic: spliced entries below commit index\nPanic: spliced entries below commit index\n\n# Dump the raw data.\ndump\n---\nraft:Entry(1) → 1@5 None [\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x01\" → \"\\x01\\x05\\x00\"]\nraft:Entry(2) → 2@5 \"foo\" [\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\" → \"\\x02\\x05\\x01\\x03foo\"]\nraft:Entry(3) → 3@6 None [\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x03\" → \"\\x03\\x06\\x00\"]\nraft:Entry(4) → 4@6 \"bar\" [\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x04\" → \"\\x04\\x06\\x01\\x03bar\"]\nraft:TermVote → term=9 vote=None [\"\\x01\" → \"\\t\\x00\"]\nraft:CommitIndex → 2@5 [\"\\x02\" → \"\\x02\\x05\"]\n" }, { "path": "src/raft/testscripts/log/status", "content": "# Status on empty engine works.\nstatus engine=true\n---\nterm=0 last=0@0 commit=0@0 vote=None engine=Status {\n name: \"bitcask\",\n keys: 0,\n size: 0,\n disk_size: 0,\n live_disk_size: 0,\n}\n\n# Write some data.\nset_term 1\nappend\nappend foo\nset_term 2 1\nappend bar\ncommit 2\n---\nappend → 1@1 None\nappend → 2@1 \"foo\"\nappend → 3@2 \"bar\"\ncommit → 2@1 \"foo\"\n\n# Status gives correct info.\nstatus engine=true\n---\nterm=2 last=3@2 commit=2@1 vote=1 engine=Status {\n name: \"bitcask\",\n keys: 5,\n size: 51,\n disk_size: 102,\n live_disk_size: 91,\n}\n" }, { "path": "src/raft/testscripts/log/term", "content": "# get_term works on empty engine.\nget_term\n---\nterm=0 vote=None\n\n# Storing a 0 term errors.\n!set_term 0\n---\nPanic: can't set term 0\n\n# set_term stores a term and empty vote, writing it to the engine\n# and flushing it to durable storage.\nset_term 3 [ops]\nget_term\n---\nengine set raft:TermVote → term=3 vote=None [\"\\x01\" → \"\\x03\\x00\"]\nengine flush\nterm=3 vote=None\n\n# set_term stores a term and vote.\nset_term 3 7 [ops]\nget_term\n---\nengine set raft:TermVote → term=3 vote=7 [\"\\x01\" → \"\\x03\\x01\\x07\"]\nengine flush\nterm=3 vote=7\n\n# set_term is idempotent, which doesn't incur an engine write.\nset_term 3 7 [ops]\nget_term\n---\nterm=3 vote=7\n\n# Moving the term into the far future is allowed.\nset_term 7\nget_term\n---\nterm=7 vote=None\n\n# Starting a new term with a vote is allowed.\nset_term 9 1\nget_term\n---\nterm=9 vote=1\n\n# Regressing the term errors.\n!set_term 8\n---\nPanic: term regression 9 → 8\n\n# Clearing the vote errors.\n!set_term 9\n---\nPanic: can't change vote\n\n# Changing the vote errors.\n!set_term 9 2\n---\nPanic: can't change vote\n\n# The above errors should not have changed the term/vote.\nget_term\ndump\n---\nterm=9 vote=1\nraft:TermVote → term=9 vote=1 [\"\\x01\" → \"\\t\\x01\\x01\"]\n" }, { "path": "src/raft/testscripts/node/append", "content": "# Can append single entries in steady state.\n\ncluster nodes=3 leader=1\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Propose a single write.\nput 1 foo=bar\n---\nc1@1 → n1 ClientRequest id=0x01 write 0x0103666f6f03626172\nn1@1 append 2@1 put foo=bar\nn1@1 → n2 Append base=1@1 [2@1]\nn1@1 → n3 Append base=1@1 [2@1]\n\nstatus\n---\nn1@1 leader last=2@1 commit=1@1 applied=1 progress={2:1→3 3:1→3}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Append it to both followers.\ndeliver\n---\nn2@1 append 2@1 put foo=bar\nn2@1 → n1 AppendResponse match_index=2\nn3@1 append 2@1 put foo=bar\nn3@1 → n1 AppendResponse match_index=2\n\n# The leader commits and applies the write.\nstabilize\n---\nn1@1 commit 2@1\nn1@1 apply 2@1 put foo=bar\nn1@1 → c1 ClientResponse id=0x01 write 0x0102\nc1@1 put foo=bar ⇒ 2\n\nstatus\n---\nn1@1 leader last=2@1 commit=2@1 applied=2 progress={2:2→3 3:2→3}\nn2@1 follower(n1) last=2@1 commit=1@1 applied=1\nn3@1 follower(n1) last=2@1 commit=1@1 applied=1\n" }, { "path": "src/raft/testscripts/node/append_base_missing", "content": "# Appends with a base beyond the node's last log entry should result in a\n# rejection at the index following the last entry, and the leader appending\n# the tail of the log.\n\ncluster nodes=3 leader=1\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Partition n3 so that it does not receive writes.\npartition 3\n---\nn3 ⇹ n1 n2\n\n# Replicate a couple of writes.\n(put 1 a=1)\n(put 1 b=2)\n(put 1 c=3)\n(stabilize heartbeat=true)\nstatus\n---\nn1@1 leader last=4@1 commit=4@1 applied=4 progress={2:4→5 3:1→5}\nn2@1 follower(n1) last=4@1 commit=4@1 applied=4\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Heal the partition, and propose another write.\nheal\nput 1 c=3\n---\nn1 n2 n3 fully connected\nc1@1 → n1 ClientRequest id=0x04 write 0x0101630133\nn1@1 append 5@1 put c=3\nn1@1 → n2 Append base=4@1 [5@1]\nn1@1 → n3 Append base=4@1 [5@1]\n\n# The 4@1 base is beyond n3's last index 1@1, so the append is rejected.\n# However, the follower returns reject_index=2 immediately after its\n# last index, rather than the original base index 4.\ndeliver 3\n---\nn3@1 → n1 AppendResponse reject_index=2\n\n# Because index 1 is already matched with the leader, it doesn't have to probe\n# and simply sends the entire tail, which is accepted.\ndeliver 1\nstatus 1\n---\nn1@1 → n3 Append base=1@1 [2@1 3@1 4@1 5@1]\nn1@1 leader last=5@1 commit=4@1 applied=4 progress={2:4→6 3:1→6}\n\ndeliver 3\n---\nn3@1 append 2@1 put a=1\nn3@1 append 3@1 put b=2\nn3@1 append 4@1 put c=3\nn3@1 append 5@1 put c=3\nn3@1 → n1 AppendResponse match_index=5\n\n# When n1 receives the ack, it commits and applies the write.\ndeliver 1\n---\nn1@1 commit 5@1\nn1@1 apply 5@1 put c=3\nn1@1 → c1 ClientResponse id=0x04 write 0x0105\nc1@1 put c=3 ⇒ 5\n\n# The progress is also updated.\nstatus\n---\nn1@1 leader last=5@1 commit=5@1 applied=5 progress={2:4→6 3:5→6}\nn2@1 follower(n1) last=4@1 commit=4@1 applied=4\nn3@1 follower(n1) last=5@1 commit=1@1 applied=1\n" }, { "path": "src/raft/testscripts/node/append_base_missing_all", "content": "# Appends to a node with an empty log should result in a rejection of index 1,\n# allowing the leader to send the entire log.\n\ncluster nodes=3\n---\nn1@0 follower() last=0@0 commit=0@0 applied=0\nn2@0 follower() last=0@0 commit=0@0 applied=0\nn3@0 follower() last=0@0 commit=0@0 applied=0\n\n# Partition n3 so that it does not receive writes.\npartition 3\n---\nn3 ⇹ n1 n2\n\n# Elect n1 as leader.\n(campaign 1)\n(stabilize)\nstatus\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:0→2}\nn2@1 follower(n1) last=1@1 commit=0@0 applied=0\nn3@0 follower() last=0@0 commit=0@0 applied=0\n\n# Replicate a couple of writes.\n(put 1 a=1)\n(put 1 b=2)\n(put 1 c=3)\n(stabilize heartbeat=true)\nstatus\n---\nn1@1 leader last=4@1 commit=4@1 applied=4 progress={2:4→5 3:0→5}\nn2@1 follower(n1) last=4@1 commit=4@1 applied=4\nn3@0 follower() last=0@0 commit=0@0 applied=0\n\n# Heal the partition, and propose another write.\nheal\nput 1 c=3\n---\nn1 n2 n3 fully connected\nc1@1 → n1 ClientRequest id=0x04 write 0x0101630133\nn1@1 append 5@1 put c=3\nn1@1 → n2 Append base=4@1 [5@1]\nn1@1 → n3 Append base=4@1 [5@1]\n\n# n3 has no entries, so it rejects with reject_index=1.\ndeliver 3\n---\nn3@0 follower() ⇨ n3@1 follower(n1)\nn3@1 → n1 AppendResponse reject_index=1\n\n# This allows n1 to send the entire log, without having to probe.\ndeliver 1\nstatus 1\n---\nn1@1 → n3 Append base=0@0 [1@1 2@1 3@1 4@1 5@1]\nn1@1 leader last=5@1 commit=4@1 applied=4 progress={2:4→6 3:0→6}\n\ndeliver 3\n---\nn3@1 append 1@1 None\nn3@1 append 2@1 put a=1\nn3@1 append 3@1 put b=2\nn3@1 append 4@1 put c=3\nn3@1 append 5@1 put c=3\nn3@1 → n1 AppendResponse match_index=5\n\n# When n1 receives the ack, it commits and applies the write.\ndeliver 1\n---\nn1@1 commit 5@1\nn1@1 apply 5@1 put c=3\nn1@1 → c1 ClientResponse id=0x04 write 0x0105\nc1@1 put c=3 ⇒ 5\n\n# The progress is also updated.\nstatus\n---\nn1@1 leader last=5@1 commit=5@1 applied=5 progress={2:4→6 3:5→6}\nn2@1 follower(n1) last=4@1 commit=4@1 applied=4\nn3@1 follower(n1) last=5@1 commit=0@0 applied=0\n" }, { "path": "src/raft/testscripts/node/append_commit_quorum", "content": "# Append results in a leader-side commit once a quorum is reached for the\n# relevant entries.\n\ncluster nodes=6 leader=1\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2 4:1→2 5:1→2 6:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\nn4@1 follower(n1) last=1@1 commit=1@1 applied=1\nn5@1 follower(n1) last=1@1 commit=1@1 applied=1\nn6@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Incrementally disconnect all nodes except one and then propose a write, to\n# generate an increasing quorum index.\n\n# Replicating 2 to n2 does not commit.\npartition 3 4 5 6\n---\nn1 n2 ⇹ n3 n4 n5 n6\n\nput 1 a=1\nstabilize\n---\nc1@1 → n1 ClientRequest id=0x01 write 0x0101610131\nn1@1 append 2@1 put a=1\nn1@1 → n2 Append base=1@1 [2@1]\nn1@1 ⇥ n3 A̶p̶p̶e̶n̶d̶ ̶b̶a̶s̶e̶=̶1̶@̶1̶ ̶[̶2̶@̶1̶]̶\nn1@1 ⇥ n4 A̶p̶p̶e̶n̶d̶ ̶b̶a̶s̶e̶=̶1̶@̶1̶ ̶[̶2̶@̶1̶]̶\nn1@1 ⇥ n5 A̶p̶p̶e̶n̶d̶ ̶b̶a̶s̶e̶=̶1̶@̶1̶ ̶[̶2̶@̶1̶]̶\nn1@1 ⇥ n6 A̶p̶p̶e̶n̶d̶ ̶b̶a̶s̶e̶=̶1̶@̶1̶ ̶[̶2̶@̶1̶]̶\nn2@1 append 2@1 put a=1\nn2@1 → n1 AppendResponse match_index=2\n\nstatus\n---\nn1@1 leader last=2@1 commit=1@1 applied=1 progress={2:2→3 3:1→3 4:1→3 5:1→3 6:1→3}\nn2@1 follower(n1) last=2@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\nn4@1 follower(n1) last=1@1 commit=1@1 applied=1\nn5@1 follower(n1) last=1@1 commit=1@1 applied=1\nn6@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Replicating 2-3 to n3 does not commit.\nheal\npartition 2 4 5 6\n---\nn1 n2 n3 n4 n5 n6 fully connected\nn1 n3 ⇹ n2 n4 n5 n6\n\nput 1 b=2\nstabilize\n---\nc1@1 → n1 ClientRequest id=0x02 write 0x0101620132\nn1@1 append 3@1 put b=2\nn1@1 ⇥ n2 A̶p̶p̶e̶n̶d̶ ̶b̶a̶s̶e̶=̶2̶@̶1̶ ̶[̶3̶@̶1̶]̶\nn1@1 → n3 Append base=2@1 [3@1]\nn1@1 ⇥ n4 A̶p̶p̶e̶n̶d̶ ̶b̶a̶s̶e̶=̶2̶@̶1̶ ̶[̶3̶@̶1̶]̶\nn1@1 ⇥ n5 A̶p̶p̶e̶n̶d̶ ̶b̶a̶s̶e̶=̶2̶@̶1̶ ̶[̶3̶@̶1̶]̶\nn1@1 ⇥ n6 A̶p̶p̶e̶n̶d̶ ̶b̶a̶s̶e̶=̶2̶@̶1̶ ̶[̶3̶@̶1̶]̶\nn3@1 → n1 AppendResponse reject_index=2\nn1@1 → n3 Append base=1@1 [2@1 3@1]\nn3@1 append 2@1 put a=1\nn3@1 append 3@1 put b=2\nn3@1 → n1 AppendResponse match_index=3\n\nstatus\n---\nn1@1 leader last=3@1 commit=1@1 applied=1 progress={2:2→4 3:3→4 4:1→4 5:1→4 6:1→4}\nn2@1 follower(n1) last=2@1 commit=1@1 applied=1\nn3@1 follower(n1) last=3@1 commit=1@1 applied=1\nn4@1 follower(n1) last=1@1 commit=1@1 applied=1\nn5@1 follower(n1) last=1@1 commit=1@1 applied=1\nn6@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Replicating 2-4 to n4 commits 2.\nheal\npartition 2 3 5 6\n---\nn1 n2 n3 n4 n5 n6 fully connected\nn1 n4 ⇹ n2 n3 n5 n6\n\nput 1 c=3\nstabilize\n---\nc1@1 → n1 ClientRequest id=0x03 write 0x0101630133\nn1@1 append 4@1 put c=3\nn1@1 ⇥ n2 A̶p̶p̶e̶n̶d̶ ̶b̶a̶s̶e̶=̶3̶@̶1̶ ̶[̶4̶@̶1̶]̶\nn1@1 ⇥ n3 A̶p̶p̶e̶n̶d̶ ̶b̶a̶s̶e̶=̶3̶@̶1̶ ̶[̶4̶@̶1̶]̶\nn1@1 → n4 Append base=3@1 [4@1]\nn1@1 ⇥ n5 A̶p̶p̶e̶n̶d̶ ̶b̶a̶s̶e̶=̶3̶@̶1̶ ̶[̶4̶@̶1̶]̶\nn1@1 ⇥ n6 A̶p̶p̶e̶n̶d̶ ̶b̶a̶s̶e̶=̶3̶@̶1̶ ̶[̶4̶@̶1̶]̶\nn4@1 → n1 AppendResponse reject_index=2\nn1@1 → n4 Append base=1@1 [2@1 3@1 4@1]\nn4@1 append 2@1 put a=1\nn4@1 append 3@1 put b=2\nn4@1 append 4@1 put c=3\nn4@1 → n1 AppendResponse match_index=4\nn1@1 commit 2@1\nn1@1 apply 2@1 put a=1\nn1@1 → c1 ClientResponse id=0x01 write 0x0102\nc1@1 put a=1 ⇒ 2\n\nstatus\n---\nn1@1 leader last=4@1 commit=2@1 applied=2 progress={2:2→5 3:3→5 4:4→5 5:1→5 6:1→5}\nn2@1 follower(n1) last=2@1 commit=1@1 applied=1\nn3@1 follower(n1) last=3@1 commit=1@1 applied=1\nn4@1 follower(n1) last=4@1 commit=1@1 applied=1\nn5@1 follower(n1) last=1@1 commit=1@1 applied=1\nn6@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Replicating 2-5 to n5 commits 3.\nheal\npartition 2 3 4 6\n---\nn1 n2 n3 n4 n5 n6 fully connected\nn1 n5 ⇹ n2 n3 n4 n6\n\nput 1 d=4\nstabilize\n---\nc1@1 → n1 ClientRequest id=0x04 write 0x0101640134\nn1@1 append 5@1 put d=4\nn1@1 ⇥ n2 A̶p̶p̶e̶n̶d̶ ̶b̶a̶s̶e̶=̶4̶@̶1̶ ̶[̶5̶@̶1̶]̶\nn1@1 ⇥ n3 A̶p̶p̶e̶n̶d̶ ̶b̶a̶s̶e̶=̶4̶@̶1̶ ̶[̶5̶@̶1̶]̶\nn1@1 ⇥ n4 A̶p̶p̶e̶n̶d̶ ̶b̶a̶s̶e̶=̶4̶@̶1̶ ̶[̶5̶@̶1̶]̶\nn1@1 → n5 Append base=4@1 [5@1]\nn1@1 ⇥ n6 A̶p̶p̶e̶n̶d̶ ̶b̶a̶s̶e̶=̶4̶@̶1̶ ̶[̶5̶@̶1̶]̶\nn5@1 → n1 AppendResponse reject_index=2\nn1@1 → n5 Append base=1@1 [2@1 3@1 4@1 5@1]\nn5@1 append 2@1 put a=1\nn5@1 append 3@1 put b=2\nn5@1 append 4@1 put c=3\nn5@1 append 5@1 put d=4\nn5@1 → n1 AppendResponse match_index=5\nn1@1 commit 3@1\nn1@1 apply 3@1 put b=2\nn1@1 → c1 ClientResponse id=0x02 write 0x0103\nc1@1 put b=2 ⇒ 3\n\nstatus\n---\nn1@1 leader last=5@1 commit=3@1 applied=3 progress={2:2→6 3:3→6 4:4→6 5:5→6 6:1→6}\nn2@1 follower(n1) last=2@1 commit=1@1 applied=1\nn3@1 follower(n1) last=3@1 commit=1@1 applied=1\nn4@1 follower(n1) last=4@1 commit=1@1 applied=1\nn5@1 follower(n1) last=5@1 commit=1@1 applied=1\nn6@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Replicating 2-6 to n6 commits 4.\nheal\npartition 2 3 4 5\n---\nn1 n2 n3 n4 n5 n6 fully connected\nn1 n6 ⇹ n2 n3 n4 n5\n\nput 1 e=5\nstabilize\n---\nc1@1 → n1 ClientRequest id=0x05 write 0x0101650135\nn1@1 append 6@1 put e=5\nn1@1 ⇥ n2 A̶p̶p̶e̶n̶d̶ ̶b̶a̶s̶e̶=̶5̶@̶1̶ ̶[̶6̶@̶1̶]̶\nn1@1 ⇥ n3 A̶p̶p̶e̶n̶d̶ ̶b̶a̶s̶e̶=̶5̶@̶1̶ ̶[̶6̶@̶1̶]̶\nn1@1 ⇥ n4 A̶p̶p̶e̶n̶d̶ ̶b̶a̶s̶e̶=̶5̶@̶1̶ ̶[̶6̶@̶1̶]̶\nn1@1 ⇥ n5 A̶p̶p̶e̶n̶d̶ ̶b̶a̶s̶e̶=̶5̶@̶1̶ ̶[̶6̶@̶1̶]̶\nn1@1 → n6 Append base=5@1 [6@1]\nn6@1 → n1 AppendResponse reject_index=2\nn1@1 → n6 Append base=1@1 [2@1 3@1 4@1 5@1 6@1]\nn6@1 append 2@1 put a=1\nn6@1 append 3@1 put b=2\nn6@1 append 4@1 put c=3\nn6@1 append 5@1 put d=4\nn6@1 append 6@1 put e=5\nn6@1 → n1 AppendResponse match_index=6\nn1@1 commit 4@1\nn1@1 apply 4@1 put c=3\nn1@1 → c1 ClientResponse id=0x03 write 0x0104\nc1@1 put c=3 ⇒ 4\n\nstatus\n---\nn1@1 leader last=6@1 commit=4@1 applied=4 progress={2:2→7 3:3→7 4:4→7 5:5→7 6:6→7}\nn2@1 follower(n1) last=2@1 commit=1@1 applied=1\nn3@1 follower(n1) last=3@1 commit=1@1 applied=1\nn4@1 follower(n1) last=4@1 commit=1@1 applied=1\nn5@1 follower(n1) last=5@1 commit=1@1 applied=1\nn6@1 follower(n1) last=6@1 commit=1@1 applied=1\n\n# Healing the partition and proposing another write replicates and commits all\n# entries.\nheal\n---\nn1 n2 n3 n4 n5 n6 fully connected\n\nput 1 f=6\nstabilize\n---\nc1@1 → n1 ClientRequest id=0x06 write 0x0101660136\nn1@1 append 7@1 put f=6\nn1@1 → n2 Append base=6@1 [7@1]\nn1@1 → n3 Append base=6@1 [7@1]\nn1@1 → n4 Append base=6@1 [7@1]\nn1@1 → n5 Append base=6@1 [7@1]\nn1@1 → n6 Append base=6@1 [7@1]\nn2@1 → n1 AppendResponse reject_index=3\nn3@1 → n1 AppendResponse reject_index=4\nn4@1 → n1 AppendResponse reject_index=5\nn5@1 → n1 AppendResponse reject_index=6\nn6@1 append 7@1 put f=6\nn6@1 → n1 AppendResponse match_index=7\nn1@1 → n2 Append base=2@1 [3@1 4@1 5@1 6@1 7@1]\nn1@1 → n3 Append base=3@1 [4@1 5@1 6@1 7@1]\nn1@1 → n4 Append base=4@1 [5@1 6@1 7@1]\nn1@1 → n5 Append base=5@1 [6@1 7@1]\nn2@1 append 3@1 put b=2\nn2@1 append 4@1 put c=3\nn2@1 append 5@1 put d=4\nn2@1 append 6@1 put e=5\nn2@1 append 7@1 put f=6\nn2@1 → n1 AppendResponse match_index=7\nn3@1 append 4@1 put c=3\nn3@1 append 5@1 put d=4\nn3@1 append 6@1 put e=5\nn3@1 append 7@1 put f=6\nn3@1 → n1 AppendResponse match_index=7\nn4@1 append 5@1 put d=4\nn4@1 append 6@1 put e=5\nn4@1 append 7@1 put f=6\nn4@1 → n1 AppendResponse match_index=7\nn5@1 append 6@1 put e=5\nn5@1 append 7@1 put f=6\nn5@1 → n1 AppendResponse match_index=7\nn1@1 commit 5@1\nn1@1 apply 5@1 put d=4\nn1@1 → c1 ClientResponse id=0x04 write 0x0105\nc1@1 put d=4 ⇒ 5\nn1@1 commit 7@1\nn1@1 apply 6@1 put e=5\nn1@1 apply 7@1 put f=6\nn1@1 → c1 ClientResponse id=0x05 write 0x0106\nc1@1 put e=5 ⇒ 6\nn1@1 → c1 ClientResponse id=0x06 write 0x0107\nc1@1 put f=6 ⇒ 7\n\nstatus\n---\nn1@1 leader last=7@1 commit=7@1 applied=7 progress={2:7→8 3:7→8 4:7→8 5:7→8 6:7→8}\nn2@1 follower(n1) last=7@1 commit=1@1 applied=1\nn3@1 follower(n1) last=7@1 commit=1@1 applied=1\nn4@1 follower(n1) last=7@1 commit=1@1 applied=1\nn5@1 follower(n1) last=7@1 commit=1@1 applied=1\nn6@1 follower(n1) last=7@1 commit=1@1 applied=1\n" }, { "path": "src/raft/testscripts/node/append_initial", "content": "# An initial append at base 0 can have a single or multiple entries.\n\ncluster nodes=3\n---\nn1@0 follower() last=0@0 commit=0@0 applied=0\nn2@0 follower() last=0@0 commit=0@0 applied=0\nn3@0 follower() last=0@0 commit=0@0 applied=0\n\n# Partition n3 so that is has an empty log.\npartition 3\n---\nn3 ⇹ n1 n2\n\n# n1 campaigns.\ncampaign 1\ndeliver\n---\nn1@0 follower() ⇨ n1@1 candidate\nn1@1 → n2 Campaign last=0@0\nn1@1 ⇥ n3 C̶a̶m̶p̶a̶i̶g̶n̶ ̶l̶a̶s̶t̶=̶0̶@̶0̶\nn2@0 follower() ⇨ n2@1 follower()\nn2@1 → n1 CampaignResponse vote=true\n\n# When n1 wins, it successfully appends an entry at base 0 to n2.\nstabilize\n---\nn1@1 candidate ⇨ n1@1 leader\nn1@1 append 1@1 None\nn1@1 → n2 Append base=0@0 [1@1]\nn1@1 ⇥ n3 A̶p̶p̶e̶n̶d̶ ̶b̶a̶s̶e̶=̶0̶@̶0̶ ̶[̶1̶@̶1̶]̶\nn1@1 → n2 Heartbeat last_index=1 commit_index=0 read_seq=0\nn1@1 ⇥ n3 H̶e̶a̶r̶t̶b̶e̶a̶t̶ ̶l̶a̶s̶t̶_̶i̶n̶d̶e̶x̶=̶1̶ ̶c̶o̶m̶m̶i̶t̶_̶i̶n̶d̶e̶x̶=̶0̶ ̶r̶e̶a̶d̶_̶s̶e̶q̶=̶0̶\nn2@1 follower() ⇨ n2@1 follower(n1)\nn2@1 append 1@1 None\nn2@1 → n1 AppendResponse match_index=1\nn2@1 → n1 HeartbeatResponse match_index=1 read_seq=0\nn1@1 commit 1@1\nn1@1 apply 1@1 None\n\nstatus\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:0→2}\nn2@1 follower(n1) last=1@1 commit=0@0 applied=0\nn3@0 follower() last=0@0 commit=0@0 applied=0\n\n# Heal the partition.\nheal\n---\nn1 n2 n3 fully connected\n\n# Propose a write. This appends entry 2 to n2 at base 1, but is rejected by n3\n# which doesn't have entry 1.\nput 1 foo=bar\ndeliver\n---\nc1@1 → n1 ClientRequest id=0x01 write 0x0103666f6f03626172\nn1@1 append 2@1 put foo=bar\nn1@1 → n2 Append base=1@1 [2@1]\nn1@1 → n3 Append base=1@1 [2@1]\nn2@1 append 2@1 put foo=bar\nn2@1 → n1 AppendResponse match_index=2\nn3@0 follower() ⇨ n3@1 follower(n1)\nn3@1 → n1 AppendResponse reject_index=1\n\n# Since n3 rejected base 1, n1 sends an append with all messages, which\n# is accepted.\nstabilize\n---\nn1@1 commit 2@1\nn1@1 apply 2@1 put foo=bar\nn1@1 → c1 ClientResponse id=0x01 write 0x0102\nc1@1 put foo=bar ⇒ 2\nn1@1 → n3 Append base=0@0 [1@1 2@1]\nn3@1 append 1@1 None\nn3@1 append 2@1 put foo=bar\nn3@1 → n1 AppendResponse match_index=2\n\nlog\n---\nn1@1 term=1 last=2@1 commit=2@1 vote=Some(1)\nn1@1 entry 1@1 None\nn1@1 entry 2@1 put foo=bar\nn2@1 term=1 last=2@1 commit=0@0 vote=Some(1)\nn2@1 entry 1@1 None\nn2@1 entry 2@1 put foo=bar\nn3@1 term=1 last=2@1 commit=0@0 vote=None\nn3@1 entry 1@1 None\nn3@1 entry 2@1 put foo=bar\n" }, { "path": "src/raft/testscripts/node/append_max_entries", "content": "# Large appends are limited to MAX_APPEND_ENTRIES, and each successful append\n# triggers the next append batch.\n\ncluster nodes=3 leader=1 max_append_entries=2\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Partition n3.\npartition 3\n---\nn3 ⇹ n1 n2\n\n# Make a bunch of writes.\n(put 1 a=1)\n(put 1 a=2)\n(put 1 a=3)\n(put 1 a=4)\n(put 1 a=5)\n(put 1 a=6)\n(put 1 a=7)\n(stabilize heartbeat=true)\nstatus\n---\nn1@1 leader last=8@1 commit=8@1 applied=8 progress={2:8→9 3:1→9}\nn2@1 follower(n1) last=8@1 commit=8@1 applied=8\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Heal the partition.\nheal\n---\nn1 n2 n3 fully connected\n\n# The next heartbeat triggers a probe.\nheartbeat 1\ndeliver\ndeliver\ndeliver\n---\nn1@1 → n2 Heartbeat last_index=8 commit_index=8 read_seq=0\nn1@1 → n3 Heartbeat last_index=8 commit_index=8 read_seq=0\nn2@1 → n1 HeartbeatResponse match_index=8 read_seq=0\nn3@1 → n1 HeartbeatResponse match_index=0 read_seq=0\nn1@1 → n3 Append base=7@1 []\nn3@1 → n1 AppendResponse reject_index=2\n\n# When the leader receives the probe response, it begins appending in batches of\n# max_append_entries until the follower is caught up.\nstabilize\n---\nn1@1 → n3 Append base=1@1 [2@1 3@1]\nn3@1 append 2@1 put a=1\nn3@1 append 3@1 put a=2\nn3@1 → n1 AppendResponse match_index=3\nn1@1 → n3 Append base=3@1 [4@1 5@1]\nn3@1 append 4@1 put a=3\nn3@1 append 5@1 put a=4\nn3@1 → n1 AppendResponse match_index=5\nn1@1 → n3 Append base=5@1 [6@1 7@1]\nn3@1 append 6@1 put a=5\nn3@1 append 7@1 put a=6\nn3@1 → n1 AppendResponse match_index=7\nn1@1 → n3 Append base=7@1 [8@1]\nn3@1 append 8@1 put a=7\nn3@1 → n1 AppendResponse match_index=8\n" }, { "path": "src/raft/testscripts/node/append_pipeline", "content": "# Multiple appends are pipelined before acks are received, without\n# retransmitting the unacked entries.\n\ncluster nodes=3 leader=1\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Propose a single write. The progress next index increases to 3.\nput 1 a=1\n---\nc1@1 → n1 ClientRequest id=0x01 write 0x0101610131\nn1@1 append 2@1 put a=1\nn1@1 → n2 Append base=1@1 [2@1]\nn1@1 → n3 Append base=1@1 [2@1]\n\nstatus\n---\nn1@1 leader last=2@1 commit=1@1 applied=1 progress={2:1→3 3:1→3}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Propose two more writes. Appends are sent without past duplicates.\nput 1 b=2\nput 1 c=3\n---\nc1@1 → n1 ClientRequest id=0x02 write 0x0101620132\nn1@1 append 3@1 put b=2\nn1@1 → n2 Append base=2@1 [3@1]\nn1@1 → n3 Append base=2@1 [3@1]\nc1@1 → n1 ClientRequest id=0x03 write 0x0101630133\nn1@1 append 4@1 put c=3\nn1@1 → n2 Append base=3@1 [4@1]\nn1@1 → n3 Append base=3@1 [4@1]\n\nstatus\n---\nn1@1 leader last=4@1 commit=1@1 applied=1 progress={2:1→5 3:1→5}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# The appends are received and acked sequentially.\ndeliver\n---\nn2@1 append 2@1 put a=1\nn2@1 → n1 AppendResponse match_index=2\nn2@1 append 3@1 put b=2\nn2@1 → n1 AppendResponse match_index=3\nn2@1 append 4@1 put c=3\nn2@1 → n1 AppendResponse match_index=4\nn3@1 append 2@1 put a=1\nn3@1 → n1 AppendResponse match_index=2\nn3@1 append 3@1 put b=2\nn3@1 → n1 AppendResponse match_index=3\nn3@1 append 4@1 put c=3\nn3@1 → n1 AppendResponse match_index=4\n\n# The leader receives the acks and commits the writes one by one,\n# without retransmitting the in-flight (to it) entries.\ndeliver\n---\nn1@1 commit 2@1\nn1@1 apply 2@1 put a=1\nn1@1 → c1 ClientResponse id=0x01 write 0x0102\nc1@1 put a=1 ⇒ 2\nn1@1 commit 3@1\nn1@1 apply 3@1 put b=2\nn1@1 → c1 ClientResponse id=0x02 write 0x0103\nc1@1 put b=2 ⇒ 3\nn1@1 commit 4@1\nn1@1 apply 4@1 put c=3\nn1@1 → c1 ClientResponse id=0x03 write 0x0104\nc1@1 put c=3 ⇒ 4\n\n# All nodes are now caught up on logs (but not commit/apply, which needs a\n# heartbeat).\nstatus\n---\nn1@1 leader last=4@1 commit=4@1 applied=4 progress={2:4→5 3:4→5}\nn2@1 follower(n1) last=4@1 commit=1@1 applied=1\nn3@1 follower(n1) last=4@1 commit=1@1 applied=1\n" }, { "path": "src/raft/testscripts/node/append_probe_divergent_first", "content": "# Appends to a previous leader and follower with a divergent tail all\n# the way back to the first entry works.\n\ncluster nodes=5 leader=1\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2 4:1→2 5:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\nn4@1 follower(n1) last=1@1 commit=1@1 applied=1\nn5@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Partition n1-n2\npartition 1 2\n---\nn1 n2 ⇹ n3 n4 n5\n\n# Elect new leaders in the majority partition and replicate a few writes.\n# Multiple leaders ensures the log has multiple terms.\n(campaign 3)\n(stabilize)\n(put 3 a=1)\n(stabilize heartbeat=true)\nstatus\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2 4:1→2 5:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@2 leader last=3@2 commit=3@2 applied=3 progress={1:0→4 2:0→4 4:3→4 5:3→4}\nn4@2 follower(n3) last=3@2 commit=3@2 applied=3\nn5@2 follower(n3) last=3@2 commit=3@2 applied=3\n\n(campaign 4)\n(stabilize)\n(put 4 b=2)\n(stabilize heartbeat=true)\nstatus\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2 4:1→2 5:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@3 follower(n4) last=5@3 commit=5@3 applied=5\nn4@3 leader last=5@3 commit=5@3 applied=5 progress={1:0→6 2:0→6 3:5→6 5:5→6}\nn5@3 follower(n4) last=5@3 commit=5@3 applied=5\n\n(campaign 5)\n(stabilize)\n(put 5 c=3)\n(stabilize heartbeat=true)\nstatus\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2 4:1→2 5:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@4 follower(n5) last=7@4 commit=7@4 applied=7\nn4@4 follower(n5) last=7@4 commit=7@4 applied=7\nn5@4 leader last=7@4 commit=7@4 applied=7 progress={1:0→8 2:0→8 3:7→8 4:7→8}\n\n# Propose writes in the minority partition as well.\n(put 1 a=2)\n(put 1 a=3)\n(put 1 a=4)\n(put 1 a=5)\n(put 1 a=6)\n(put 1 a=7)\n(stabilize)\nstatus\n---\nn1@1 leader last=7@1 commit=1@1 applied=1 progress={2:7→8 3:1→8 4:1→8 5:1→8}\nn2@1 follower(n1) last=7@1 commit=1@1 applied=1\nn3@4 follower(n5) last=7@4 commit=7@4 applied=7\nn4@4 follower(n5) last=7@4 commit=7@4 applied=7\nn5@4 leader last=7@4 commit=7@4 applied=7 progress={1:0→8 2:0→8 3:7→8 4:7→8}\n\nlog 1 5\n---\nn1@1 term=1 last=7@1 commit=1@1 vote=Some(1)\nn1@1 entry 1@1 None\nn1@1 entry 2@1 put a=2\nn1@1 entry 3@1 put a=3\nn1@1 entry 4@1 put a=4\nn1@1 entry 5@1 put a=5\nn1@1 entry 6@1 put a=6\nn1@1 entry 7@1 put a=7\nn5@4 term=4 last=7@4 commit=7@4 vote=Some(5)\nn5@4 entry 1@1 None\nn5@4 entry 2@2 None\nn5@4 entry 3@2 put a=1\nn5@4 entry 4@3 None\nn5@4 entry 5@3 put b=2\nn5@4 entry 6@4 None\nn5@4 entry 7@4 put c=3\n\n# Heal the partition.\nheal\n---\nn1 n2 n3 n4 n5 fully connected\n\n# Propose another write on the majority leader.\nput 5 d=4\n---\nc5@4 → n5 ClientRequest id=0x0a write 0x0101640134\nn5@4 append 8@4 put d=4\nn5@4 → n1 Append base=7@4 [8@4]\nn5@4 → n2 Append base=7@4 [8@4]\nn5@4 → n3 Append base=7@4 [8@4]\nn5@4 → n4 Append base=7@4 [8@4]\n\n# Delivering the appends to n1 and n2 should reject them. It also cancels the\n# in-flight write requests on n1.\ndeliver 1 2\n---\nn1@1 leader ⇨ n1@4 follower(n5)\nn1@1 → c1 ClientResponse id=0x04 Error::Abort\nc1@1 put a=2 ⇒ Error::Abort (operation aborted)\nn1@1 → c1 ClientResponse id=0x05 Error::Abort\nc1@1 put a=3 ⇒ Error::Abort (operation aborted)\nn1@1 → c1 ClientResponse id=0x06 Error::Abort\nc1@1 put a=4 ⇒ Error::Abort (operation aborted)\nn1@1 → c1 ClientResponse id=0x07 Error::Abort\nc1@1 put a=5 ⇒ Error::Abort (operation aborted)\nn1@1 → c1 ClientResponse id=0x08 Error::Abort\nc1@1 put a=6 ⇒ Error::Abort (operation aborted)\nn1@1 → c1 ClientResponse id=0x09 Error::Abort\nc1@1 put a=7 ⇒ Error::Abort (operation aborted)\nn1@4 → n5 AppendResponse reject_index=7\nn2@1 follower(n1) ⇨ n2@4 follower(n5)\nn2@4 → n5 AppendResponse reject_index=7\n\n# n5 will probe the previous base, which is again rejected. This repeats until\n# a common base is found at 1@1.\ndeliver 5\nstatus 5\ndeliver 1 2\n---\nn5@4 → n1 Append base=6@4 []\nn5@4 → n2 Append base=6@4 []\nn5@4 leader last=8@4 commit=7@4 applied=7 progress={1:0→7 2:0→7 3:7→9 4:7→9}\nn1@4 → n5 AppendResponse reject_index=6\nn2@4 → n5 AppendResponse reject_index=6\n\ndeliver 5\ndeliver 1 2\nstatus 5\n---\nn5@4 → n1 Append base=5@3 []\nn5@4 → n2 Append base=5@3 []\nn1@4 → n5 AppendResponse reject_index=5\nn2@4 → n5 AppendResponse reject_index=5\nn5@4 leader last=8@4 commit=7@4 applied=7 progress={1:0→6 2:0→6 3:7→9 4:7→9}\n\ndeliver 5\nstatus 5\ndeliver 1 2\n---\nn5@4 → n1 Append base=4@3 []\nn5@4 → n2 Append base=4@3 []\nn5@4 leader last=8@4 commit=7@4 applied=7 progress={1:0→5 2:0→5 3:7→9 4:7→9}\nn1@4 → n5 AppendResponse reject_index=4\nn2@4 → n5 AppendResponse reject_index=4\n\ndeliver 5\nstatus 5\ndeliver 1 2\n---\nn5@4 → n1 Append base=3@2 []\nn5@4 → n2 Append base=3@2 []\nn5@4 leader last=8@4 commit=7@4 applied=7 progress={1:0→4 2:0→4 3:7→9 4:7→9}\nn1@4 → n5 AppendResponse reject_index=3\nn2@4 → n5 AppendResponse reject_index=3\n\ndeliver 5\nstatus 5\ndeliver 1 2\n---\nn5@4 → n1 Append base=2@2 []\nn5@4 → n2 Append base=2@2 []\nn5@4 leader last=8@4 commit=7@4 applied=7 progress={1:0→3 2:0→3 3:7→9 4:7→9}\nn1@4 → n5 AppendResponse reject_index=2\nn2@4 → n5 AppendResponse reject_index=2\n\ndeliver 5\nstatus 5\ndeliver 1 2\n---\nn5@4 → n1 Append base=1@1 []\nn5@4 → n2 Append base=1@1 []\nn5@4 leader last=8@4 commit=7@4 applied=7 progress={1:0→2 2:0→2 3:7→9 4:7→9}\nn1@4 → n5 AppendResponse match_index=1\nn2@4 → n5 AppendResponse match_index=1\n\n# n5 can now replicate the tail to n1 and n2, allowing n5 to commit it.\ndeliver 5\nstatus 5\ndeliver 1 2\n---\nn5@4 → n1 Append base=1@1 [2@2 3@2 4@3 5@3 6@4 7@4 8@4]\nn5@4 → n2 Append base=1@1 [2@2 3@2 4@3 5@3 6@4 7@4 8@4]\nn5@4 leader last=8@4 commit=7@4 applied=7 progress={1:1→9 2:1→9 3:7→9 4:7→9}\nn1@4 append 2@2 None\nn1@4 append 3@2 put a=1\nn1@4 append 4@3 None\nn1@4 append 5@3 put b=2\nn1@4 append 6@4 None\nn1@4 append 7@4 put c=3\nn1@4 append 8@4 put d=4\nn1@4 → n5 AppendResponse match_index=8\nn2@4 append 2@2 None\nn2@4 append 3@2 put a=1\nn2@4 append 4@3 None\nn2@4 append 5@3 put b=2\nn2@4 append 6@4 None\nn2@4 append 7@4 put c=3\nn2@4 append 8@4 put d=4\nn2@4 → n5 AppendResponse match_index=8\n\ndeliver 5\n---\nn5@4 commit 8@4\nn5@4 apply 8@4 put d=4\nn5@4 → c5 ClientResponse id=0x0a write 0x0108\nc5@4 put d=4 ⇒ 8\n\nstatus\n---\nn1@4 follower(n5) last=8@4 commit=1@1 applied=1\nn2@4 follower(n5) last=8@4 commit=1@1 applied=1\nn3@4 follower(n5) last=7@4 commit=7@4 applied=7\nn4@4 follower(n5) last=7@4 commit=7@4 applied=7\nn5@4 leader last=8@4 commit=8@4 applied=8 progress={1:8→9 2:8→9 3:7→9 4:7→9}\n\n# Stabilize the cluster.\n(stabilize heartbeat=true)\nstatus\n---\nn1@4 follower(n5) last=8@4 commit=8@4 applied=8\nn2@4 follower(n5) last=8@4 commit=8@4 applied=8\nn3@4 follower(n5) last=8@4 commit=8@4 applied=8\nn4@4 follower(n5) last=8@4 commit=8@4 applied=8\nn5@4 leader last=8@4 commit=8@4 applied=8 progress={1:8→9 2:8→9 3:8→9 4:8→9}\n" }, { "path": "src/raft/testscripts/node/append_probe_divergent_long", "content": "# Appends to a previous leader and follower with a long divergent tail requires\n# the leader to repeatedly probe until it finds a common base.\n\ncluster nodes=5 leader=1\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2 4:1→2 5:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\nn4@1 follower(n1) last=1@1 commit=1@1 applied=1\nn5@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Make a couple of writes to ensure a common log prefix.\n(put 1 a=1)\n(put 1 b=2)\n(stabilize)\nstatus\n---\nn1@1 leader last=3@1 commit=3@1 applied=3 progress={2:3→4 3:3→4 4:3→4 5:3→4}\nn2@1 follower(n1) last=3@1 commit=1@1 applied=1\nn3@1 follower(n1) last=3@1 commit=1@1 applied=1\nn4@1 follower(n1) last=3@1 commit=1@1 applied=1\nn5@1 follower(n1) last=3@1 commit=1@1 applied=1\n\n# Partition n1-n2\npartition 1 2\n---\nn1 n2 ⇹ n3 n4 n5\n\n# Elect new leaders in the majority partition and replicate a few writes.\n# Multiple leaders ensures the log has multiple terms.\n(campaign 3)\n(stabilize)\n(put 3 c=3)\n(stabilize heartbeat=true)\nstatus\n---\nn1@1 leader last=3@1 commit=3@1 applied=3 progress={2:3→4 3:3→4 4:3→4 5:3→4}\nn2@1 follower(n1) last=3@1 commit=1@1 applied=1\nn3@2 leader last=5@2 commit=5@2 applied=5 progress={1:0→6 2:0→6 4:5→6 5:5→6}\nn4@2 follower(n3) last=5@2 commit=5@2 applied=5\nn5@2 follower(n3) last=5@2 commit=5@2 applied=5\n\n(campaign 4)\n(stabilize)\n(put 4 d=4)\n(stabilize heartbeat=true)\nstatus\n---\nn1@1 leader last=3@1 commit=3@1 applied=3 progress={2:3→4 3:3→4 4:3→4 5:3→4}\nn2@1 follower(n1) last=3@1 commit=1@1 applied=1\nn3@3 follower(n4) last=7@3 commit=7@3 applied=7\nn4@3 leader last=7@3 commit=7@3 applied=7 progress={1:0→8 2:0→8 3:7→8 5:7→8}\nn5@3 follower(n4) last=7@3 commit=7@3 applied=7\n\n(campaign 5)\n(stabilize)\n(put 5 e=5)\n(stabilize heartbeat=true)\nstatus\n---\nn1@1 leader last=3@1 commit=3@1 applied=3 progress={2:3→4 3:3→4 4:3→4 5:3→4}\nn2@1 follower(n1) last=3@1 commit=1@1 applied=1\nn3@4 follower(n5) last=9@4 commit=9@4 applied=9\nn4@4 follower(n5) last=9@4 commit=9@4 applied=9\nn5@4 leader last=9@4 commit=9@4 applied=9 progress={1:0→10 2:0→10 3:9→10 4:9→10}\n\n# Propose writes in the minority partition as well, to build up a log\n# longer than the majority log.\n(put 1 a=2)\n(put 1 a=3)\n(put 1 a=4)\n(put 1 a=5)\n(put 1 a=6)\n(put 1 a=7)\n(put 1 a=8)\n(put 1 a=9)\n(put 1 a=10)\n(stabilize)\nstatus\n---\nn1@1 leader last=12@1 commit=3@1 applied=3 progress={2:12→13 3:3→13 4:3→13 5:3→13}\nn2@1 follower(n1) last=12@1 commit=1@1 applied=1\nn3@4 follower(n5) last=9@4 commit=9@4 applied=9\nn4@4 follower(n5) last=9@4 commit=9@4 applied=9\nn5@4 leader last=9@4 commit=9@4 applied=9 progress={1:0→10 2:0→10 3:9→10 4:9→10}\n\nlog 1 5\n---\nn1@1 term=1 last=12@1 commit=3@1 vote=Some(1)\nn1@1 entry 1@1 None\nn1@1 entry 2@1 put a=1\nn1@1 entry 3@1 put b=2\nn1@1 entry 4@1 put a=2\nn1@1 entry 5@1 put a=3\nn1@1 entry 6@1 put a=4\nn1@1 entry 7@1 put a=5\nn1@1 entry 8@1 put a=6\nn1@1 entry 9@1 put a=7\nn1@1 entry 10@1 put a=8\nn1@1 entry 11@1 put a=9\nn1@1 entry 12@1 put a=10\nn5@4 term=4 last=9@4 commit=9@4 vote=Some(5)\nn5@4 entry 1@1 None\nn5@4 entry 2@1 put a=1\nn5@4 entry 3@1 put b=2\nn5@4 entry 4@2 None\nn5@4 entry 5@2 put c=3\nn5@4 entry 6@3 None\nn5@4 entry 7@3 put d=4\nn5@4 entry 8@4 None\nn5@4 entry 9@4 put e=5\n\n# Heal the partition.\nheal\n---\nn1 n2 n3 n4 n5 fully connected\n\n# Propose another write on the majority leader.\nput 5 f=6\n---\nc5@4 → n5 ClientRequest id=0x0f write 0x0101660136\nn5@4 append 10@4 put f=6\nn5@4 → n1 Append base=9@4 [10@4]\nn5@4 → n2 Append base=9@4 [10@4]\nn5@4 → n3 Append base=9@4 [10@4]\nn5@4 → n4 Append base=9@4 [10@4]\n\n# Delivering the appends to n1 and n2 should reject them. It also cancels the\n# in-flight write requests on n1.\ndeliver 1 2\n---\nn1@1 leader ⇨ n1@4 follower(n5)\nn1@1 → c1 ClientResponse id=0x06 Error::Abort\nc1@1 put a=2 ⇒ Error::Abort (operation aborted)\nn1@1 → c1 ClientResponse id=0x07 Error::Abort\nc1@1 put a=3 ⇒ Error::Abort (operation aborted)\nn1@1 → c1 ClientResponse id=0x08 Error::Abort\nc1@1 put a=4 ⇒ Error::Abort (operation aborted)\nn1@1 → c1 ClientResponse id=0x09 Error::Abort\nc1@1 put a=5 ⇒ Error::Abort (operation aborted)\nn1@1 → c1 ClientResponse id=0x0a Error::Abort\nc1@1 put a=6 ⇒ Error::Abort (operation aborted)\nn1@1 → c1 ClientResponse id=0x0b Error::Abort\nc1@1 put a=7 ⇒ Error::Abort (operation aborted)\nn1@1 → c1 ClientResponse id=0x0c Error::Abort\nc1@1 put a=8 ⇒ Error::Abort (operation aborted)\nn1@1 → c1 ClientResponse id=0x0d Error::Abort\nc1@1 put a=9 ⇒ Error::Abort (operation aborted)\nn1@1 → c1 ClientResponse id=0x0e Error::Abort\nc1@1 put a=10 ⇒ Error::Abort (operation aborted)\nn1@4 → n5 AppendResponse reject_index=9\nn2@1 follower(n1) ⇨ n2@4 follower(n5)\nn2@4 → n5 AppendResponse reject_index=9\n\n# n5 will probe the previous base, which is again rejected. This repeats until\n# a common base is found at 3@1.\ndeliver 5\nstatus 5\ndeliver 1 2\n---\nn5@4 → n1 Append base=8@4 []\nn5@4 → n2 Append base=8@4 []\nn5@4 leader last=10@4 commit=9@4 applied=9 progress={1:0→9 2:0→9 3:9→11 4:9→11}\nn1@4 → n5 AppendResponse reject_index=8\nn2@4 → n5 AppendResponse reject_index=8\n\ndeliver 5\ndeliver 1 2\nstatus 5\n---\nn5@4 → n1 Append base=7@3 []\nn5@4 → n2 Append base=7@3 []\nn1@4 → n5 AppendResponse reject_index=7\nn2@4 → n5 AppendResponse reject_index=7\nn5@4 leader last=10@4 commit=9@4 applied=9 progress={1:0→8 2:0→8 3:9→11 4:9→11}\n\ndeliver 5\nstatus 5\ndeliver 1 2\n---\nn5@4 → n1 Append base=6@3 []\nn5@4 → n2 Append base=6@3 []\nn5@4 leader last=10@4 commit=9@4 applied=9 progress={1:0→7 2:0→7 3:9→11 4:9→11}\nn1@4 → n5 AppendResponse reject_index=6\nn2@4 → n5 AppendResponse reject_index=6\n\ndeliver 5\nstatus 5\ndeliver 1 2\n---\nn5@4 → n1 Append base=5@2 []\nn5@4 → n2 Append base=5@2 []\nn5@4 leader last=10@4 commit=9@4 applied=9 progress={1:0→6 2:0→6 3:9→11 4:9→11}\nn1@4 → n5 AppendResponse reject_index=5\nn2@4 → n5 AppendResponse reject_index=5\n\ndeliver 5\nstatus 5\ndeliver 1 2\n---\nn5@4 → n1 Append base=4@2 []\nn5@4 → n2 Append base=4@2 []\nn5@4 leader last=10@4 commit=9@4 applied=9 progress={1:0→5 2:0→5 3:9→11 4:9→11}\nn1@4 → n5 AppendResponse reject_index=4\nn2@4 → n5 AppendResponse reject_index=4\n\ndeliver 5\nstatus 5\ndeliver 1 2\n---\nn5@4 → n1 Append base=3@1 []\nn5@4 → n2 Append base=3@1 []\nn5@4 leader last=10@4 commit=9@4 applied=9 progress={1:0→4 2:0→4 3:9→11 4:9→11}\nn1@4 → n5 AppendResponse match_index=3\nn2@4 → n5 AppendResponse match_index=3\n\n# n5 can now replicate the tail to n1 and n2, allowing n5 to commit it.\ndeliver 5\nstatus 5\ndeliver 1 2\n---\nn5@4 → n1 Append base=3@1 [4@2 5@2 6@3 7@3 8@4 9@4 10@4]\nn5@4 → n2 Append base=3@1 [4@2 5@2 6@3 7@3 8@4 9@4 10@4]\nn5@4 leader last=10@4 commit=9@4 applied=9 progress={1:3→11 2:3→11 3:9→11 4:9→11}\nn1@4 append 4@2 None\nn1@4 append 5@2 put c=3\nn1@4 append 6@3 None\nn1@4 append 7@3 put d=4\nn1@4 append 8@4 None\nn1@4 append 9@4 put e=5\nn1@4 append 10@4 put f=6\nn1@4 → n5 AppendResponse match_index=10\nn2@4 append 4@2 None\nn2@4 append 5@2 put c=3\nn2@4 append 6@3 None\nn2@4 append 7@3 put d=4\nn2@4 append 8@4 None\nn2@4 append 9@4 put e=5\nn2@4 append 10@4 put f=6\nn2@4 → n5 AppendResponse match_index=10\n\ndeliver 5\n---\nn5@4 commit 10@4\nn5@4 apply 10@4 put f=6\nn5@4 → c5 ClientResponse id=0x0f write 0x010a\nc5@4 put f=6 ⇒ 10\n\nstatus\n---\nn1@4 follower(n5) last=10@4 commit=3@1 applied=3\nn2@4 follower(n5) last=10@4 commit=1@1 applied=1\nn3@4 follower(n5) last=9@4 commit=9@4 applied=9\nn4@4 follower(n5) last=9@4 commit=9@4 applied=9\nn5@4 leader last=10@4 commit=10@4 applied=10 progress={1:10→11 2:10→11 3:9→11 4:9→11}\n\n# Stabilize the cluster.\n(stabilize heartbeat=true)\nstatus\n---\nn1@4 follower(n5) last=10@4 commit=10@4 applied=10\nn2@4 follower(n5) last=10@4 commit=10@4 applied=10\nn3@4 follower(n5) last=10@4 commit=10@4 applied=10\nn4@4 follower(n5) last=10@4 commit=10@4 applied=10\nn5@4 leader last=10@4 commit=10@4 applied=10 progress={1:10→11 2:10→11 3:10→11 4:10→11}\n" }, { "path": "src/raft/testscripts/node/append_probe_divergent_short", "content": "# Appends to a previous leader and follower with a shorter divergent tail skips\n# the missing entries before probing.\n\ncluster nodes=5 leader=1\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2 4:1→2 5:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\nn4@1 follower(n1) last=1@1 commit=1@1 applied=1\nn5@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Make a couple of writes to ensure a common log prefix.\n(put 1 a=1)\n(put 1 b=2)\n(stabilize)\nstatus\n---\nn1@1 leader last=3@1 commit=3@1 applied=3 progress={2:3→4 3:3→4 4:3→4 5:3→4}\nn2@1 follower(n1) last=3@1 commit=1@1 applied=1\nn3@1 follower(n1) last=3@1 commit=1@1 applied=1\nn4@1 follower(n1) last=3@1 commit=1@1 applied=1\nn5@1 follower(n1) last=3@1 commit=1@1 applied=1\n\n# Partition n1-n2\npartition 1 2\n---\nn1 n2 ⇹ n3 n4 n5\n\n# Elect new leaders in the majority partition and replicate a few writes.\n# Multiple leaders ensures the log has multiple terms.\n(campaign 3)\n(stabilize)\n(put 3 c=3)\n(stabilize heartbeat=true)\nstatus\n---\nn1@1 leader last=3@1 commit=3@1 applied=3 progress={2:3→4 3:3→4 4:3→4 5:3→4}\nn2@1 follower(n1) last=3@1 commit=1@1 applied=1\nn3@2 leader last=5@2 commit=5@2 applied=5 progress={1:0→6 2:0→6 4:5→6 5:5→6}\nn4@2 follower(n3) last=5@2 commit=5@2 applied=5\nn5@2 follower(n3) last=5@2 commit=5@2 applied=5\n\n(campaign 4)\n(stabilize)\n(put 4 d=4)\n(stabilize heartbeat=true)\nstatus\n---\nn1@1 leader last=3@1 commit=3@1 applied=3 progress={2:3→4 3:3→4 4:3→4 5:3→4}\nn2@1 follower(n1) last=3@1 commit=1@1 applied=1\nn3@3 follower(n4) last=7@3 commit=7@3 applied=7\nn4@3 leader last=7@3 commit=7@3 applied=7 progress={1:0→8 2:0→8 3:7→8 5:7→8}\nn5@3 follower(n4) last=7@3 commit=7@3 applied=7\n\n(campaign 5)\n(stabilize)\n(put 5 e=5)\n(stabilize heartbeat=true)\nstatus\n---\nn1@1 leader last=3@1 commit=3@1 applied=3 progress={2:3→4 3:3→4 4:3→4 5:3→4}\nn2@1 follower(n1) last=3@1 commit=1@1 applied=1\nn3@4 follower(n5) last=9@4 commit=9@4 applied=9\nn4@4 follower(n5) last=9@4 commit=9@4 applied=9\nn5@4 leader last=9@4 commit=9@4 applied=9 progress={1:0→10 2:0→10 3:9→10 4:9→10}\n\n# Propose a single write in the minority partition. The divergent minority log\n# is much shorter than the majority log.\n(put 1 a=2)\n(stabilize)\nstatus\n---\nn1@1 leader last=4@1 commit=3@1 applied=3 progress={2:4→5 3:3→5 4:3→5 5:3→5}\nn2@1 follower(n1) last=4@1 commit=1@1 applied=1\nn3@4 follower(n5) last=9@4 commit=9@4 applied=9\nn4@4 follower(n5) last=9@4 commit=9@4 applied=9\nn5@4 leader last=9@4 commit=9@4 applied=9 progress={1:0→10 2:0→10 3:9→10 4:9→10}\n\nlog 1 5\n---\nn1@1 term=1 last=4@1 commit=3@1 vote=Some(1)\nn1@1 entry 1@1 None\nn1@1 entry 2@1 put a=1\nn1@1 entry 3@1 put b=2\nn1@1 entry 4@1 put a=2\nn5@4 term=4 last=9@4 commit=9@4 vote=Some(5)\nn5@4 entry 1@1 None\nn5@4 entry 2@1 put a=1\nn5@4 entry 3@1 put b=2\nn5@4 entry 4@2 None\nn5@4 entry 5@2 put c=3\nn5@4 entry 6@3 None\nn5@4 entry 7@3 put d=4\nn5@4 entry 8@4 None\nn5@4 entry 9@4 put e=5\n\n# Heal the partition.\nheal\n---\nn1 n2 n3 n4 n5 fully connected\n\n# Propose another write on the majority leader.\nput 5 f=6\n---\nc5@4 → n5 ClientRequest id=0x07 write 0x0101660136\nn5@4 append 10@4 put f=6\nn5@4 → n1 Append base=9@4 [10@4]\nn5@4 → n2 Append base=9@4 [10@4]\nn5@4 → n3 Append base=9@4 [10@4]\nn5@4 → n4 Append base=9@4 [10@4]\n\n# Delivering the appends to n1 and n2 should reject them, but with a\n# reject_index=5 after their last index instead of the original base 9. It also\n# cancels the in-flight write requests on n1.\ndeliver 1 2\n---\nn1@1 leader ⇨ n1@4 follower(n5)\nn1@1 → c1 ClientResponse id=0x06 Error::Abort\nc1@1 put a=2 ⇒ Error::Abort (operation aborted)\nn1@4 → n5 AppendResponse reject_index=5\nn2@1 follower(n1) ⇨ n2@4 follower(n5)\nn2@4 → n5 AppendResponse reject_index=5\n\n# n5 will probe the previous base, which is again rejected. This repeats until\n# a common base is found at 3@1.\ndeliver 5\nstatus 5\ndeliver 1 2\n---\nn5@4 → n1 Append base=4@2 []\nn5@4 → n2 Append base=4@2 []\nn5@4 leader last=10@4 commit=9@4 applied=9 progress={1:0→5 2:0→5 3:9→11 4:9→11}\nn1@4 → n5 AppendResponse reject_index=4\nn2@4 → n5 AppendResponse reject_index=4\n\ndeliver 5\ndeliver 1 2\nstatus 5\n---\nn5@4 → n1 Append base=3@1 []\nn5@4 → n2 Append base=3@1 []\nn1@4 → n5 AppendResponse match_index=3\nn2@4 → n5 AppendResponse match_index=3\nn5@4 leader last=10@4 commit=9@4 applied=9 progress={1:0→4 2:0→4 3:9→11 4:9→11}\n\n# n5 can now replicate the tail to n1 and n2, allowing n5 to commit it.\ndeliver 5\ndeliver 1 2\n---\nn5@4 → n1 Append base=3@1 [4@2 5@2 6@3 7@3 8@4 9@4 10@4]\nn5@4 → n2 Append base=3@1 [4@2 5@2 6@3 7@3 8@4 9@4 10@4]\nn1@4 append 4@2 None\nn1@4 append 5@2 put c=3\nn1@4 append 6@3 None\nn1@4 append 7@3 put d=4\nn1@4 append 8@4 None\nn1@4 append 9@4 put e=5\nn1@4 append 10@4 put f=6\nn1@4 → n5 AppendResponse match_index=10\nn2@4 append 4@2 None\nn2@4 append 5@2 put c=3\nn2@4 append 6@3 None\nn2@4 append 7@3 put d=4\nn2@4 append 8@4 None\nn2@4 append 9@4 put e=5\nn2@4 append 10@4 put f=6\nn2@4 → n5 AppendResponse match_index=10\n\ndeliver 5\n---\nn5@4 commit 10@4\nn5@4 apply 10@4 put f=6\nn5@4 → c5 ClientResponse id=0x07 write 0x010a\nc5@4 put f=6 ⇒ 10\n\nstatus\n---\nn1@4 follower(n5) last=10@4 commit=3@1 applied=3\nn2@4 follower(n5) last=10@4 commit=1@1 applied=1\nn3@4 follower(n5) last=9@4 commit=9@4 applied=9\nn4@4 follower(n5) last=9@4 commit=9@4 applied=9\nn5@4 leader last=10@4 commit=10@4 applied=10 progress={1:10→11 2:10→11 3:9→11 4:9→11}\n\n# Stabilize the cluster.\n(stabilize heartbeat=true)\nstatus\n---\nn1@4 follower(n5) last=10@4 commit=10@4 applied=10\nn2@4 follower(n5) last=10@4 commit=10@4 applied=10\nn3@4 follower(n5) last=10@4 commit=10@4 applied=10\nn4@4 follower(n5) last=10@4 commit=10@4 applied=10\nn5@4 leader last=10@4 commit=10@4 applied=10 progress={1:10→11 2:10→11 3:10→11 4:10→11}\n" }, { "path": "src/raft/testscripts/node/append_probe_divergent_single", "content": "# An append replaces a conflict at the tail for a single term.\n\ncluster nodes=5 leader=1\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2 4:1→2 5:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\nn4@1 follower(n1) last=1@1 commit=1@1 applied=1\nn5@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Partition n3-n5.\npartition 3 4 5\n---\nn1 n2 ⇹ n3 n4 n5\n\n# Propose and replicate a write in the minority partition.\nput 1 a=1\nstabilize\n---\nc1@1 → n1 ClientRequest id=0x01 write 0x0101610131\nn1@1 append 2@1 put a=1\nn1@1 → n2 Append base=1@1 [2@1]\nn1@1 ⇥ n3 A̶p̶p̶e̶n̶d̶ ̶b̶a̶s̶e̶=̶1̶@̶1̶ ̶[̶2̶@̶1̶]̶\nn1@1 ⇥ n4 A̶p̶p̶e̶n̶d̶ ̶b̶a̶s̶e̶=̶1̶@̶1̶ ̶[̶2̶@̶1̶]̶\nn1@1 ⇥ n5 A̶p̶p̶e̶n̶d̶ ̶b̶a̶s̶e̶=̶1̶@̶1̶ ̶[̶2̶@̶1̶]̶\nn2@1 append 2@1 put a=1\nn2@1 → n1 AppendResponse match_index=2\n\nlog 1 2\n---\nn1@1 term=1 last=2@1 commit=1@1 vote=Some(1)\nn1@1 entry 1@1 None\nn1@1 entry 2@1 put a=1\nn2@1 term=1 last=2@1 commit=1@1 vote=Some(1)\nn2@1 entry 1@1 None\nn2@1 entry 2@1 put a=1\n\n# Elect n5 as a new majority partition leader. It appends an empty entry.\n(campaign 5)\n(stabilize heartbeat=true)\nstatus\n---\nn1@1 leader last=2@1 commit=1@1 applied=1 progress={2:2→3 3:1→3 4:1→3 5:1→3}\nn2@1 follower(n1) last=2@1 commit=1@1 applied=1\nn3@2 follower(n5) last=2@2 commit=2@2 applied=2\nn4@2 follower(n5) last=2@2 commit=2@2 applied=2\nn5@2 leader last=2@2 commit=2@2 applied=2 progress={1:0→3 2:0→3 3:2→3 4:2→3}\n\n# Heal the partition and propose a new write.\nheal\nput 5 b=2\n---\nn1 n2 n3 n4 n5 fully connected\nc5@2 → n5 ClientRequest id=0x02 write 0x0101620132\nn5@2 append 3@2 put b=2\nn5@2 → n1 Append base=2@2 [3@2]\nn5@2 → n2 Append base=2@2 [3@2]\nn5@2 → n3 Append base=2@2 [3@2]\nn5@2 → n4 Append base=2@2 [3@2]\n\n# Delivering the append messages to n1,n2 will make them follow n5 and\n# reject the appends due to a log mismatch.\ndeliver 1 2\n---\nn1@1 leader ⇨ n1@2 follower(n5)\nn1@1 → c1 ClientResponse id=0x01 Error::Abort\nc1@1 put a=1 ⇒ Error::Abort (operation aborted)\nn1@2 → n5 AppendResponse reject_index=2\nn2@1 follower(n1) ⇨ n2@2 follower(n5)\nn2@2 → n5 AppendResponse reject_index=2\n\n# n5 probes index 1, which succeeds. 1 and 2 still has the old logs.\ndeliver 5\ndeliver 1 2\n---\nn5@2 → n1 Append base=1@1 []\nn5@2 → n2 Append base=1@1 []\nn1@2 → n5 AppendResponse match_index=1\nn2@2 → n5 AppendResponse match_index=1\n\nlog 1 2\n---\nn1@2 term=2 last=2@1 commit=1@1 vote=None\nn1@2 entry 1@1 None\nn1@2 entry 2@1 put a=1\nn2@2 term=2 last=2@1 commit=1@1 vote=None\nn2@2 entry 1@1 None\nn2@2 entry 2@1 put a=1\n\n# n5 now replicates the tail of its log, which replaces the old logs.\ndeliver 5\ndeliver 1 2\n---\nn5@2 → n1 Append base=1@1 [2@2 3@2]\nn5@2 → n2 Append base=1@1 [2@2 3@2]\nn1@2 append 2@2 None\nn1@2 append 3@2 put b=2\nn1@2 → n5 AppendResponse match_index=3\nn2@2 append 2@2 None\nn2@2 append 3@2 put b=2\nn2@2 → n5 AppendResponse match_index=3\n\nlog 1 2\n---\nn1@2 term=2 last=3@2 commit=1@1 vote=None\nn1@2 entry 1@1 None\nn1@2 entry 2@2 None\nn1@2 entry 3@2 put b=2\nn2@2 term=2 last=3@2 commit=1@1 vote=None\nn2@2 entry 1@1 None\nn2@2 entry 2@2 None\nn2@2 entry 3@2 put b=2\n\n# Stabilize the cluster.\n(stabilize heartbeat=true)\nstatus\n---\nn1@2 follower(n5) last=3@2 commit=3@2 applied=3\nn2@2 follower(n5) last=3@2 commit=3@2 applied=3\nn3@2 follower(n5) last=3@2 commit=3@2 applied=3\nn4@2 follower(n5) last=3@2 commit=3@2 applied=3\nn5@2 leader last=3@2 commit=3@2 applied=3 progress={1:3→4 2:3→4 3:3→4 4:3→4}\n" }, { "path": "src/raft/testscripts/node/append_response_beyond_last_index_panics", "content": "# A successful AppendResponse with last index beyond leader's last log\n# should panic.\n\ncluster nodes=3 leader=1\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Propose a write.\nput 1 foo=bar\n---\nc1@1 → n1 ClientRequest id=0x01 write 0x0103666f6f03626172\nn1@1 append 2@1 put foo=bar\nn1@1 → n2 Append base=1@1 [2@1]\nn1@1 → n3 Append base=1@1 [2@1]\n\n# An AppendResponse beyond leader's last log should panic.\n!step 1 '{\"from\":2, \"to\":1, \"term\":1, \"message\":{\"AppendResponse\":{\"match_index\":3,\"reject_index\":0}}}'\n---\nPanic: future match index\n" }, { "path": "src/raft/testscripts/node/append_response_stale_reject", "content": "# A successful AppendResponse with a reject_index below the match index\n# should be ignored.\n\ncluster nodes=3 leader=1\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Replicate a write.\n(put 1 a=1)\n(stabilize heartbeat=true)\nstatus\n---\nn1@1 leader last=2@1 commit=2@1 applied=2 progress={2:2→3 3:2→3}\nn2@1 follower(n1) last=2@1 commit=2@1 applied=2\nn3@1 follower(n1) last=2@1 commit=2@1 applied=2\n\n# Propose a few writes.\n(put 1 b=2)\n(put 1 c=3)\nstatus\n---\nn1@1 leader last=4@1 commit=2@1 applied=2 progress={2:2→5 3:2→5}\nn2@1 follower(n1) last=2@1 commit=2@1 applied=2\nn3@1 follower(n1) last=2@1 commit=2@1 applied=2\n\n# A reject_index below the follower's progress match index is ignored.\nstep 1 '{\"from\":2,\"to\":1,\"term\":1,\"message\":{\"AppendResponse\":{\"match_index\":0,\"reject_index\":2}}}'\nstatus\n---\nn1@1 leader last=4@1 commit=2@1 applied=2 progress={2:2→5 3:2→5}\nn2@1 follower(n1) last=2@1 commit=2@1 applied=2\nn3@1 follower(n1) last=2@1 commit=2@1 applied=2\n\nstep 1 '{\"from\":2,\"to\":1,\"term\":1,\"message\":{\"AppendResponse\":{\"match_index\":0,\"reject_index\":1}}}'\nstatus\n---\nn1@1 leader last=4@1 commit=2@1 applied=2 progress={2:2→5 3:2→5}\nn2@1 follower(n1) last=2@1 commit=2@1 applied=2\nn3@1 follower(n1) last=2@1 commit=2@1 applied=2\n\n# The writes are still replicated without any probes.\nstabilize\n---\nn2@1 append 3@1 put b=2\nn2@1 → n1 AppendResponse match_index=3\nn2@1 append 4@1 put c=3\nn2@1 → n1 AppendResponse match_index=4\nn3@1 append 3@1 put b=2\nn3@1 → n1 AppendResponse match_index=3\nn3@1 append 4@1 put c=3\nn3@1 → n1 AppendResponse match_index=4\nn1@1 commit 3@1\nn1@1 apply 3@1 put b=2\nn1@1 → c1 ClientResponse id=0x02 write 0x0103\nc1@1 put b=2 ⇒ 3\nn1@1 commit 4@1\nn1@1 apply 4@1 put c=3\nn1@1 → c1 ClientResponse id=0x03 write 0x0104\nc1@1 put c=3 ⇒ 4\n" }, { "path": "src/raft/testscripts/node/election", "content": "# A node campaigns and wins leadership once the election timeout passes. Uses\n# ticks directly to also test tick handling.\n\ncluster nodes=3 heartbeat_interval=1 election_timeout=2\n---\nn1@0 follower() last=0@0 commit=0@0 applied=0\nn2@0 follower() last=0@0 commit=0@0 applied=0\nn3@0 follower() last=0@0 commit=0@0 applied=0\n\n# Tick all nodes. Then tick n1 again to make it campaign.\ntick\n---\nok\n\ntick 1\n---\nn1@0 follower() ⇨ n1@1 candidate\nn1@1 → n2 Campaign last=0@0\nn1@1 → n3 Campaign last=0@0\n\n# n2,n3 grant n1 their votes.\ndeliver\n---\nn2@0 follower() ⇨ n2@1 follower()\nn2@1 → n1 CampaignResponse vote=true\nn3@0 follower() ⇨ n3@1 follower()\nn3@1 → n1 CampaignResponse vote=true\n\n# n1 wins the election and becomes leader.\ndeliver\n---\nn1@1 candidate ⇨ n1@1 leader\nn1@1 append 1@1 None\nn1@1 → n2 Append base=0@0 [1@1]\nn1@1 → n3 Append base=0@0 [1@1]\nn1@1 → n2 Heartbeat last_index=1 commit_index=0 read_seq=0\nn1@1 → n3 Heartbeat last_index=1 commit_index=0 read_seq=0\n\n# All nodes become n1 followers.\nstabilize\n---\nn2@1 follower() ⇨ n2@1 follower(n1)\nn2@1 append 1@1 None\nn2@1 → n1 AppendResponse match_index=1\nn2@1 → n1 HeartbeatResponse match_index=1 read_seq=0\nn3@1 follower() ⇨ n3@1 follower(n1)\nn3@1 append 1@1 None\nn3@1 → n1 AppendResponse match_index=1\nn3@1 → n1 HeartbeatResponse match_index=1 read_seq=0\nn1@1 commit 1@1\nn1@1 apply 1@1 None\n\n# n1's heartbeats are accepted by followers, who commit and apply the entry.\ntick 1\n---\nn1@1 → n2 Heartbeat last_index=1 commit_index=1 read_seq=0\nn1@1 → n3 Heartbeat last_index=1 commit_index=1 read_seq=0\n\nstabilize\n---\nn2@1 commit 1@1\nn2@1 apply 1@1 None\nn2@1 → n1 HeartbeatResponse match_index=1 read_seq=0\nn3@1 commit 1@1\nn3@1 apply 1@1 None\nn3@1 → n1 HeartbeatResponse match_index=1 read_seq=0\n\nstatus\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\n" }, { "path": "src/raft/testscripts/node/election_candidate_behind_leader", "content": "# A candidate that lags behind the leader can still win the election\n# as long as it isn't behind the quorum.\n\ncluster nodes=5 leader=1\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2 4:1→2 5:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\nn4@1 follower(n1) last=1@1 commit=1@1 applied=1\nn5@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Partition n1+n2 away from the cluster.\npartition 1 2\n---\nn1 n2 ⇹ n3 n4 n5\n\n# Replica a write on n1+n2. The write can't be committed, because n1 doesn't\n# have quorum.\n(put 1 foo=bar)\n(stabilize)\nstatus\n---\nn1@1 leader last=2@1 commit=1@1 applied=1 progress={2:2→3 3:1→3 4:1→3 5:1→3}\nn2@1 follower(n1) last=2@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\nn4@1 follower(n1) last=1@1 commit=1@1 applied=1\nn5@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# List the logs on n1 n2 n3 to show the replicated but uncommitted entry.\nlog 1 2 3\n---\nn1@1 term=1 last=2@1 commit=1@1 vote=Some(1)\nn1@1 entry 1@1 None\nn1@1 entry 2@1 put foo=bar\nn2@1 term=1 last=2@1 commit=1@1 vote=Some(1)\nn2@1 entry 1@1 None\nn2@1 entry 2@1 put foo=bar\nn3@1 term=1 last=1@1 commit=1@1 vote=Some(1)\nn3@1 entry 1@1 None\n\n# Heal the partition.\nheal\n---\nn1 n2 n3 n4 n5 fully connected\n\n# Make n5 campaign. n3+n4 grant their votes, n1+n2 reject it. n1 aborts the\n# in-flight write request because the term changes.\ncampaign 5\ndeliver\n---\nn5@1 follower(n1) ⇨ n5@2 candidate\nn5@2 → n1 Campaign last=1@1\nn5@2 → n2 Campaign last=1@1\nn5@2 → n3 Campaign last=1@1\nn5@2 → n4 Campaign last=1@1\nn1@1 leader ⇨ n1@2 follower()\nn1@1 → c1 ClientResponse id=0x01 Error::Abort\nc1@1 put foo=bar ⇒ Error::Abort (operation aborted)\nn1@2 → n5 CampaignResponse vote=false\nn2@1 follower(n1) ⇨ n2@2 follower()\nn2@2 → n5 CampaignResponse vote=false\nn3@1 follower(n1) ⇨ n3@2 follower()\nn3@2 → n5 CampaignResponse vote=true\nn4@1 follower(n1) ⇨ n4@2 follower()\nn4@2 → n5 CampaignResponse vote=true\n\n# n5 wins the election and becomes leader.\nstabilize heartbeat=true\n---\nn5@2 candidate ⇨ n5@2 leader\nn5@2 append 2@2 None\nn5@2 → n1 Append base=1@1 [2@2]\nn5@2 → n2 Append base=1@1 [2@2]\nn5@2 → n3 Append base=1@1 [2@2]\nn5@2 → n4 Append base=1@1 [2@2]\nn5@2 → n1 Heartbeat last_index=2 commit_index=1 read_seq=0\nn5@2 → n2 Heartbeat last_index=2 commit_index=1 read_seq=0\nn5@2 → n3 Heartbeat last_index=2 commit_index=1 read_seq=0\nn5@2 → n4 Heartbeat last_index=2 commit_index=1 read_seq=0\nn1@2 follower() ⇨ n1@2 follower(n5)\nn1@2 append 2@2 None\nn1@2 → n5 AppendResponse match_index=2\nn1@2 → n5 HeartbeatResponse match_index=2 read_seq=0\nn2@2 follower() ⇨ n2@2 follower(n5)\nn2@2 append 2@2 None\nn2@2 → n5 AppendResponse match_index=2\nn2@2 → n5 HeartbeatResponse match_index=2 read_seq=0\nn3@2 follower() ⇨ n3@2 follower(n5)\nn3@2 append 2@2 None\nn3@2 → n5 AppendResponse match_index=2\nn3@2 → n5 HeartbeatResponse match_index=2 read_seq=0\nn4@2 follower() ⇨ n4@2 follower(n5)\nn4@2 append 2@2 None\nn4@2 → n5 AppendResponse match_index=2\nn4@2 → n5 HeartbeatResponse match_index=2 read_seq=0\nn5@2 commit 2@2\nn5@2 apply 2@2 None\nn5@2 → n1 Heartbeat last_index=2 commit_index=2 read_seq=0\nn5@2 → n2 Heartbeat last_index=2 commit_index=2 read_seq=0\nn5@2 → n3 Heartbeat last_index=2 commit_index=2 read_seq=0\nn5@2 → n4 Heartbeat last_index=2 commit_index=2 read_seq=0\nn1@2 commit 2@2\nn1@2 apply 2@2 None\nn1@2 → n5 HeartbeatResponse match_index=2 read_seq=0\nn2@2 commit 2@2\nn2@2 apply 2@2 None\nn2@2 → n5 HeartbeatResponse match_index=2 read_seq=0\nn3@2 commit 2@2\nn3@2 apply 2@2 None\nn3@2 → n5 HeartbeatResponse match_index=2 read_seq=0\nn4@2 commit 2@2\nn4@2 apply 2@2 None\nn4@2 → n5 HeartbeatResponse match_index=2 read_seq=0\n\n# n1+n2's in-flight write at log position 2 has been replaced by the\n# empty log entry appended by n5 when it became leader.\nlog 1 2\n---\nn1@2 term=2 last=2@2 commit=2@2 vote=None\nn1@2 entry 1@1 None\nn1@2 entry 2@2 None\nn2@2 term=2 last=2@2 commit=2@2 vote=None\nn2@2 entry 1@1 None\nn2@2 entry 2@2 None\n\nstatus\n---\nn1@2 follower(n5) last=2@2 commit=2@2 applied=2\nn2@2 follower(n5) last=2@2 commit=2@2 applied=2\nn3@2 follower(n5) last=2@2 commit=2@2 applied=2\nn4@2 follower(n5) last=2@2 commit=2@2 applied=2\nn5@2 leader last=2@2 commit=2@2 applied=2 progress={1:2→3 2:2→3 3:2→3 4:2→3}\n" }, { "path": "src/raft/testscripts/node/election_candidate_behind_quorum", "content": "# A candidate that lags behind the quorum can't win an election.\n\ncluster nodes=5 leader=1\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2 4:1→2 5:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\nn4@1 follower(n1) last=1@1 commit=1@1 applied=1\nn5@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Partition n4+n5 away from the cluster.\npartition 4 5\n---\nn4 n5 ⇹ n1 n2 n3\n\n# Replicate a write on n1. n4+n5 now lag behind the quorum. Don't yet propagate\n# the commit index to n2+n3, to make sure it won't grant the vote just because\n# n5 is caught up to their local view of the commit index.\n(put 1 foo=bar)\n(stabilize)\nstatus\n---\nn1@1 leader last=2@1 commit=2@1 applied=2 progress={2:2→3 3:2→3 4:1→3 5:1→3}\nn2@1 follower(n1) last=2@1 commit=1@1 applied=1\nn3@1 follower(n1) last=2@1 commit=1@1 applied=1\nn4@1 follower(n1) last=1@1 commit=1@1 applied=1\nn5@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Heal the partition.\nheal\n---\nn1 n2 n3 n4 n5 fully connected\n\n# Make n5 campaign. n4 grants its vote, but the others reject it because it is\n# behind the quorum. However, the term bump will convert the other nodes to\n# leaderless followers.\nheal\ncampaign 5\nstabilize\n---\nn1 n2 n3 n4 n5 fully connected\nn5@1 follower(n1) ⇨ n5@2 candidate\nn5@2 → n1 Campaign last=1@1\nn5@2 → n2 Campaign last=1@1\nn5@2 → n3 Campaign last=1@1\nn5@2 → n4 Campaign last=1@1\nn1@1 leader ⇨ n1@2 follower()\nn1@2 → n5 CampaignResponse vote=false\nn2@1 follower(n1) ⇨ n2@2 follower()\nn2@2 → n5 CampaignResponse vote=false\nn3@1 follower(n1) ⇨ n3@2 follower()\nn3@2 → n5 CampaignResponse vote=false\nn4@1 follower(n1) ⇨ n4@2 follower()\nn4@2 → n5 CampaignResponse vote=true\n\nstatus\n---\nn1@2 follower() last=2@1 commit=2@1 applied=2\nn2@2 follower() last=2@1 commit=1@1 applied=1\nn3@2 follower() last=2@1 commit=1@1 applied=1\nn4@2 follower() last=1@1 commit=1@1 applied=1\nn5@2 candidate last=1@1 commit=1@1 applied=1\n\n# n2 can campaign and win the election.\n(campaign 2)\n(stabilize heartbeat=true)\nstatus\n---\nn1@3 follower(n2) last=3@3 commit=3@3 applied=3\nn2@3 leader last=3@3 commit=3@3 applied=3 progress={1:3→4 3:3→4 4:3→4 5:3→4}\nn3@3 follower(n2) last=3@3 commit=3@3 applied=3\nn4@3 follower(n2) last=3@3 commit=3@3 applied=3\nn5@3 follower(n2) last=3@3 commit=3@3 applied=3\n" }, { "path": "src/raft/testscripts/node/election_contested", "content": "# A leader can be elected even when there are multiple candidates.\n\ncluster nodes=5 election_timeout=2\n---\nn1@0 follower() last=0@0 commit=0@0 applied=0\nn2@0 follower() last=0@0 commit=0@0 applied=0\nn3@0 follower() last=0@0 commit=0@0 applied=0\nn4@0 follower() last=0@0 commit=0@0 applied=0\nn5@0 follower() last=0@0 commit=0@0 applied=0\n\n# n1 and n5 campaign.\ntick\ntick 1 5\n---\nn1@0 follower() ⇨ n1@1 candidate\nn1@1 → n2 Campaign last=0@0\nn1@1 → n3 Campaign last=0@0\nn1@1 → n4 Campaign last=0@0\nn1@1 → n5 Campaign last=0@0\nn5@0 follower() ⇨ n5@1 candidate\nn5@1 → n1 Campaign last=0@0\nn5@1 → n2 Campaign last=0@0\nn5@1 → n3 Campaign last=0@0\nn5@1 → n4 Campaign last=0@0\n\n# n1 and n5 ignore each other, since they're both campaigning.\ndeliver 1 5\n---\nn1@1 → n5 CampaignResponse vote=false\nn5@1 → n1 CampaignResponse vote=false\n\n# n1 reaches n2,n3 first, but n5 reaches n4 first.\ndeliver 2 3\ndeliver 4 from=5\ndeliver 4\n---\nn2@0 follower() ⇨ n2@1 follower()\nn2@1 → n1 CampaignResponse vote=true\nn2@1 → n5 CampaignResponse vote=false\nn3@0 follower() ⇨ n3@1 follower()\nn3@1 → n1 CampaignResponse vote=true\nn3@1 → n5 CampaignResponse vote=false\nn4@0 follower() ⇨ n4@1 follower()\nn4@1 → n5 CampaignResponse vote=true\nn4@1 → n1 CampaignResponse vote=false\n\n# n1 and n5 receive their votes. n1 has quorum and becomes leader.\ndeliver\n---\nn1@1 candidate ⇨ n1@1 leader\nn1@1 append 1@1 None\nn1@1 → n2 Append base=0@0 [1@1]\nn1@1 → n3 Append base=0@0 [1@1]\nn1@1 → n4 Append base=0@0 [1@1]\nn1@1 → n5 Append base=0@0 [1@1]\nn1@1 → n2 Heartbeat last_index=1 commit_index=0 read_seq=0\nn1@1 → n3 Heartbeat last_index=1 commit_index=0 read_seq=0\nn1@1 → n4 Heartbeat last_index=1 commit_index=0 read_seq=0\nn1@1 → n5 Heartbeat last_index=1 commit_index=0 read_seq=0\n\n# All nodes accept n1 as leader in term 1 and become followers.\nstabilize\n---\nn2@1 follower() ⇨ n2@1 follower(n1)\nn2@1 append 1@1 None\nn2@1 → n1 AppendResponse match_index=1\nn2@1 → n1 HeartbeatResponse match_index=1 read_seq=0\nn3@1 follower() ⇨ n3@1 follower(n1)\nn3@1 append 1@1 None\nn3@1 → n1 AppendResponse match_index=1\nn3@1 → n1 HeartbeatResponse match_index=1 read_seq=0\nn4@1 follower() ⇨ n4@1 follower(n1)\nn4@1 append 1@1 None\nn4@1 → n1 AppendResponse match_index=1\nn4@1 → n1 HeartbeatResponse match_index=1 read_seq=0\nn5@1 candidate ⇨ n5@1 follower(n1)\nn5@1 append 1@1 None\nn5@1 → n1 AppendResponse match_index=1\nn5@1 → n1 HeartbeatResponse match_index=1 read_seq=0\nn1@1 commit 1@1\nn1@1 apply 1@1 None\n\nstatus\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2 4:1→2 5:1→2}\nn2@1 follower(n1) last=1@1 commit=0@0 applied=0\nn3@1 follower(n1) last=1@1 commit=0@0 applied=0\nn4@1 follower(n1) last=1@1 commit=0@0 applied=0\nn5@1 follower(n1) last=1@1 commit=0@0 applied=0\n" }, { "path": "src/raft/testscripts/node/election_tie", "content": "# No leader can be elected with an election tie.\n\ncluster nodes=3 election_timeout=2\n---\nn1@0 follower() last=0@0 commit=0@0 applied=0\nn2@0 follower() last=0@0 commit=0@0 applied=0\nn3@0 follower() last=0@0 commit=0@0 applied=0\n\n# Tick all nodes twice to make them all campaign.\ntick\ntick\n---\nn1@0 follower() ⇨ n1@1 candidate\nn1@1 → n2 Campaign last=0@0\nn1@1 → n3 Campaign last=0@0\nn2@0 follower() ⇨ n2@1 candidate\nn2@1 → n1 Campaign last=0@0\nn2@1 → n3 Campaign last=0@0\nn3@0 follower() ⇨ n3@1 candidate\nn3@1 → n1 Campaign last=0@0\nn3@1 → n2 Campaign last=0@0\n\n# Stabilizing the cluster will not result in a leader.\nstabilize\n---\nn1@1 → n2 CampaignResponse vote=false\nn1@1 → n3 CampaignResponse vote=false\nn2@1 → n1 CampaignResponse vote=false\nn2@1 → n3 CampaignResponse vote=false\nn3@1 → n1 CampaignResponse vote=false\nn3@1 → n2 CampaignResponse vote=false\n\nstatus\n---\nn1@1 candidate last=0@0 commit=0@0 applied=0\nn2@1 candidate last=0@0 commit=0@0 applied=0\nn3@1 candidate last=0@0 commit=0@0 applied=0\n\n# A node can call another election in a new term and win.\ntick 2\ntick 2\n---\nn2@1 candidate ⇨ n2@2 candidate\nn2@2 → n1 Campaign last=0@0\nn2@2 → n3 Campaign last=0@0\n\ndeliver\n---\nn1@1 candidate ⇨ n1@2 follower()\nn1@2 → n2 CampaignResponse vote=true\nn3@1 candidate ⇨ n3@2 follower()\nn3@2 → n2 CampaignResponse vote=true\n\ndeliver\n---\nn2@2 candidate ⇨ n2@2 leader\nn2@2 append 1@2 None\nn2@2 → n1 Append base=0@0 [1@2]\nn2@2 → n3 Append base=0@0 [1@2]\nn2@2 → n1 Heartbeat last_index=1 commit_index=0 read_seq=0\nn2@2 → n3 Heartbeat last_index=1 commit_index=0 read_seq=0\n\nstabilize\n---\nn1@2 follower() ⇨ n1@2 follower(n2)\nn1@2 append 1@2 None\nn1@2 → n2 AppendResponse match_index=1\nn1@2 → n2 HeartbeatResponse match_index=1 read_seq=0\nn3@2 follower() ⇨ n3@2 follower(n2)\nn3@2 append 1@2 None\nn3@2 → n2 AppendResponse match_index=1\nn3@2 → n2 HeartbeatResponse match_index=1 read_seq=0\nn2@2 commit 1@2\nn2@2 apply 1@2 None\n\nstatus\n---\nn1@2 follower(n2) last=1@2 commit=0@0 applied=0\nn2@2 leader last=1@2 commit=1@2 applied=1 progress={1:1→2 3:1→2}\nn3@2 follower(n2) last=1@2 commit=0@0 applied=0\n" }, { "path": "src/raft/testscripts/node/election_tie_even", "content": "# No leader can be elected with an election tie between an even number of nodes.\n\ncluster nodes=4 election_timeout=2\n---\nn1@0 follower() last=0@0 commit=0@0 applied=0\nn2@0 follower() last=0@0 commit=0@0 applied=0\nn3@0 follower() last=0@0 commit=0@0 applied=0\nn4@0 follower() last=0@0 commit=0@0 applied=0\n\n# n1 and n4 campaign.\ntick\ntick 1 4\n---\nn1@0 follower() ⇨ n1@1 candidate\nn1@1 → n2 Campaign last=0@0\nn1@1 → n3 Campaign last=0@0\nn1@1 → n4 Campaign last=0@0\nn4@0 follower() ⇨ n4@1 candidate\nn4@1 → n1 Campaign last=0@0\nn4@1 → n2 Campaign last=0@0\nn4@1 → n3 Campaign last=0@0\n\n# n2 votes for n1, n3 votes for n4.\ndeliver 2\ndeliver 3 from=4\ndeliver 3\n---\nn2@0 follower() ⇨ n2@1 follower()\nn2@1 → n1 CampaignResponse vote=true\nn2@1 → n4 CampaignResponse vote=false\nn3@0 follower() ⇨ n3@1 follower()\nn3@1 → n4 CampaignResponse vote=true\nn3@1 → n1 CampaignResponse vote=false\n\n# Stabilizing the cluster will not result in a leader.\nstabilize\n---\nn1@1 → n4 CampaignResponse vote=false\nn4@1 → n1 CampaignResponse vote=false\n\nstatus\n---\nn1@1 candidate last=0@0 commit=0@0 applied=0\nn2@1 follower() last=0@0 commit=0@0 applied=0\nn3@1 follower() last=0@0 commit=0@0 applied=0\nn4@1 candidate last=0@0 commit=0@0 applied=0\n\n# A node can call another election in a new term and win.\ntick 3\ntick 3\n---\nn3@1 follower() ⇨ n3@2 candidate\nn3@2 → n1 Campaign last=0@0\nn3@2 → n2 Campaign last=0@0\nn3@2 → n4 Campaign last=0@0\n\ndeliver\n---\nn1@1 candidate ⇨ n1@2 follower()\nn1@2 → n3 CampaignResponse vote=true\nn2@1 follower() ⇨ n2@2 follower()\nn2@2 → n3 CampaignResponse vote=true\nn4@1 candidate ⇨ n4@2 follower()\nn4@2 → n3 CampaignResponse vote=true\n\ndeliver\n---\nn3@2 candidate ⇨ n3@2 leader\nn3@2 append 1@2 None\nn3@2 → n1 Append base=0@0 [1@2]\nn3@2 → n2 Append base=0@0 [1@2]\nn3@2 → n4 Append base=0@0 [1@2]\nn3@2 → n1 Heartbeat last_index=1 commit_index=0 read_seq=0\nn3@2 → n2 Heartbeat last_index=1 commit_index=0 read_seq=0\nn3@2 → n4 Heartbeat last_index=1 commit_index=0 read_seq=0\n\nstabilize\n---\nn1@2 follower() ⇨ n1@2 follower(n3)\nn1@2 append 1@2 None\nn1@2 → n3 AppendResponse match_index=1\nn1@2 → n3 HeartbeatResponse match_index=1 read_seq=0\nn2@2 follower() ⇨ n2@2 follower(n3)\nn2@2 append 1@2 None\nn2@2 → n3 AppendResponse match_index=1\nn2@2 → n3 HeartbeatResponse match_index=1 read_seq=0\nn4@2 follower() ⇨ n4@2 follower(n3)\nn4@2 append 1@2 None\nn4@2 → n3 AppendResponse match_index=1\nn4@2 → n3 HeartbeatResponse match_index=1 read_seq=0\nn3@2 commit 1@2\nn3@2 apply 1@2 None\n\nstatus\n---\nn1@2 follower(n3) last=1@2 commit=0@0 applied=0\nn2@2 follower(n3) last=1@2 commit=0@0 applied=0\nn3@2 leader last=1@2 commit=1@2 applied=1 progress={1:1→2 2:1→2 4:1→2}\nn4@2 follower(n3) last=1@2 commit=0@0 applied=0\n" }, { "path": "src/raft/testscripts/node/heartbeat_commits_follower", "content": "# A heartbeat will commit and apply an entry on a follower.\n\ncluster nodes=3 leader=1\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Write on the leader, which replicates then commits and applies locally.\nput 1 foo=bar\nstabilize\n---\nc1@1 → n1 ClientRequest id=0x01 write 0x0103666f6f03626172\nn1@1 append 2@1 put foo=bar\nn1@1 → n2 Append base=1@1 [2@1]\nn1@1 → n3 Append base=1@1 [2@1]\nn2@1 append 2@1 put foo=bar\nn2@1 → n1 AppendResponse match_index=2\nn3@1 append 2@1 put foo=bar\nn3@1 → n1 AppendResponse match_index=2\nn1@1 commit 2@1\nn1@1 apply 2@1 put foo=bar\nn1@1 → c1 ClientResponse id=0x01 write 0x0102\nc1@1 put foo=bar ⇒ 2\n\n# The write has been replicated, but not yet committed and applied on followers.\nstatus\n---\nn1@1 leader last=2@1 commit=2@1 applied=2 progress={2:2→3 3:2→3}\nn2@1 follower(n1) last=2@1 commit=1@1 applied=1\nn3@1 follower(n1) last=2@1 commit=1@1 applied=1\n\n# A heartbeat commits and applies on followers.\nheartbeat 1\nstabilize\n---\nn1@1 → n2 Heartbeat last_index=2 commit_index=2 read_seq=0\nn1@1 → n3 Heartbeat last_index=2 commit_index=2 read_seq=0\nn2@1 commit 2@1\nn2@1 apply 2@1 put foo=bar\nn2@1 → n1 HeartbeatResponse match_index=2 read_seq=0\nn3@1 commit 2@1\nn3@1 apply 2@1 put foo=bar\nn3@1 → n1 HeartbeatResponse match_index=2 read_seq=0\n\nstatus\n---\nn1@1 leader last=2@1 commit=2@1 applied=2 progress={2:2→3 3:2→3}\nn2@1 follower(n1) last=2@1 commit=2@1 applied=2\nn3@1 follower(n1) last=2@1 commit=2@1 applied=2\n" }, { "path": "src/raft/testscripts/node/heartbeat_converts_candidate", "content": "# A heartbeat from a leader should convert a candidate in the same term to a\n# follower.\n\ncluster nodes=3\n---\nn1@0 follower() last=0@0 commit=0@0 applied=0\nn2@0 follower() last=0@0 commit=0@0 applied=0\nn3@0 follower() last=0@0 commit=0@0 applied=0\n\n# Partition n3 away from the cluster.\npartition 3\n---\nn3 ⇹ n1 n2\n\n# Both n1 and n3 campaign. n2 votes for n1.\ncampaign 1 3\ndeliver\n---\nn1@0 follower() ⇨ n1@1 candidate\nn1@1 → n2 Campaign last=0@0\nn1@1 ⇥ n3 C̶a̶m̶p̶a̶i̶g̶n̶ ̶l̶a̶s̶t̶=̶0̶@̶0̶\nn3@0 follower() ⇨ n3@1 candidate\nn3@1 ⇥ n1 C̶a̶m̶p̶a̶i̶g̶n̶ ̶l̶a̶s̶t̶=̶0̶@̶0̶\nn3@1 ⇥ n2 C̶a̶m̶p̶a̶i̶g̶n̶ ̶l̶a̶s̶t̶=̶0̶@̶0̶\nn2@0 follower() ⇨ n2@1 follower()\nn2@1 → n1 CampaignResponse vote=true\n\n# n1 assumes leadership and heartbeats, committing entry 1.\n(stabilize heartbeat=true)\nstatus\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:0→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 candidate last=0@0 commit=0@0 applied=0\n\n# Heal the partition.\nheal\n---\nn1 n2 n3 fully connected\n\n# The next heartbeat from n1 converts n3 to a follower in term 1.\nheartbeat 1\nstabilize\n---\nn1@1 → n2 Heartbeat last_index=1 commit_index=1 read_seq=0\nn1@1 → n3 Heartbeat last_index=1 commit_index=1 read_seq=0\nn2@1 → n1 HeartbeatResponse match_index=1 read_seq=0\nn3@1 candidate ⇨ n3@1 follower(n1)\nn3@1 → n1 HeartbeatResponse match_index=0 read_seq=0\nn1@1 → n3 Append base=0@0 [1@1]\nn3@1 append 1@1 None\nn3@1 → n1 AppendResponse match_index=1\n\nstatus\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=0@0 applied=0\n" }, { "path": "src/raft/testscripts/node/heartbeat_converts_follower", "content": "# A heartbeat from a leader should convert a follower of a different leader in a\n# past term to a follower.\n\ncluster nodes=3 leader=1\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Partition n2 away from the cluster.\npartition 2\n---\nn2 ⇹ n1 n3\n\n# Elect n3 as a new leader.\n(campaign 3)\n(stabilize heartbeat=true)\nstatus\n---\nn1@2 follower(n3) last=2@2 commit=2@2 applied=2\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@2 leader last=2@2 commit=2@2 applied=2 progress={1:2→3 2:0→3}\n\n# Heal the partition.\nheal\n---\nn1 n2 n3 fully connected\n\n# The next heartbeat from n3 converts n2 to a follower in term 2.\nheartbeat 3\nstabilize heartbeat=true\n---\nn3@2 → n1 Heartbeat last_index=2 commit_index=2 read_seq=0\nn3@2 → n2 Heartbeat last_index=2 commit_index=2 read_seq=0\nn1@2 → n3 HeartbeatResponse match_index=2 read_seq=0\nn2@1 follower(n1) ⇨ n2@2 follower(n3)\nn2@2 → n3 HeartbeatResponse match_index=0 read_seq=0\nn3@2 → n2 Append base=1@1 []\nn2@2 → n3 AppendResponse match_index=1\nn3@2 → n2 Append base=1@1 [2@2]\nn2@2 append 2@2 None\nn2@2 → n3 AppendResponse match_index=2\nn3@2 → n1 Heartbeat last_index=2 commit_index=2 read_seq=0\nn3@2 → n2 Heartbeat last_index=2 commit_index=2 read_seq=0\nn1@2 → n3 HeartbeatResponse match_index=2 read_seq=0\nn2@2 commit 2@2\nn2@2 apply 2@2 None\nn2@2 → n3 HeartbeatResponse match_index=2 read_seq=0\n\nstatus\n---\nn1@2 follower(n3) last=2@2 commit=2@2 applied=2\nn2@2 follower(n3) last=2@2 commit=2@2 applied=2\nn3@2 leader last=2@2 commit=2@2 applied=2 progress={1:2→3 2:2→3}\n" }, { "path": "src/raft/testscripts/node/heartbeat_converts_follower_leaderless", "content": "# A heartbeat from a leader should convert a leaderless follower.\n\ncluster nodes=3\n---\nn1@0 follower() last=0@0 commit=0@0 applied=0\nn2@0 follower() last=0@0 commit=0@0 applied=0\nn3@0 follower() last=0@0 commit=0@0 applied=0\n\n# Partition n3 away from the cluster.\npartition 3\n---\nn3 ⇹ n1 n2\n\n# Elect n1 as a new leader.\n(campaign 1)\n(stabilize heartbeat=true)\nstatus\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:0→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@0 follower() last=0@0 commit=0@0 applied=0\n\n# Heal the partition.\nheal\n---\nn1 n2 n3 fully connected\n\n# The next heartbeat from n1 converts n3 to a follower in term 1.\nheartbeat 1\nstabilize\n---\nn1@1 → n2 Heartbeat last_index=1 commit_index=1 read_seq=0\nn1@1 → n3 Heartbeat last_index=1 commit_index=1 read_seq=0\nn2@1 → n1 HeartbeatResponse match_index=1 read_seq=0\nn3@0 follower() ⇨ n3@1 follower(n1)\nn3@1 → n1 HeartbeatResponse match_index=0 read_seq=0\nn1@1 → n3 Append base=0@0 [1@1]\nn3@1 append 1@1 None\nn3@1 → n1 AppendResponse match_index=1\n\nstatus\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=0@0 applied=0\n" }, { "path": "src/raft/testscripts/node/heartbeat_converts_leader", "content": "# A heartbeat from a leader should convert a leader in a past term to a\n# follower.\n\ncluster nodes=3 leader=3\n---\nn1@1 follower(n3) last=1@1 commit=1@1 applied=1\nn2@1 follower(n3) last=1@1 commit=1@1 applied=1\nn3@1 leader last=1@1 commit=1@1 applied=1 progress={1:1→2 2:1→2}\n\n# Partition n3 away from the cluster.\npartition 3\n---\nn3 ⇹ n1 n2\n\n# Elect n1 as a new leader.\n(campaign 1)\n(stabilize heartbeat=true)\nstatus\n---\nn1@2 leader last=2@2 commit=2@2 applied=2 progress={2:2→3 3:0→3}\nn2@2 follower(n1) last=2@2 commit=2@2 applied=2\nn3@1 leader last=1@1 commit=1@1 applied=1 progress={1:1→2 2:1→2}\n\n# Heal the partition.\nheal\n---\nn1 n2 n3 fully connected\n\n# The next heartbeat from n1 converts n3 to a follower in term 2.\nheartbeat 1\nstabilize heartbeat=true\n---\nn1@2 → n2 Heartbeat last_index=2 commit_index=2 read_seq=0\nn1@2 → n3 Heartbeat last_index=2 commit_index=2 read_seq=0\nn2@2 → n1 HeartbeatResponse match_index=2 read_seq=0\nn3@1 leader ⇨ n3@2 follower(n1)\nn3@2 → n1 HeartbeatResponse match_index=0 read_seq=0\nn1@2 → n3 Append base=1@1 []\nn3@2 → n1 AppendResponse match_index=1\nn1@2 → n3 Append base=1@1 [2@2]\nn3@2 append 2@2 None\nn3@2 → n1 AppendResponse match_index=2\nn1@2 → n2 Heartbeat last_index=2 commit_index=2 read_seq=0\nn1@2 → n3 Heartbeat last_index=2 commit_index=2 read_seq=0\nn2@2 → n1 HeartbeatResponse match_index=2 read_seq=0\nn3@2 commit 2@2\nn3@2 apply 2@2 None\nn3@2 → n1 HeartbeatResponse match_index=2 read_seq=0\n\nstatus\n---\nn1@2 leader last=2@2 commit=2@2 applied=2 progress={2:2→3 3:2→3}\nn2@2 follower(n1) last=2@2 commit=2@2 applied=2\nn3@2 follower(n1) last=2@2 commit=2@2 applied=2\n" }, { "path": "src/raft/testscripts/node/heartbeat_lost_append_duplicate", "content": "# Duplicate heartbeats and responses with a lost append will\n# trigger duplicate resends, but it will eventually resolve.\n\ncluster nodes=3 leader=1\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Partition the leader, submit a write whose appends are dropped,\n# then heal the partition again.\npartition 1\n---\nn1 ⇹ n2 n3\n\nput 1 foo=bar\n---\nc1@1 → n1 ClientRequest id=0x01 write 0x0103666f6f03626172\nn1@1 append 2@1 put foo=bar\nn1@1 ⇥ n2 A̶p̶p̶e̶n̶d̶ ̶b̶a̶s̶e̶=̶1̶@̶1̶ ̶[̶2̶@̶1̶]̶\nn1@1 ⇥ n3 A̶p̶p̶e̶n̶d̶ ̶b̶a̶s̶e̶=̶1̶@̶1̶ ̶[̶2̶@̶1̶]̶\n\nheal\n---\nn1 n2 n3 fully connected\n\n# The next heartbeat will result in match_index=0 since the followers\n# don't have the last_index. 3 heartbeats are made.\nheartbeat 1\nheartbeat 1\nheartbeat 1\ndeliver\n---\nn1@1 → n2 Heartbeat last_index=2 commit_index=1 read_seq=0\nn1@1 → n3 Heartbeat last_index=2 commit_index=1 read_seq=0\nn1@1 → n2 Heartbeat last_index=2 commit_index=1 read_seq=0\nn1@1 → n3 Heartbeat last_index=2 commit_index=1 read_seq=0\nn1@1 → n2 Heartbeat last_index=2 commit_index=1 read_seq=0\nn1@1 → n3 Heartbeat last_index=2 commit_index=1 read_seq=0\nn2@1 → n1 HeartbeatResponse match_index=0 read_seq=0\nn2@1 → n1 HeartbeatResponse match_index=0 read_seq=0\nn2@1 → n1 HeartbeatResponse match_index=0 read_seq=0\nn3@1 → n1 HeartbeatResponse match_index=0 read_seq=0\nn3@1 → n1 HeartbeatResponse match_index=0 read_seq=0\nn3@1 → n1 HeartbeatResponse match_index=0 read_seq=0\n\n# The leader has previously matched the followers at index 1.\nstatus 1\n---\nn1@1 leader last=2@1 commit=1@1 applied=1 progress={2:1→3 3:1→3}\n\n# When it receives the heartbeat responses, it sends duplicates of the missing\n# entries.\ndeliver\n---\nn1@1 → n2 Append base=1@1 [2@1]\nn1@1 → n2 Append base=1@1 [2@1]\nn1@1 → n2 Append base=1@1 [2@1]\nn1@1 → n3 Append base=1@1 [2@1]\nn1@1 → n3 Append base=1@1 [2@1]\nn1@1 → n3 Append base=1@1 [2@1]\n\n# The followers accept the duplicate appends and the leader commits and applies.\nstabilize\n---\nn2@1 append 2@1 put foo=bar\nn2@1 → n1 AppendResponse match_index=2\nn2@1 → n1 AppendResponse match_index=2\nn2@1 → n1 AppendResponse match_index=2\nn3@1 append 2@1 put foo=bar\nn3@1 → n1 AppendResponse match_index=2\nn3@1 → n1 AppendResponse match_index=2\nn3@1 → n1 AppendResponse match_index=2\nn1@1 commit 2@1\nn1@1 apply 2@1 put foo=bar\nn1@1 → c1 ClientResponse id=0x01 write 0x0102\nc1@1 put foo=bar ⇒ 2\n" }, { "path": "src/raft/testscripts/node/heartbeat_lost_append_multiple", "content": "# A heartbeat response triggers a probe and resend of lost appends.\n\ncluster nodes=3 leader=1\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Partition the leader, submit three writes whose appends are dropped, then heal\n# the partition again.\npartition 1\n---\nn1 ⇹ n2 n3\n\nput 1 a=1\nput 1 b=2\nput 1 c=3\n---\nc1@1 → n1 ClientRequest id=0x01 write 0x0101610131\nn1@1 append 2@1 put a=1\nn1@1 ⇥ n2 A̶p̶p̶e̶n̶d̶ ̶b̶a̶s̶e̶=̶1̶@̶1̶ ̶[̶2̶@̶1̶]̶\nn1@1 ⇥ n3 A̶p̶p̶e̶n̶d̶ ̶b̶a̶s̶e̶=̶1̶@̶1̶ ̶[̶2̶@̶1̶]̶\nc1@1 → n1 ClientRequest id=0x02 write 0x0101620132\nn1@1 append 3@1 put b=2\nn1@1 ⇥ n2 A̶p̶p̶e̶n̶d̶ ̶b̶a̶s̶e̶=̶2̶@̶1̶ ̶[̶3̶@̶1̶]̶\nn1@1 ⇥ n3 A̶p̶p̶e̶n̶d̶ ̶b̶a̶s̶e̶=̶2̶@̶1̶ ̶[̶3̶@̶1̶]̶\nc1@1 → n1 ClientRequest id=0x03 write 0x0101630133\nn1@1 append 4@1 put c=3\nn1@1 ⇥ n2 A̶p̶p̶e̶n̶d̶ ̶b̶a̶s̶e̶=̶3̶@̶1̶ ̶[̶4̶@̶1̶]̶\nn1@1 ⇥ n3 A̶p̶p̶e̶n̶d̶ ̶b̶a̶s̶e̶=̶3̶@̶1̶ ̶[̶4̶@̶1̶]̶\n\nheal\nstatus\n---\nn1 n2 n3 fully connected\nn1@1 leader last=4@1 commit=1@1 applied=1 progress={2:1→5 3:1→5}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# The next heartbeat will result in match_index=0 since the followers\n# don't have the last_index.\nheartbeat 1\ndeliver\n---\nn1@1 → n2 Heartbeat last_index=4 commit_index=1 read_seq=0\nn1@1 → n3 Heartbeat last_index=4 commit_index=1 read_seq=0\nn2@1 → n1 HeartbeatResponse match_index=0 read_seq=0\nn3@1 → n1 HeartbeatResponse match_index=0 read_seq=0\n\n# The leader has previously matched the followers at index 1.\nstatus 1\n---\nn1@1 leader last=4@1 commit=1@1 applied=1 progress={2:1→5 3:1→5}\n\n# When it receives the heartbeat response, it probes the previous index 3.\ndeliver\n---\nn1@1 → n2 Append base=3@1 []\nn1@1 → n3 Append base=3@1 []\n\n# The followers don't have index 3. They don't have index 2 either, but they\n# do have 1, so they respond with a reject_index=2.\ndeliver\n---\nn2@1 → n1 AppendResponse reject_index=2\nn3@1 → n1 AppendResponse reject_index=2\n\n# The leader has already matched index 1, so it doesn't have to probe for it,\n# and can simply send the tail of the log.\ndeliver\n---\nn1@1 → n2 Append base=1@1 [2@1 3@1 4@1]\nn1@1 → n3 Append base=1@1 [2@1 3@1 4@1]\n\n# The followers accept the append and the leader commits and applies.\nstabilize\n---\nn2@1 append 2@1 put a=1\nn2@1 append 3@1 put b=2\nn2@1 append 4@1 put c=3\nn2@1 → n1 AppendResponse match_index=4\nn3@1 append 2@1 put a=1\nn3@1 append 3@1 put b=2\nn3@1 append 4@1 put c=3\nn3@1 → n1 AppendResponse match_index=4\nn1@1 commit 4@1\nn1@1 apply 2@1 put a=1\nn1@1 apply 3@1 put b=2\nn1@1 apply 4@1 put c=3\nn1@1 → c1 ClientResponse id=0x01 write 0x0102\nc1@1 put a=1 ⇒ 2\nn1@1 → c1 ClientResponse id=0x02 write 0x0103\nc1@1 put b=2 ⇒ 3\nn1@1 → c1 ClientResponse id=0x03 write 0x0104\nc1@1 put c=3 ⇒ 4\n" }, { "path": "src/raft/testscripts/node/heartbeat_lost_append_single", "content": "# A heartbeat response triggers a resend of a lost append.\n\ncluster nodes=3 leader=1\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Partition the leader, submit a write whose appends are dropped,\n# then heal the partition again.\npartition 1\n---\nn1 ⇹ n2 n3\n\nput 1 foo=bar\n---\nc1@1 → n1 ClientRequest id=0x01 write 0x0103666f6f03626172\nn1@1 append 2@1 put foo=bar\nn1@1 ⇥ n2 A̶p̶p̶e̶n̶d̶ ̶b̶a̶s̶e̶=̶1̶@̶1̶ ̶[̶2̶@̶1̶]̶\nn1@1 ⇥ n3 A̶p̶p̶e̶n̶d̶ ̶b̶a̶s̶e̶=̶1̶@̶1̶ ̶[̶2̶@̶1̶]̶\n\nheal\n---\nn1 n2 n3 fully connected\n\n# The next heartbeat will result in match_index=0 since the followers\n# don't have the last_index.\nheartbeat 1\ndeliver\n---\nn1@1 → n2 Heartbeat last_index=2 commit_index=1 read_seq=0\nn1@1 → n3 Heartbeat last_index=2 commit_index=1 read_seq=0\nn2@1 → n1 HeartbeatResponse match_index=0 read_seq=0\nn3@1 → n1 HeartbeatResponse match_index=0 read_seq=0\n\n# The leader has previously matched the followers at index 1.\nstatus 1\n---\nn1@1 leader last=2@1 commit=1@1 applied=1 progress={2:1→3 3:1→3}\n\n# When it receives the heartbeat response, instead of probing index 1 and then\n# sending the actual entries, it simply sends the entries.\ndeliver\n---\nn1@1 → n2 Append base=1@1 [2@1]\nn1@1 → n3 Append base=1@1 [2@1]\n\n# The followers accept the append and the leader commits and applies.\nstabilize\n---\nn2@1 append 2@1 put foo=bar\nn2@1 → n1 AppendResponse match_index=2\nn3@1 append 2@1 put foo=bar\nn3@1 → n1 AppendResponse match_index=2\nn1@1 commit 2@1\nn1@1 apply 2@1 put foo=bar\nn1@1 → c1 ClientResponse id=0x01 write 0x0102\nc1@1 put foo=bar ⇒ 2\n" }, { "path": "src/raft/testscripts/node/heartbeat_lost_read", "content": "# Heartbeats will recover from a lost read message.\n\ncluster nodes=5 leader=1\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2 4:1→2 5:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\nn4@1 follower(n1) last=1@1 commit=1@1 applied=1\nn5@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Write a key and replicate it.\n(put 1 foo=bar)\n(stabilize heartbeat=true)\n---\nok\n\n# Partition the leader, and submit a read.\npartition 1\n---\nn1 ⇹ n2 n3 n4 n5\n\nget 1 foo\n---\nc1@1 → n1 ClientRequest id=0x02 read 0x0003666f6f\nn1@1 ⇥ n2 R̶e̶a̶d̶ ̶s̶e̶q̶=̶1̶\nn1@1 ⇥ n3 R̶e̶a̶d̶ ̶s̶e̶q̶=̶1̶\nn1@1 ⇥ n4 R̶e̶a̶d̶ ̶s̶e̶q̶=̶1̶\nn1@1 ⇥ n5 R̶e̶a̶d̶ ̶s̶e̶q̶=̶1̶\n\nheal\n---\nn1 n2 n3 n4 n5 fully connected\n\n# The next heartbeat will detect the failed read, and serve it when\n# it has a quorum.\nheartbeat 1\ndeliver\n---\nn1@1 → n2 Heartbeat last_index=2 commit_index=2 read_seq=1\nn1@1 → n3 Heartbeat last_index=2 commit_index=2 read_seq=1\nn1@1 → n4 Heartbeat last_index=2 commit_index=2 read_seq=1\nn1@1 → n5 Heartbeat last_index=2 commit_index=2 read_seq=1\nn2@1 → n1 HeartbeatResponse match_index=2 read_seq=1\nn3@1 → n1 HeartbeatResponse match_index=2 read_seq=1\nn4@1 → n1 HeartbeatResponse match_index=2 read_seq=1\nn5@1 → n1 HeartbeatResponse match_index=2 read_seq=1\n\n# The first response does not provide quorum.\ndeliver 1 from=2\n---\nok\n\n# The second does, and the read is served.\ndeliver 1 from=3\n---\nn1@1 → c1 ClientResponse id=0x02 read 0x000103626172\nc1@1 get foo ⇒ bar\n" }, { "path": "src/raft/testscripts/node/heartbeat_match_commits", "content": "# A heartbeat response can advance a follower match index and commit+apply.\n\ncluster nodes=3 leader=1\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Submit a write to the leader.\nput 1 foo=bar\n---\nc1@1 → n1 ClientRequest id=0x01 write 0x0103666f6f03626172\nn1@1 append 2@1 put foo=bar\nn1@1 → n2 Append base=1@1 [2@1]\nn1@1 → n3 Append base=1@1 [2@1]\n\n# Partition n1 away from the followers as they send the append acks, then heal\n# the partition.\npartition 1\n---\nn1 ⇹ n2 n3\n\nstabilize\n---\nn2@1 append 2@1 put foo=bar\nn2@1 ⇥ n1 A̶p̶p̶e̶n̶d̶R̶e̶s̶p̶o̶n̶s̶e̶ ̶m̶a̶t̶c̶h̶_̶i̶n̶d̶e̶x̶=̶2̶\nn3@1 append 2@1 put foo=bar\nn3@1 ⇥ n1 A̶p̶p̶e̶n̶d̶R̶e̶s̶p̶o̶n̶s̶e̶ ̶m̶a̶t̶c̶h̶_̶i̶n̶d̶e̶x̶=̶2̶\n\nheal\n---\nn1 n2 n3 fully connected\n\n# The write has been replicated, but not yet committed and applied.\nstatus\n---\nn1@1 leader last=2@1 commit=1@1 applied=1 progress={2:1→3 3:1→3}\nn2@1 follower(n1) last=2@1 commit=1@1 applied=1\nn3@1 follower(n1) last=2@1 commit=1@1 applied=1\n\n# The leader heartbeats. The followers confirm they are caught up.\nheartbeat 1\ndeliver\n---\nn1@1 → n2 Heartbeat last_index=2 commit_index=1 read_seq=0\nn1@1 → n3 Heartbeat last_index=2 commit_index=1 read_seq=0\nn2@1 → n1 HeartbeatResponse match_index=2 read_seq=0\nn3@1 → n1 HeartbeatResponse match_index=2 read_seq=0\n\n# When the leader receives the first heartbeat, it commits and applies\n# the write.\ndeliver 1 from=2\n---\nn1@1 commit 2@1\nn1@1 apply 2@1 put foo=bar\nn1@1 → c1 ClientResponse id=0x01 write 0x0102\nc1@1 put foo=bar ⇒ 2\n\nstatus 1\n---\nn1@1 leader last=2@1 commit=2@1 applied=2 progress={2:2→3 3:1→3}\n\n# Delivery of the second heartbeat advances the match index, but\n# there is nothing more to do.\ndeliver\nstatus\n---\nn1@1 leader last=2@1 commit=2@1 applied=2 progress={2:2→3 3:2→3}\nn2@1 follower(n1) last=2@1 commit=1@1 applied=1\nn3@1 follower(n1) last=2@1 commit=1@1 applied=1\n" }, { "path": "src/raft/testscripts/node/heartbeat_multiple_leaders_panic", "content": "# A heartbeat will panic if there are multiple leaders in a term.\n\ncluster nodes=3 leader=1\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Leader panics if it sees another leader in the same term.\n!step 1 '{\"from\":2, \"to\":1, \"term\":1, \"message\": {\"Heartbeat\":{\"last_index\":1,\"commit_index\":0, \"commit_term\":0, \"read_seq\":0}}}'\n---\nPanic: saw other leader 2 in term 1\n\n# Follower panics too.\n!step 2 '{\"from\":3, \"to\":2, \"term\":1, \"message\": {\"Heartbeat\":{\"last_index\":1,\"commit_index\":0, \"commit_term\":0, \"read_seq\":0}}}'\n---\nPanic: assertion `left == right` failed: multiple leaders in term\n left: 3\n right: 1\n" }, { "path": "src/raft/testscripts/node/heartbeat_old_commit_index", "content": "# A heartbeat with an old commit index is ignored by a follower.\n\ncluster nodes=3 leader=1\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Replicate a write.\n(put 1 foo=bar)\n(stabilize heartbeat=true)\nstatus\n---\nn1@1 leader last=2@1 commit=2@1 applied=2 progress={2:2→3 3:2→3}\nn2@1 follower(n1) last=2@1 commit=2@1 applied=2\nn3@1 follower(n1) last=2@1 commit=2@1 applied=2\n\n# Step a heartbeat with an outdated commit index.\nstep 2 '{\"from\":1, \"to\":2, \"term\":1, \"message\":{\"Heartbeat\":{\"last_index\":2,\"commit_index\":1,\"commit_term\":1,\"read_seq\":0}}}'\nstabilize\n---\nn2@1 → n1 HeartbeatResponse match_index=2 read_seq=0\n" }, { "path": "src/raft/testscripts/node/heartbeat_old_last_index", "content": "# A heartbeat with an old last index is matched by a follower.\n\ncluster nodes=3 leader=1\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Replicate a write.\n(put 1 foo=bar)\n(stabilize heartbeat=true)\nstatus\n---\nn1@1 leader last=2@1 commit=2@1 applied=2 progress={2:2→3 3:2→3}\nn2@1 follower(n1) last=2@1 commit=2@1 applied=2\nn3@1 follower(n1) last=2@1 commit=2@1 applied=2\n\n# Step a heartbeat with an outdated last index.\nstep 2 '{\"from\":1, \"to\":2, \"term\":1, \"message\":{\"Heartbeat\":{\"last_index\":1,\"commit_index\":1,\"commit_term\":1,\"read_seq\":0}}}'\nstabilize\n---\nn2@1 → n1 HeartbeatResponse match_index=1 read_seq=0\n" }, { "path": "src/raft/testscripts/node/heartbeat_probe_divergent", "content": "# A heartbeat while the leader is probing a follower with a long divergent tail\n# doesn't disrupt the probing, and won't result in a quadratically increasing\n# amount of probes with each heartbeat.\n\ncluster nodes=3 leader=1\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Make a couple of writes to ensure a common log prefix.\n(put 1 a=1)\n(put 1 b=2)\n(stabilize)\nstatus\n---\nn1@1 leader last=3@1 commit=3@1 applied=3 progress={2:3→4 3:3→4}\nn2@1 follower(n1) last=3@1 commit=1@1 applied=1\nn3@1 follower(n1) last=3@1 commit=1@1 applied=1\n\n# Partition n1\npartition 1\n---\nn1 ⇹ n2 n3\n\n# Elect new leaders in the majority partition and replicate a few writes.\n(campaign 2)\n(stabilize)\n(put 2 c=3)\n(put 2 d=4)\n(stabilize heartbeat=true)\nstatus\n---\nn1@1 leader last=3@1 commit=3@1 applied=3 progress={2:3→4 3:3→4}\nn2@2 leader last=6@2 commit=6@2 applied=6 progress={1:0→7 3:6→7}\nn3@2 follower(n2) last=6@2 commit=6@2 applied=6\n\n(campaign 3)\n(stabilize)\n(put 2 e=5)\n(put 2 f=6)\n(stabilize heartbeat=true)\nstatus\n---\nn1@1 leader last=3@1 commit=3@1 applied=3 progress={2:3→4 3:3→4}\nn2@3 follower(n3) last=9@3 commit=9@3 applied=9\nn3@3 leader last=9@3 commit=9@3 applied=9 progress={1:0→10 2:9→10}\n\n# Propose writes in the minority partition as well, to build up a divergent log.\n(put 1 a=2)\n(put 1 a=3)\n(put 1 a=4)\n(put 1 a=5)\n(put 1 a=6)\n(put 1 a=7)\n(put 1 a=8)\n(put 1 a=9)\n(stabilize)\nstatus\n---\nn1@1 leader last=11@1 commit=3@1 applied=3 progress={2:3→12 3:3→12}\nn2@3 follower(n3) last=9@3 commit=9@3 applied=9\nn3@3 leader last=9@3 commit=9@3 applied=9 progress={1:0→10 2:9→10}\n\n# Heal the partition.\nheal\n---\nn1 n2 n3 fully connected\n\n# Propose another write on the majority leader to start probing.\nput 3 g=7\n---\nc3@3 → n3 ClientRequest id=0x0f write 0x0101670137\nn3@3 append 10@3 put g=7\nn3@3 → n1 Append base=9@3 [10@3]\nn3@3 → n2 Append base=9@3 [10@3]\n\n# The append should be rejected by n1, canceling the writes.\ndeliver 1\n---\nn1@1 leader ⇨ n1@3 follower(n3)\nn1@1 → c1 ClientResponse id=0x07 Error::Abort\nc1@1 put a=2 ⇒ Error::Abort (operation aborted)\nn1@1 → c1 ClientResponse id=0x08 Error::Abort\nc1@1 put a=3 ⇒ Error::Abort (operation aborted)\nn1@1 → c1 ClientResponse id=0x09 Error::Abort\nc1@1 put a=4 ⇒ Error::Abort (operation aborted)\nn1@1 → c1 ClientResponse id=0x0a Error::Abort\nc1@1 put a=5 ⇒ Error::Abort (operation aborted)\nn1@1 → c1 ClientResponse id=0x0b Error::Abort\nc1@1 put a=6 ⇒ Error::Abort (operation aborted)\nn1@1 → c1 ClientResponse id=0x0c Error::Abort\nc1@1 put a=7 ⇒ Error::Abort (operation aborted)\nn1@1 → c1 ClientResponse id=0x0d Error::Abort\nc1@1 put a=8 ⇒ Error::Abort (operation aborted)\nn1@1 → c1 ClientResponse id=0x0e Error::Abort\nc1@1 put a=9 ⇒ Error::Abort (operation aborted)\nn1@3 → n3 AppendResponse reject_index=9\n\n# n3 begins probing, and also heartbeats.\ndeliver 3\nheartbeat 3\ndeliver 1\nstatus 3\n---\nn3@3 → n1 Append base=8@3 []\nn3@3 → n1 Heartbeat last_index=10 commit_index=9 read_seq=0\nn3@3 → n2 Heartbeat last_index=10 commit_index=9 read_seq=0\nn1@3 → n3 AppendResponse reject_index=8\nn1@3 → n3 HeartbeatResponse match_index=0 read_seq=0\nn3@3 leader last=10@3 commit=9@3 applied=9 progress={1:0→9 2:9→11}\n\n# n3 receives probe and heartbeat responses, resulting in duplicate\n# probes being sent at base index 7.\ndeliver 3\nstatus 3\n---\nn3@3 → n1 Append base=7@3 []\nn3@3 → n1 Append base=7@3 []\nn3@3 leader last=10@3 commit=9@3 applied=9 progress={1:0→8 2:9→11}\n\ndeliver 1\n---\nn1@3 → n3 AppendResponse reject_index=7\nn1@3 → n3 AppendResponse reject_index=7\n\n# However, when receiving the duplicate probe responses, they are\n# deduplicated and only a single new probe is sent.\ndeliver 3\n---\nn3@3 → n1 Append base=6@2 []\n\ndeliver 1\n---\nn1@3 → n3 AppendResponse reject_index=6\n\n# n3 heartbeats again before sending the next probe. This results in\n# two probes: the heartbeat response resends the probe at base 5, while\n# the probe response triggers a new probe at base 4.\nheartbeat 3\ndeliver 3\n---\nn3@3 → n1 Heartbeat last_index=10 commit_index=9 read_seq=0\nn3@3 → n2 Heartbeat last_index=10 commit_index=9 read_seq=0\nn3@3 → n1 Append base=5@2 []\n\ndeliver 1\n---\nn1@3 → n3 HeartbeatResponse match_index=0 read_seq=0\nn1@3 → n3 AppendResponse reject_index=5\n\ndeliver 3\n---\nn3@3 → n1 Append base=5@2 []\nn3@3 → n1 Append base=4@2 []\n\ndeliver 1\n---\nn1@3 → n3 AppendResponse reject_index=5\nn1@3 → n3 AppendResponse reject_index=4\n\n# The probe response at reject_index=5 is ignored, since we're already probed\n# it. Only a single new probe is sent at base 4.\ndeliver 3\n---\nn3@3 → n1 Append base=3@1 []\n\n# When delivered, we finally get a match, and the follower gets caught up.\ndeliver 1\n---\nn1@3 → n3 AppendResponse match_index=3\n\ndeliver 3\n---\nn3@3 → n1 Append base=3@1 [4@2 5@2 6@2 7@3 8@3 9@3 10@3]\n\ndeliver 1\n---\nn1@3 append 4@2 None\nn1@3 append 5@2 put c=3\nn1@3 append 6@2 put d=4\nn1@3 append 7@3 None\nn1@3 append 8@3 put e=5\nn1@3 append 9@3 put f=6\nn1@3 append 10@3 put g=7\nn1@3 → n3 AppendResponse match_index=10\n\ndeliver 3\n---\nn3@3 commit 10@3\nn3@3 apply 10@3 put g=7\nn3@3 → c3 ClientResponse id=0x0f write 0x010a\nc3@3 put g=7 ⇒ 10\n" }, { "path": "src/raft/testscripts/node/old_campaign_rejected", "content": "# Old campaign messages (in the same term) are ignored by leaders and followers\n# once a leader is elected.\n\ncluster nodes=3\n---\nn1@0 follower() last=0@0 commit=0@0 applied=0\nn2@0 follower() last=0@0 commit=0@0 applied=0\nn3@0 follower() last=0@0 commit=0@0 applied=0\n\n# n1 and n2 campaign.\ncampaign 1 2\n---\nn1@0 follower() ⇨ n1@1 candidate\nn1@1 → n2 Campaign last=0@0\nn1@1 → n3 Campaign last=0@0\nn2@0 follower() ⇨ n2@1 candidate\nn2@1 → n1 Campaign last=0@0\nn2@1 → n3 Campaign last=0@0\n\n# n3 receives n1's Campaign message and grants its vote.\ndeliver 3 from=1\n---\nn3@0 follower() ⇨ n3@1 follower()\nn3@1 → n1 CampaignResponse vote=true\n\n# n1 becomes leader.\ndeliver 1 from=3\n---\nn1@1 candidate ⇨ n1@1 leader\nn1@1 append 1@1 None\nn1@1 → n2 Append base=0@0 [1@1]\nn1@1 → n3 Append base=0@0 [1@1]\nn1@1 → n2 Heartbeat last_index=1 commit_index=0 read_seq=0\nn1@1 → n3 Heartbeat last_index=1 commit_index=0 read_seq=0\n\n# n3 receives n1's heartbeat and becomes follower.\ndeliver 3 from=1\n---\nn3@1 follower() ⇨ n3@1 follower(n1)\nn3@1 append 1@1 None\nn3@1 → n1 AppendResponse match_index=1\nn3@1 → n1 HeartbeatResponse match_index=1 read_seq=0\n\nstatus\n---\nn1@1 leader last=1@1 commit=0@0 applied=0 progress={2:0→2 3:0→2}\nn2@1 candidate last=0@0 commit=0@0 applied=0\nn3@1 follower(n1) last=1@1 commit=0@0 applied=0\n\n# n1 and n3 receive n2's Campaign message and reject it.\ndeliver 1 3 from=2\n---\nn1@1 → n2 CampaignResponse vote=false\nn3@1 → n2 CampaignResponse vote=false\n\n# Stabilizing the cluster results in everyone following n1.\n(stabilize heartbeat=true)\nstatus\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\n" }, { "path": "src/raft/testscripts/node/old_campaign_response_ignored", "content": "# Old campaign responses (in the same term) are ignored by leaders and followers\n# once a leader is elected.\n\ncluster nodes=7\n---\nn1@0 follower() last=0@0 commit=0@0 applied=0\nn2@0 follower() last=0@0 commit=0@0 applied=0\nn3@0 follower() last=0@0 commit=0@0 applied=0\nn4@0 follower() last=0@0 commit=0@0 applied=0\nn5@0 follower() last=0@0 commit=0@0 applied=0\nn6@0 follower() last=0@0 commit=0@0 applied=0\nn7@0 follower() last=0@0 commit=0@0 applied=0\n\n# n1 and n2 campaign.\ncampaign 1 2\n---\nn1@0 follower() ⇨ n1@1 candidate\nn1@1 → n2 Campaign last=0@0\nn1@1 → n3 Campaign last=0@0\nn1@1 → n4 Campaign last=0@0\nn1@1 → n5 Campaign last=0@0\nn1@1 → n6 Campaign last=0@0\nn1@1 → n7 Campaign last=0@0\nn2@0 follower() ⇨ n2@1 candidate\nn2@1 → n1 Campaign last=0@0\nn2@1 → n3 Campaign last=0@0\nn2@1 → n4 Campaign last=0@0\nn2@1 → n5 Campaign last=0@0\nn2@1 → n6 Campaign last=0@0\nn2@1 → n7 Campaign last=0@0\n\n# n3-n6 vote for n1, n7 votes for n2.\ndeliver 3 4 5 6 from=1\ndeliver 7 from=2\n---\nn3@0 follower() ⇨ n3@1 follower()\nn3@1 → n1 CampaignResponse vote=true\nn4@0 follower() ⇨ n4@1 follower()\nn4@1 → n1 CampaignResponse vote=true\nn5@0 follower() ⇨ n5@1 follower()\nn5@1 → n1 CampaignResponse vote=true\nn6@0 follower() ⇨ n6@1 follower()\nn6@1 → n1 CampaignResponse vote=true\nn7@0 follower() ⇨ n7@1 follower()\nn7@1 → n2 CampaignResponse vote=true\n\n# n1 receives votes from n3-n5 and assumes leadership.\ndeliver 1 from=3\ndeliver 1 from=4\ndeliver 1 from=5\n---\nn1@1 candidate ⇨ n1@1 leader\nn1@1 append 1@1 None\nn1@1 → n2 Append base=0@0 [1@1]\nn1@1 → n3 Append base=0@0 [1@1]\nn1@1 → n4 Append base=0@0 [1@1]\nn1@1 → n5 Append base=0@0 [1@1]\nn1@1 → n6 Append base=0@0 [1@1]\nn1@1 → n7 Append base=0@0 [1@1]\nn1@1 → n2 Heartbeat last_index=1 commit_index=0 read_seq=0\nn1@1 → n3 Heartbeat last_index=1 commit_index=0 read_seq=0\nn1@1 → n4 Heartbeat last_index=1 commit_index=0 read_seq=0\nn1@1 → n5 Heartbeat last_index=1 commit_index=0 read_seq=0\nn1@1 → n6 Heartbeat last_index=1 commit_index=0 read_seq=0\nn1@1 → n7 Heartbeat last_index=1 commit_index=0 read_seq=0\n\n# n2 receives n1's heartbeats and becomes follower.\ndeliver 2 from=1\n---\nn2@1 → n1 CampaignResponse vote=false\nn2@1 candidate ⇨ n2@1 follower(n1)\nn2@1 append 1@1 None\nn2@1 → n1 AppendResponse match_index=1\nn2@1 → n1 HeartbeatResponse match_index=1 read_seq=0\n\n# n1 (leader) receives n6's vote and ignores it. n2 (follower) receives n7's\n# vote and ignores it. They remain leader and follower.\ndeliver 1 from=6\ndeliver 2 from=7\nstatus\n---\nn1@1 leader last=1@1 commit=0@0 applied=0 progress={2:0→2 3:0→2 4:0→2 5:0→2 6:0→2 7:0→2}\nn2@1 follower(n1) last=1@1 commit=0@0 applied=0\nn3@1 follower() last=0@0 commit=0@0 applied=0\nn4@1 follower() last=0@0 commit=0@0 applied=0\nn5@1 follower() last=0@0 commit=0@0 applied=0\nn6@1 follower() last=0@0 commit=0@0 applied=0\nn7@1 follower() last=0@0 commit=0@0 applied=0\n\n# Stabilizing the cluster results in everyone following n1.\n(stabilize heartbeat=true)\nstatus\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2 4:1→2 5:1→2 6:1→2 7:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\nn4@1 follower(n1) last=1@1 commit=1@1 applied=1\nn5@1 follower(n1) last=1@1 commit=1@1 applied=1\nn6@1 follower(n1) last=1@1 commit=1@1 applied=1\nn7@1 follower(n1) last=1@1 commit=1@1 applied=1\n" }, { "path": "src/raft/testscripts/node/old_heartbeat_ignored", "content": "# A heartbeat from an old leader should be ignored.\n\n# Make n3 leader.\ncluster nodes=3 leader=3\n---\nn1@1 follower(n3) last=1@1 commit=1@1 applied=1\nn2@1 follower(n3) last=1@1 commit=1@1 applied=1\nn3@1 leader last=1@1 commit=1@1 applied=1 progress={1:1→2 2:1→2}\n\n# Partition n3 away from the cluster.\npartition 3\n---\nn3 ⇹ n1 n2\n\n# Elect n1 as a new leader.\n(campaign 1)\n(stabilize heartbeat=true)\nstatus\n---\nn1@2 leader last=2@2 commit=2@2 applied=2 progress={2:2→3 3:0→3}\nn2@2 follower(n1) last=2@2 commit=2@2 applied=2\nn3@1 leader last=1@1 commit=1@1 applied=1 progress={1:1→2 2:1→2}\n\n# Heal the partition.\nheal\n---\nn1 n2 n3 fully connected\n\n# The next heartbeat from n3 is ignored.\nheartbeat 3\nstabilize\n---\nn3@1 → n1 Heartbeat last_index=1 commit_index=1 read_seq=0\nn3@1 → n2 Heartbeat last_index=1 commit_index=1 read_seq=0\n\nstatus\n---\nn1@2 leader last=2@2 commit=2@2 applied=2 progress={2:2→3 3:0→3}\nn2@2 follower(n1) last=2@2 commit=2@2 applied=2\nn3@1 leader last=1@1 commit=1@1 applied=1 progress={1:1→2 2:1→2}\n" }, { "path": "src/raft/testscripts/node/request_candidate_abort", "content": "# Client read/write requests fail on candidates.\n\ncluster nodes=3\n---\nn1@0 follower() last=0@0 commit=0@0 applied=0\nn2@0 follower() last=0@0 commit=0@0 applied=0\nn3@0 follower() last=0@0 commit=0@0 applied=0\n\n# n1 campaigns.\ncampaign 1\n---\nn1@0 follower() ⇨ n1@1 candidate\nn1@1 → n2 Campaign last=0@0\nn1@1 → n3 Campaign last=0@0\n\n# A read request on n1 should be rejected.\nget 1 foo\n---\nc1@1 → n1 ClientRequest id=0x01 read 0x0003666f6f\nn1@1 → c1 ClientResponse id=0x01 Error::Abort\nc1@1 get foo ⇒ Error::Abort (operation aborted)\n\n# A write request on n1 should be rejected.\nput 1 foo=bar\n---\nc1@1 → n1 ClientRequest id=0x02 write 0x0103666f6f03626172\nn1@1 → c1 ClientResponse id=0x02 Error::Abort\nc1@1 put foo=bar ⇒ Error::Abort (operation aborted)\n" }, { "path": "src/raft/testscripts/node/request_follower", "content": "# Client read/write requests are proxied by followers.\n\ncluster nodes=3 leader=1\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# An initial get on a follower yields None.\nget 2 foo\nstabilize\n---\nc2@1 → n2 ClientRequest id=0x01 read 0x0003666f6f\nn2@1 → n1 ClientRequest id=0x01 read 0x0003666f6f\nn1@1 → n2 Read seq=1\nn1@1 → n3 Read seq=1\nn2@1 → n1 ReadResponse seq=1\nn3@1 → n1 ReadResponse seq=1\nn1@1 → n2 ClientResponse id=0x01 read 0x0000\nn2@1 → c2 ClientResponse id=0x01 read 0x0000\nc2@1 get foo ⇒ None\n\n# Write a value on the follower.\nput 2 foo=bar\nstabilize\n(stabilize heartbeat=true)\n---\nc2@1 → n2 ClientRequest id=0x02 write 0x0103666f6f03626172\nn2@1 → n1 ClientRequest id=0x02 write 0x0103666f6f03626172\nn1@1 append 2@1 put foo=bar\nn1@1 → n2 Append base=1@1 [2@1]\nn1@1 → n3 Append base=1@1 [2@1]\nn2@1 append 2@1 put foo=bar\nn2@1 → n1 AppendResponse match_index=2\nn3@1 append 2@1 put foo=bar\nn3@1 → n1 AppendResponse match_index=2\nn1@1 commit 2@1\nn1@1 apply 2@1 put foo=bar\nn1@1 → n2 ClientResponse id=0x02 write 0x0102\nn2@1 → c2 ClientResponse id=0x02 write 0x0102\nc2@1 put foo=bar ⇒ 2\n\n# Read the value back on the follower.\nget 2 foo\nstabilize\n---\nc2@1 → n2 ClientRequest id=0x03 read 0x0003666f6f\nn2@1 → n1 ClientRequest id=0x03 read 0x0003666f6f\nn1@1 → n2 Read seq=2\nn1@1 → n3 Read seq=2\nn2@1 → n1 ReadResponse seq=2\nn3@1 → n1 ReadResponse seq=2\nn1@1 → n2 ClientResponse id=0x03 read 0x000103626172\nn2@1 → c2 ClientResponse id=0x03 read 0x000103626172\nc2@1 get foo ⇒ bar\n" }, { "path": "src/raft/testscripts/node/request_follower_campaign_abort", "content": "# A follower aborts in-flight requests when it steps down.\n\ncluster nodes=3 leader=1\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Submit a read and write on n2.\nput 2 foo=bar\nget 2 foo\n---\nc2@1 → n2 ClientRequest id=0x01 write 0x0103666f6f03626172\nn2@1 → n1 ClientRequest id=0x01 write 0x0103666f6f03626172\nc2@1 → n2 ClientRequest id=0x02 read 0x0003666f6f\nn2@1 → n1 ClientRequest id=0x02 read 0x0003666f6f\n\n# n3 campaigns before n2's requests achieve quorum.\ncampaign 3\n---\nn3@1 follower(n1) ⇨ n3@2 candidate\nn3@2 → n1 Campaign last=1@1\nn3@2 → n2 Campaign last=1@1\n\n# When n2 receives the campaign message, the requests are aborted.\ndeliver 2 from=3\n---\nn2@1 follower(n1) ⇨ n2@2 follower()\nn2@1 → c2 ClientResponse id=0x01 Error::Abort\nc2@1 put foo=bar ⇒ Error::Abort (operation aborted)\nn2@1 → c2 ClientResponse id=0x02 Error::Abort\nc2@1 get foo ⇒ Error::Abort (operation aborted)\nn2@2 → n3 CampaignResponse vote=true\n" }, { "path": "src/raft/testscripts/node/request_follower_disconnect_stall", "content": "# Client read/write requests stall if the follower is disconnected from the\n# leader when the request is submitted. They are not retried, nor aborted.\n\ncluster nodes=3 leader=1\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Partition n3 away from the cluster.\npartition 3\n---\nn3 ⇹ n1 n2\n\n# Submit write and read requests to n3. They don't return a result.\nput 3 foo=bar\nget 3 foo\nstabilize\n---\nc3@1 → n3 ClientRequest id=0x01 write 0x0103666f6f03626172\nn3@1 ⇥ n1 C̶l̶i̶e̶n̶t̶R̶e̶q̶u̶e̶s̶t̶ ̶i̶d̶=̶0̶x̶0̶1̶ ̶w̶r̶i̶t̶e̶ ̶0̶x̶0̶1̶0̶3̶6̶6̶6̶f̶6̶f̶0̶3̶6̶2̶6̶1̶7̶2̶\nc3@1 → n3 ClientRequest id=0x02 read 0x0003666f6f\nn3@1 ⇥ n1 C̶l̶i̶e̶n̶t̶R̶e̶q̶u̶e̶s̶t̶ ̶i̶d̶=̶0̶x̶0̶2̶ ̶r̶e̶a̶d̶ ̶0̶x̶0̶0̶0̶3̶6̶6̶6̶f̶6̶f̶\n\n# Heal the partition and heartbeat. The requests still don't return a result.\nheal\n---\nn1 n2 n3 fully connected\n\nstabilize heartbeat=true\n---\nn1@1 → n2 Heartbeat last_index=1 commit_index=1 read_seq=0\nn1@1 → n3 Heartbeat last_index=1 commit_index=1 read_seq=0\nn2@1 → n1 HeartbeatResponse match_index=1 read_seq=0\nn3@1 → n1 HeartbeatResponse match_index=1 read_seq=0\n" }, { "path": "src/raft/testscripts/node/request_follower_leaderless_abort", "content": "# Client read/write requests fail on leaderless followers.\n\ncluster nodes=3\n---\nn1@0 follower() last=0@0 commit=0@0 applied=0\nn2@0 follower() last=0@0 commit=0@0 applied=0\nn3@0 follower() last=0@0 commit=0@0 applied=0\n\n# A read request on n1 should be rejected.\nget 1 foo\n---\nc1@0 → n1 ClientRequest id=0x01 read 0x0003666f6f\nn1@0 → c1 ClientResponse id=0x01 Error::Abort\nc1@0 get foo ⇒ Error::Abort (operation aborted)\n\n# A write request on n1 should be rejected.\nput 1 foo=bar\n---\nc1@0 → n1 ClientRequest id=0x02 write 0x0103666f6f03626172\nn1@0 → c1 ClientResponse id=0x02 Error::Abort\nc1@0 put foo=bar ⇒ Error::Abort (operation aborted)\n" }, { "path": "src/raft/testscripts/node/request_leader", "content": "# Client read/write requests succeed on leaders.\n\ncluster nodes=3 leader=1\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# An initial get on the leader yields None.\nget 1 foo\nstabilize\n---\nc1@1 → n1 ClientRequest id=0x01 read 0x0003666f6f\nn1@1 → n2 Read seq=1\nn1@1 → n3 Read seq=1\nn2@1 → n1 ReadResponse seq=1\nn3@1 → n1 ReadResponse seq=1\nn1@1 → c1 ClientResponse id=0x01 read 0x0000\nc1@1 get foo ⇒ None\n\n# Write a value on the leader.\nput 1 foo=bar\nstabilize\n(stabilize heartbeat=true)\n---\nc1@1 → n1 ClientRequest id=0x02 write 0x0103666f6f03626172\nn1@1 append 2@1 put foo=bar\nn1@1 → n2 Append base=1@1 [2@1]\nn1@1 → n3 Append base=1@1 [2@1]\nn2@1 append 2@1 put foo=bar\nn2@1 → n1 AppendResponse match_index=2\nn3@1 append 2@1 put foo=bar\nn3@1 → n1 AppendResponse match_index=2\nn1@1 commit 2@1\nn1@1 apply 2@1 put foo=bar\nn1@1 → c1 ClientResponse id=0x02 write 0x0102\nc1@1 put foo=bar ⇒ 2\n\n# Read the value back on the leader.\nget 1 foo\nstabilize\n---\nc1@1 → n1 ClientRequest id=0x03 read 0x0003666f6f\nn1@1 → n2 Read seq=2\nn1@1 → n3 Read seq=2\nn2@1 → n1 ReadResponse seq=2\nn3@1 → n1 ReadResponse seq=2\nn1@1 → c1 ClientResponse id=0x03 read 0x000103626172\nc1@1 get foo ⇒ bar\n" }, { "path": "src/raft/testscripts/node/request_leader_campaign_abort", "content": "# A leader aborts in-flight requests when it steps down.\n\ncluster nodes=3 leader=1\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Submit a read and write on n1.\nput 1 foo=bar\nget 1 foo\n---\nc1@1 → n1 ClientRequest id=0x01 write 0x0103666f6f03626172\nn1@1 append 2@1 put foo=bar\nn1@1 → n2 Append base=1@1 [2@1]\nn1@1 → n3 Append base=1@1 [2@1]\nc1@1 → n1 ClientRequest id=0x02 read 0x0003666f6f\nn1@1 → n2 Read seq=1\nn1@1 → n3 Read seq=1\n\n# n2 campaigns before n1's requests achieve quorum.\ncampaign 2\n---\nn2@1 follower(n1) ⇨ n2@2 candidate\nn2@2 → n1 Campaign last=1@1\nn2@2 → n3 Campaign last=1@1\n\n# When n1 receives the campaign message, the requests are aborted.\ndeliver 1 from=2\n---\nn1@1 leader ⇨ n1@2 follower()\nn1@1 → c1 ClientResponse id=0x01 Error::Abort\nc1@1 put foo=bar ⇒ Error::Abort (operation aborted)\nn1@1 → c1 ClientResponse id=0x02 Error::Abort\nc1@1 get foo ⇒ Error::Abort (operation aborted)\nn1@2 → n2 CampaignResponse vote=false\n" }, { "path": "src/raft/testscripts/node/request_leader_change_linearizability", "content": "# A new leader that's behind on commit/apply shouldn't serve stale reads.\n\ncluster nodes=3 leader=1\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Write an initial value, and propagate the commit index.\n(put 1 a=1)\n(stabilize heartbeat=true)\nstatus\n---\nn1@1 leader last=2@1 commit=2@1 applied=2 progress={2:2→3 3:2→3}\nn2@1 follower(n1) last=2@1 commit=2@1 applied=2\nn3@1 follower(n1) last=2@1 commit=2@1 applied=2\n\n# Write another value, but don't propagate the commit index.\n(put 1 b=2)\n(stabilize)\nstatus\n---\nn1@1 leader last=3@1 commit=3@1 applied=3 progress={2:3→4 3:3→4}\nn2@1 follower(n1) last=3@1 commit=2@1 applied=2\nn3@1 follower(n1) last=3@1 commit=2@1 applied=2\n\n# n2 now campaigns and wins, while being behind on commit/apply.\ncampaign 2\ndeliver\n---\nn2@1 follower(n1) ⇨ n2@2 candidate\nn2@2 → n1 Campaign last=3@1\nn2@2 → n3 Campaign last=3@1\nn1@1 leader ⇨ n1@2 follower()\nn1@2 → n2 CampaignResponse vote=true\nn3@1 follower(n1) ⇨ n3@2 follower()\nn3@2 → n2 CampaignResponse vote=true\n\n# The initial append doesn't make it to the followers, so its commit index\n# trails the previous leader.\npartition 2\ndeliver 2\n---\nn2 ⇹ n1 n3\nn2@2 candidate ⇨ n2@2 leader\nn2@2 append 4@2 None\nn2@2 ⇥ n1 A̶p̶p̶e̶n̶d̶ ̶b̶a̶s̶e̶=̶3̶@̶1̶ ̶[̶4̶@̶2̶]̶\nn2@2 ⇥ n3 A̶p̶p̶e̶n̶d̶ ̶b̶a̶s̶e̶=̶3̶@̶1̶ ̶[̶4̶@̶2̶]̶\nn2@2 ⇥ n1 H̶e̶a̶r̶t̶b̶e̶a̶t̶ ̶l̶a̶s̶t̶_̶i̶n̶d̶e̶x̶=̶4̶ ̶c̶o̶m̶m̶i̶t̶_̶i̶n̶d̶e̶x̶=̶2̶ ̶r̶e̶a̶d̶_̶s̶e̶q̶=̶0̶\nn2@2 ⇥ n3 H̶e̶a̶r̶t̶b̶e̶a̶t̶ ̶l̶a̶s̶t̶_̶i̶n̶d̶e̶x̶=̶4̶ ̶c̶o̶m̶m̶i̶t̶_̶i̶n̶d̶e̶x̶=̶2̶ ̶r̶e̶a̶d̶_̶s̶e̶q̶=̶0̶\n\nheal\nstatus\n---\nn1 n2 n3 fully connected\nn1@2 follower() last=3@1 commit=3@1 applied=3\nn2@2 leader last=4@2 commit=2@1 applied=2 progress={1:0→5 3:0→5}\nn3@2 follower() last=3@1 commit=2@1 applied=2\n\n# Reading from n2 should not result in a stale read even if followers\n# confirm the read sequence.\nget 2 b\ndeliver\ndeliver\n---\nc2@2 → n2 ClientRequest id=0x03 read 0x000162\nn2@2 → n1 Read seq=1\nn2@2 → n3 Read seq=1\nn1@2 follower() ⇨ n1@2 follower(n2)\nn1@2 → n2 ReadResponse seq=1\nn3@2 follower() ⇨ n3@2 follower(n2)\nn3@2 → n2 ReadResponse seq=1\n\n# The leader heartbeats and detects the lost appends.\nheartbeat 2\ndeliver\ndeliver\ndeliver\n---\nn2@2 → n1 Heartbeat last_index=4 commit_index=2 read_seq=1\nn2@2 → n3 Heartbeat last_index=4 commit_index=2 read_seq=1\nn1@2 → n2 HeartbeatResponse match_index=0 read_seq=1\nn3@2 → n2 HeartbeatResponse match_index=0 read_seq=1\nn2@2 → n1 Append base=3@1 []\nn2@2 → n3 Append base=3@1 []\nn1@2 → n2 AppendResponse match_index=3\nn3@2 → n2 AppendResponse match_index=3\n\n# It resends the missing log entry.\ndeliver\ndeliver\n---\nn2@2 → n1 Append base=3@1 [4@2]\nn2@2 → n3 Append base=3@1 [4@2]\nn1@2 append 4@2 None\nn1@2 → n2 AppendResponse match_index=4\nn3@2 append 4@2 None\nn3@2 → n2 AppendResponse match_index=4\n\n# Once the leader receives the acks it commits the entry. The read can now be\n# served, resulting in an up-to-date b=2.\nstabilize\n---\nn2@2 commit 4@2\nn2@2 apply 3@1 put b=2\nn2@2 apply 4@2 None\nn2@2 → c2 ClientResponse id=0x03 read 0x00010132\nc2@2 get b ⇒ 2\n" }, { "path": "src/raft/testscripts/node/request_leader_disconnect", "content": "# Client read/write requests succeed if the leader is disconnected from the\n# quorum when the request is submitted but it later reconnects.\n\ncluster nodes=3 leader=1\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Partition n1 away from the cluster.\npartition 1\n---\nn1 ⇹ n2 n3\n\n# Submit write and read requests to n1. They don't return a result.\nput 1 foo=bar\nget 1 foo\nstabilize\n---\nc1@1 → n1 ClientRequest id=0x01 write 0x0103666f6f03626172\nn1@1 append 2@1 put foo=bar\nn1@1 ⇥ n2 A̶p̶p̶e̶n̶d̶ ̶b̶a̶s̶e̶=̶1̶@̶1̶ ̶[̶2̶@̶1̶]̶\nn1@1 ⇥ n3 A̶p̶p̶e̶n̶d̶ ̶b̶a̶s̶e̶=̶1̶@̶1̶ ̶[̶2̶@̶1̶]̶\nc1@1 → n1 ClientRequest id=0x02 read 0x0003666f6f\nn1@1 ⇥ n2 R̶e̶a̶d̶ ̶s̶e̶q̶=̶1̶\nn1@1 ⇥ n3 R̶e̶a̶d̶ ̶s̶e̶q̶=̶1̶\n\n# Heal the partition and heartbeat. The requests eventually return results.\nheal\n---\nn1 n2 n3 fully connected\n\nstabilize heartbeat=true\n---\nn1@1 → n2 Heartbeat last_index=2 commit_index=1 read_seq=1\nn1@1 → n3 Heartbeat last_index=2 commit_index=1 read_seq=1\nn2@1 → n1 HeartbeatResponse match_index=0 read_seq=1\nn3@1 → n1 HeartbeatResponse match_index=0 read_seq=1\nn1@1 → c1 ClientResponse id=0x02 read 0x0000\nc1@1 get foo ⇒ None\nn1@1 → n2 Append base=1@1 [2@1]\nn1@1 → n3 Append base=1@1 [2@1]\nn2@1 append 2@1 put foo=bar\nn2@1 → n1 AppendResponse match_index=2\nn3@1 append 2@1 put foo=bar\nn3@1 → n1 AppendResponse match_index=2\nn1@1 commit 2@1\nn1@1 apply 2@1 put foo=bar\nn1@1 → c1 ClientResponse id=0x01 write 0x0102\nc1@1 put foo=bar ⇒ 2\n" }, { "path": "src/raft/testscripts/node/request_leader_read_quorum", "content": "# Client read requests are only processed once a quorum confirms the read sequence.\n\ncluster nodes=5 leader=1\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2 4:1→2 5:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\nn4@1 follower(n1) last=1@1 commit=1@1 applied=1\nn5@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Write foo=bar.\n(put 1 foo=bar)\n(stabilize heartbeat=true)\n---\nok\n\n# Read it once.\n(get 1 foo)\n(stabilize)\n---\nok\n\n# Attempt to read it again. The read only returns once a quorum have\n# confirmed the read sequence.\nget 1 foo\n---\nc1@1 → n1 ClientRequest id=0x03 read 0x0003666f6f\nn1@1 → n2 Read seq=2\nn1@1 → n3 Read seq=2\nn1@1 → n4 Read seq=2\nn1@1 → n5 Read seq=2\n\ndeliver 2\ndeliver 1\n---\nn2@1 → n1 ReadResponse seq=2\n\ndeliver 3\ndeliver 1\n---\nn3@1 → n1 ReadResponse seq=2\nn1@1 → c1 ClientResponse id=0x03 read 0x000103626172\nc1@1 get foo ⇒ bar\n\n(stabilize)\n---\nok\n" }, { "path": "src/raft/testscripts/node/request_leader_read_quorum_sequence", "content": "# Client read requests are only served once a quorum confirm the read sequence\n# number, including higher sequence numbers.\n\ncluster nodes=5 leader=1\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2 4:1→2 5:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\nn4@1 follower(n1) last=1@1 commit=1@1 applied=1\nn5@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Write foo=bar and read it back.\n(put 1 foo=bar)\n(stabilize heartbeat=true)\n(get 1 foo)\n(stabilize)\n---\nok\n\n# Send a heartbeat with sequence number 1, and deliver it to all followers.\nheartbeat 1\ndeliver\n---\nn1@1 → n2 Heartbeat last_index=2 commit_index=2 read_seq=1\nn1@1 → n3 Heartbeat last_index=2 commit_index=2 read_seq=1\nn1@1 → n4 Heartbeat last_index=2 commit_index=2 read_seq=1\nn1@1 → n5 Heartbeat last_index=2 commit_index=2 read_seq=1\nn2@1 → n1 HeartbeatResponse match_index=2 read_seq=1\nn3@1 → n1 HeartbeatResponse match_index=2 read_seq=1\nn4@1 → n1 HeartbeatResponse match_index=2 read_seq=1\nn5@1 → n1 HeartbeatResponse match_index=2 read_seq=1\n\n# Partition n1 away.\npartition 1\n---\nn1 ⇹ n2 n3 n4 n5\n\n# Perform a read at sequence number 2. The read messages are lost.\nget 1 foo\n---\nc1@1 → n1 ClientRequest id=0x03 read 0x0003666f6f\nn1@1 ⇥ n2 R̶e̶a̶d̶ ̶s̶e̶q̶=̶2̶\nn1@1 ⇥ n3 R̶e̶a̶d̶ ̶s̶e̶q̶=̶2̶\nn1@1 ⇥ n4 R̶e̶a̶d̶ ̶s̶e̶q̶=̶2̶\nn1@1 ⇥ n5 R̶e̶a̶d̶ ̶s̶e̶q̶=̶2̶\n\n# Deliver the heartbeat responses at sequence number 1. These should not satisfy\n# the read at sequence number 2.\ndeliver 1\n---\nok\n\n# Heal the partition and perform another read at sequence number 3. Followers\n# respond to the reads at sequence number 3.\nheal\nget 1 foo\n---\nn1 n2 n3 n4 n5 fully connected\nc1@1 → n1 ClientRequest id=0x04 read 0x0003666f6f\nn1@1 → n2 Read seq=3\nn1@1 → n3 Read seq=3\nn1@1 → n4 Read seq=3\nn1@1 → n5 Read seq=3\n\ndeliver\n---\nn2@1 → n1 ReadResponse seq=3\nn3@1 → n1 ReadResponse seq=3\nn4@1 → n1 ReadResponse seq=3\nn5@1 → n1 ReadResponse seq=3\n\n# Once n1 receives two responses it has a read quorum and serves both the read\n# at seqnums 2 (id=0x03) and 3 (id=0x04).\ndeliver 1 from=3\ndeliver 1 from=5\n---\nn1@1 → c1 ClientResponse id=0x03 read 0x000103626172\nc1@1 get foo ⇒ bar\nn1@1 → c1 ClientResponse id=0x04 read 0x000103626172\nc1@1 get foo ⇒ bar\n" }, { "path": "src/raft/testscripts/node/request_leader_single", "content": "# Client read/write requests succeed on a lone leader.\n\ncluster nodes=1\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={}\n\n# An initial get on the leader yields None.\nget 1 foo\nstabilize\n---\nc1@1 → n1 ClientRequest id=0x01 read 0x0003666f6f\nn1@1 → c1 ClientResponse id=0x01 read 0x0000\nc1@1 get foo ⇒ None\n\n# Write a value on the leader.\nput 1 foo=bar\nstabilize heartbeat=true\n---\nc1@1 → n1 ClientRequest id=0x02 write 0x0103666f6f03626172\nn1@1 append 2@1 put foo=bar\nn1@1 commit 2@1\nn1@1 apply 2@1 put foo=bar\nn1@1 → c1 ClientResponse id=0x02 write 0x0102\nc1@1 put foo=bar ⇒ 2\n\n# Read the value back on the leader.\nget 1 foo\nstabilize\n---\nc1@1 → n1 ClientRequest id=0x03 read 0x0003666f6f\nn1@1 → c1 ClientResponse id=0x03 read 0x000103626172\nc1@1 get foo ⇒ bar\n" }, { "path": "src/raft/testscripts/node/request_status", "content": "# Status requests return the cluster status.\n\ncluster nodes=3 leader=1\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Partition away n3, so not all nodes have the same log position.\npartition 3\n---\nn3 ⇹ n1 n2\n\n# Replicate a write, but not the commit index.\n(put 1 foo=bar)\n(stabilize)\nstatus\n---\nn1@1 leader last=2@1 commit=2@1 applied=2 progress={2:2→3 3:1→3}\nn2@1 follower(n1) last=2@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Run a status request on the leader.\nstatus request=true 1\nstabilize\n---\nc1@1 → n1 ClientRequest id=0x02 status\nn1@1 → c1 ClientResponse id=0x02 status Status { leader: 1, term: 1, match_index: {1: 2, 2: 2, 3: 1}, commit_index: 2, applied_index: 2, storage: Status { name: \"bitcask\", keys: 4, size: 41, disk_size: 84, live_disk_size: 73 } }\nc1@1 status ⇒ Status {\n leader: 1,\n term: 1,\n match_index: {\n 1: 2,\n 2: 2,\n 3: 1,\n },\n commit_index: 2,\n applied_index: 2,\n storage: Status {\n name: \"bitcask\",\n keys: 4,\n size: 41,\n disk_size: 84,\n live_disk_size: 73,\n },\n}\n\n# Run a status request on a follower.\nstatus request=true 2\nstabilize\n---\nc2@1 → n2 ClientRequest id=0x03 status\nn2@1 → n1 ClientRequest id=0x03 status\nn1@1 → n2 ClientResponse id=0x03 status Status { leader: 1, term: 1, match_index: {1: 2, 2: 2, 3: 1}, commit_index: 2, applied_index: 2, storage: Status { name: \"bitcask\", keys: 4, size: 41, disk_size: 84, live_disk_size: 73 } }\nn2@1 → c2 ClientResponse id=0x03 status Status { leader: 1, term: 1, match_index: {1: 2, 2: 2, 3: 1}, commit_index: 2, applied_index: 2, storage: Status { name: \"bitcask\", keys: 4, size: 41, disk_size: 84, live_disk_size: 73 } }\nc2@1 status ⇒ Status {\n leader: 1,\n term: 1,\n match_index: {\n 1: 2,\n 2: 2,\n 3: 1,\n },\n commit_index: 2,\n applied_index: 2,\n storage: Status {\n name: \"bitcask\",\n keys: 4,\n size: 41,\n disk_size: 84,\n live_disk_size: 73,\n },\n}\n" }, { "path": "src/raft/testscripts/node/request_status_single", "content": "# Status requests return the cluster status on a single node.\n\ncluster nodes=1\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={}\n\n# Perform a write.\n(put 1 foo=bar)\n(stabilize)\nstatus\n---\nn1@1 leader last=2@1 commit=2@1 applied=2 progress={}\n\n# Run a status request on the leader.\nstatus request=true 1\nstabilize\n---\nc1@1 → n1 ClientRequest id=0x02 status\nn1@1 → c1 ClientResponse id=0x02 status Status { leader: 1, term: 1, match_index: {1: 2}, commit_index: 2, applied_index: 2, storage: Status { name: \"bitcask\", keys: 4, size: 41, disk_size: 84, live_disk_size: 73 } }\nc1@1 status ⇒ Status {\n leader: 1,\n term: 1,\n match_index: {\n 1: 2,\n },\n commit_index: 2,\n applied_index: 2,\n storage: Status {\n name: \"bitcask\",\n keys: 4,\n size: 41,\n disk_size: 84,\n live_disk_size: 73,\n },\n}\n" }, { "path": "src/raft/testscripts/node/restart", "content": "# Restarting a cluster that's fully caught up retains the existing state and\n# allows trivially electing a new leader.\n\ncluster nodes=3 leader=1\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Replicate a couple of writes.\n(put 1 a=1)\n(put 1 b=2)\n(stabilize heartbeat=true)\n---\nok\n\n# Dump the current status, log, and state.\nstatus\n---\nn1@1 leader last=3@1 commit=3@1 applied=3 progress={2:3→4 3:3→4}\nn2@1 follower(n1) last=3@1 commit=3@1 applied=3\nn3@1 follower(n1) last=3@1 commit=3@1 applied=3\n\nlog\n---\nn1@1 term=1 last=3@1 commit=3@1 vote=Some(1)\nn1@1 entry 1@1 None\nn1@1 entry 2@1 put a=1\nn1@1 entry 3@1 put b=2\nn2@1 term=1 last=3@1 commit=3@1 vote=Some(1)\nn2@1 entry 1@1 None\nn2@1 entry 2@1 put a=1\nn2@1 entry 3@1 put b=2\nn3@1 term=1 last=3@1 commit=3@1 vote=Some(1)\nn3@1 entry 1@1 None\nn3@1 entry 2@1 put a=1\nn3@1 entry 3@1 put b=2\n\nstate\n---\nn1@1 applied=3\nn1@1 state a=1\nn1@1 state b=2\nn2@1 applied=3\nn2@1 state a=1\nn2@1 state b=2\nn3@1 applied=3\nn3@1 state a=1\nn3@1 state b=2\n\n# Restart the nodes. They retain the same status, logs, and state.\nrestart\n---\nn1@1 follower() last=3@1 commit=3@1 applied=3\nn2@1 follower() last=3@1 commit=3@1 applied=3\nn3@1 follower() last=3@1 commit=3@1 applied=3\n\nlog\n---\nn1@1 term=1 last=3@1 commit=3@1 vote=Some(1)\nn1@1 entry 1@1 None\nn1@1 entry 2@1 put a=1\nn1@1 entry 3@1 put b=2\nn2@1 term=1 last=3@1 commit=3@1 vote=Some(1)\nn2@1 entry 1@1 None\nn2@1 entry 2@1 put a=1\nn2@1 entry 3@1 put b=2\nn3@1 term=1 last=3@1 commit=3@1 vote=Some(1)\nn3@1 entry 1@1 None\nn3@1 entry 2@1 put a=1\nn3@1 entry 3@1 put b=2\n\nstate\n---\nn1@1 applied=3\nn1@1 state a=1\nn1@1 state b=2\nn2@1 applied=3\nn2@1 state a=1\nn2@1 state b=2\nn3@1 applied=3\nn3@1 state a=1\nn3@1 state b=2\n\n# Elect a new leader.\ncampaign 3\nstabilize heartbeat=true\n---\nn3@1 follower() ⇨ n3@2 candidate\nn3@2 → n1 Campaign last=3@1\nn3@2 → n2 Campaign last=3@1\nn1@1 follower() ⇨ n1@2 follower()\nn1@2 → n3 CampaignResponse vote=true\nn2@1 follower() ⇨ n2@2 follower()\nn2@2 → n3 CampaignResponse vote=true\nn3@2 candidate ⇨ n3@2 leader\nn3@2 append 4@2 None\nn3@2 → n1 Append base=3@1 [4@2]\nn3@2 → n2 Append base=3@1 [4@2]\nn3@2 → n1 Heartbeat last_index=4 commit_index=3 read_seq=0\nn3@2 → n2 Heartbeat last_index=4 commit_index=3 read_seq=0\nn1@2 follower() ⇨ n1@2 follower(n3)\nn1@2 append 4@2 None\nn1@2 → n3 AppendResponse match_index=4\nn1@2 → n3 HeartbeatResponse match_index=4 read_seq=0\nn2@2 follower() ⇨ n2@2 follower(n3)\nn2@2 append 4@2 None\nn2@2 → n3 AppendResponse match_index=4\nn2@2 → n3 HeartbeatResponse match_index=4 read_seq=0\nn3@2 commit 4@2\nn3@2 → n1 Heartbeat last_index=4 commit_index=4 read_seq=0\nn3@2 → n2 Heartbeat last_index=4 commit_index=4 read_seq=0\nn1@2 commit 4@2\nn1@2 → n3 HeartbeatResponse match_index=4 read_seq=0\nn2@2 commit 4@2\nn2@2 → n3 HeartbeatResponse match_index=4 read_seq=0\n\nstatus\n---\nn1@2 follower(n3) last=4@2 commit=4@2 applied=4\nn2@2 follower(n3) last=4@2 commit=4@2 applied=4\nn3@2 leader last=4@2 commit=4@2 applied=4 progress={1:4→5 2:4→5}\n" }, { "path": "src/raft/testscripts/node/restart_apply", "content": "# Restarting a node and wiping its state machine will reapply the state.\n\ncluster nodes=3 leader=1\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Replicate a couple of writes.\n(put 1 a=1)\n(put 1 b=2)\n(stabilize heartbeat=true)\n---\nok\n\n# Restart n3 and clear its state machine. The node will apply all pending\n# entries when restarting.\nrestart 3 applied_index=0\n---\nn3@1 apply 1@1 None\nn3@1 apply 2@1 put a=1\nn3@1 apply 3@1 put b=2\nn3@1 follower() last=3@1 commit=3@1 applied=3\n\nstate 3\n---\nn3@1 applied=3\nn3@1 state a=1\nn3@1 state b=2\n\n# Restart n3 and lose the last write. It will also be reapplied.\nrestart 3 applied_index=2\n---\nn3@1 apply 3@1 put b=2\nn3@1 follower() last=3@1 commit=3@1 applied=3\n\nstate 3\n---\nn3@1 applied=3\nn3@1 state a=1\nn3@1 state b=2\n" }, { "path": "src/raft/testscripts/node/restart_commit_recover", "content": "# Restarting the cluster and wiping the commit indexes allows\n# a new leader to recover the commit index.\n\ncluster nodes=3 leader=1\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Replicate a couple of writes, but don't propagate the commit index.\n(put 1 a=1)\n(put 1 b=2)\n(stabilize)\nstatus\n---\nn1@1 leader last=3@1 commit=3@1 applied=3 progress={2:3→4 3:3→4}\nn2@1 follower(n1) last=3@1 commit=1@1 applied=1\nn3@1 follower(n1) last=3@1 commit=1@1 applied=1\n\n# Restart all nodes and wipe the commit index.\nrestart commit_index=0\n---\nn1@1 follower() last=3@1 commit=0@0 applied=3\nn2@1 follower() last=3@1 commit=0@0 applied=1\nn3@1 follower() last=3@1 commit=0@0 applied=1\n\n# n3 campaigns for leadership and recovers the commit index.\ncampaign 3\nstabilize\n---\nn3@1 follower() ⇨ n3@2 candidate\nn3@2 → n1 Campaign last=3@1\nn3@2 → n2 Campaign last=3@1\nn1@1 follower() ⇨ n1@2 follower()\nn1@2 → n3 CampaignResponse vote=true\nn2@1 follower() ⇨ n2@2 follower()\nn2@2 → n3 CampaignResponse vote=true\nn3@2 candidate ⇨ n3@2 leader\nn3@2 append 4@2 None\nn3@2 → n1 Append base=3@1 [4@2]\nn3@2 → n2 Append base=3@1 [4@2]\nn3@2 → n1 Heartbeat last_index=4 commit_index=0 read_seq=0\nn3@2 → n2 Heartbeat last_index=4 commit_index=0 read_seq=0\nn1@2 follower() ⇨ n1@2 follower(n3)\nn1@2 append 4@2 None\nn1@2 → n3 AppendResponse match_index=4\nn1@2 → n3 HeartbeatResponse match_index=4 read_seq=0\nn2@2 follower() ⇨ n2@2 follower(n3)\nn2@2 append 4@2 None\nn2@2 → n3 AppendResponse match_index=4\nn2@2 → n3 HeartbeatResponse match_index=4 read_seq=0\nn3@2 commit 4@2\n\nstatus\n---\nn1@2 follower(n3) last=4@2 commit=0@0 applied=3\nn2@2 follower(n3) last=4@2 commit=0@0 applied=1\nn3@2 leader last=4@2 commit=4@2 applied=4 progress={1:4→5 2:4→5}\n\n# A heartbeat propagates the commit index.\nheartbeat 3\nstabilize\n---\nn3@2 → n1 Heartbeat last_index=4 commit_index=4 read_seq=0\nn3@2 → n2 Heartbeat last_index=4 commit_index=4 read_seq=0\nn1@2 commit 4@2\nn1@2 → n3 HeartbeatResponse match_index=4 read_seq=0\nn2@2 commit 4@2\nn2@2 → n3 HeartbeatResponse match_index=4 read_seq=0\n" }, { "path": "src/raft/testscripts/node/restart_term_vote", "content": "# The term/vote is retained across a restart.\n\ncluster nodes=3\n---\nn1@0 follower() last=0@0 commit=0@0 applied=0\nn2@0 follower() last=0@0 commit=0@0 applied=0\nn3@0 follower() last=0@0 commit=0@0 applied=0\n\n# Start a new election on n1.\ncampaign 1\n---\nn1@0 follower() ⇨ n1@1 candidate\nn1@1 → n2 Campaign last=0@0\nn1@1 → n3 Campaign last=0@0\n\n# n3 votes for n1, and then restarts.\ndeliver 3\n---\nn3@0 follower() ⇨ n3@1 follower()\nn3@1 → n1 CampaignResponse vote=true\n\nrestart 3\n---\nn3@1 follower() last=0@0 commit=0@0 applied=0\n\n# n3 still has a record of the term and vote in the log.\nlog 3\n---\nn3@1 term=1 last=0@0 commit=0@0 vote=Some(1)\n\n# n2 also campaigns. n3 does not grant its vote.\ncampaign 2\n---\nn2@0 follower() ⇨ n2@1 candidate\nn2@1 → n1 Campaign last=0@0\nn2@1 → n3 Campaign last=0@0\n\ndeliver 3\n---\nn3@1 → n2 CampaignResponse vote=false\n\n# n1 wins leadership.\n(stabilize)\nstatus\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2}\nn2@1 follower(n1) last=1@1 commit=0@0 applied=0\nn3@1 follower(n1) last=1@1 commit=0@0 applied=0\n" }, { "path": "src/raft/testscripts/node/tick_candidate", "content": "# Ticking a candidate will eventually hold a new election in a later term.\n\ncluster nodes=3 heartbeat_interval=1 election_timeout=2\n---\nn1@0 follower() last=0@0 commit=0@0 applied=0\nn2@0 follower() last=0@0 commit=0@0 applied=0\nn3@0 follower() last=0@0 commit=0@0 applied=0\n\n# n1 campaigns.\ncampaign 1\n---\nn1@0 follower() ⇨ n1@1 candidate\nn1@1 → n2 Campaign last=0@0\nn1@1 → n3 Campaign last=0@0\n\n# A single tick does nothing.\ntick 1\n---\nok\n\n# Another tick campaigns in a later term.\ntick 1\n---\nn1@1 candidate ⇨ n1@2 candidate\nn1@2 → n2 Campaign last=0@0\nn1@2 → n3 Campaign last=0@0\n\nstatus\n---\nn1@2 candidate last=0@0 commit=0@0 applied=0\nn2@0 follower() last=0@0 commit=0@0 applied=0\nn3@0 follower() last=0@0 commit=0@0 applied=0\n" }, { "path": "src/raft/testscripts/node/tick_follower", "content": "# Ticking a follower will transition it to candidate if it hasn't\n# heard from the leader in a while.\n\ncluster nodes=3 leader=1 heartbeat_interval=1 election_timeout=2\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# A single follower tick does nothing.\ntick 2\n---\nok\n\n# If n1 heartbeats, the election counter is reset, and another n2 tick does nothing.\nheartbeat 1\nstabilize\n---\nn1@1 → n2 Heartbeat last_index=1 commit_index=1 read_seq=0\nn1@1 → n3 Heartbeat last_index=1 commit_index=1 read_seq=0\nn2@1 → n1 HeartbeatResponse match_index=1 read_seq=0\nn3@1 → n1 HeartbeatResponse match_index=1 read_seq=0\n\ntick 2\n---\nok\n\n# Ticking n2 again exceeds the election timeout, making it campaign.\ntick 2\n---\nn2@1 follower(n1) ⇨ n2@2 candidate\nn2@2 → n1 Campaign last=1@1\nn2@2 → n3 Campaign last=1@1\n\nstatus\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2}\nn2@2 candidate last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\n" }, { "path": "src/raft/testscripts/node/tick_follower_leaderless", "content": "# Ticking a leaderless follower will eventually transition it to candidate.\n\ncluster nodes=3 heartbeat_interval=1 election_timeout=2\n---\nn1@0 follower() last=0@0 commit=0@0 applied=0\nn2@0 follower() last=0@0 commit=0@0 applied=0\nn3@0 follower() last=0@0 commit=0@0 applied=0\n\n# A single follower tick does nothing.\ntick 2\n---\nok\n\n# Another tick makes it campaign.\ntick 2\n---\nn2@0 follower() ⇨ n2@1 candidate\nn2@1 → n1 Campaign last=0@0\nn2@1 → n3 Campaign last=0@0\n\nstatus\n---\nn1@0 follower() last=0@0 commit=0@0 applied=0\nn2@1 candidate last=0@0 commit=0@0 applied=0\nn3@0 follower() last=0@0 commit=0@0 applied=0\n" }, { "path": "src/raft/testscripts/node/tick_leader", "content": "# Ticking a leader should cause it to emit heartbeats, even when it doesn't\n# hear back from any followers.\n\ncluster nodes=3 leader=1 heartbeat_interval=1 election_timeout=2\n---\nn1@1 leader last=1@1 commit=1@1 applied=1 progress={2:1→2 3:1→2}\nn2@1 follower(n1) last=1@1 commit=1@1 applied=1\nn3@1 follower(n1) last=1@1 commit=1@1 applied=1\n\n# Ticking n1 will emit a heartbeat.\ntick 1\n---\nn1@1 → n2 Heartbeat last_index=1 commit_index=1 read_seq=0\nn1@1 → n3 Heartbeat last_index=1 commit_index=1 read_seq=0\n\n# Ticking n1 again will emit further heartbeats, even when it hasn't heard from\n# any followers.\ntick 1\ntick 1\ntick 1\n---\nn1@1 → n2 Heartbeat last_index=1 commit_index=1 read_seq=0\nn1@1 → n3 Heartbeat last_index=1 commit_index=1 read_seq=0\nn1@1 → n2 Heartbeat last_index=1 commit_index=1 read_seq=0\nn1@1 → n3 Heartbeat last_index=1 commit_index=1 read_seq=0\nn1@1 → n2 Heartbeat last_index=1 commit_index=1 read_seq=0\nn1@1 → n3 Heartbeat last_index=1 commit_index=1 read_seq=0\n" }, { "path": "src/server.rs", "content": "use std::collections::HashMap;\nuse std::io::{BufReader, BufWriter, Write as _};\nuse std::net::{TcpListener, TcpStream, ToSocketAddrs};\nuse std::time::Duration;\n\nuse crossbeam::channel::{Receiver, Sender};\nuse log::{debug, error, info};\nuse serde::{Deserialize, Serialize};\nuse uuid::Uuid;\n\nuse crate::encoding::{self, Value as _};\nuse crate::error::Result;\nuse crate::raft;\nuse crate::sql;\nuse crate::sql::engine::{Catalog as _, Engine as _};\nuse crate::sql::execution::StatementResult;\nuse crate::sql::types::{Row, Table};\nuse crate::storage;\n\n/// The outbound Raft peer channel capacity. This buffers messages when a Raft\n/// peer is slow or unavailable. Beyond this, messages will be dropped.\nconst RAFT_PEER_CHANNEL_CAPACITY: usize = 1000;\n\n/// The retry interval when connecting to a Raft peer.\nconst RAFT_PEER_RETRY_INTERVAL: Duration = Duration::from_secs(1);\n\n/// A toyDB server. Routes messages to/from an inner Raft node.\n///\n/// * Listens for inbound SQL connections from clients via TCP and passes\n/// requests to the local Raft node.\n///\n/// * Listens for inbound Raft connections from other toyDB nodes via TCP and\n/// passes messages to the local Raft node.\n///\n/// * Connects to other toyDB nodes via TCP and sends outbound Raft messages\n/// from the local Raft node.\npub struct Server {\n /// The inner Raft node.\n node: raft::Node,\n /// Outbound messages from the Raft node.\n node_rx: Receiver,\n /// Raft peer IDs and addresses.\n peers: HashMap,\n}\n\nimpl Server {\n /// Creates a new toyDB server.\n pub fn new(\n id: raft::NodeID,\n peers: HashMap,\n raft_log: raft::Log,\n raft_state: Box,\n ) -> Result {\n let (node_tx, node_rx) = crossbeam::channel::unbounded();\n let node = raft::Node::new(\n id,\n peers.keys().copied().collect(),\n raft_log,\n raft_state,\n node_tx,\n raft::Options::default(),\n )?;\n Ok(Self { node, peers, node_rx })\n }\n\n /// Serves Raft and SQL requests indefinitely. Consumes the server.\n pub fn serve(self, raft_addr: impl ToSocketAddrs, sql_addr: impl ToSocketAddrs) -> Result<()> {\n let raft_listener = TcpListener::bind(raft_addr)?;\n let sql_listener = TcpListener::bind(sql_addr)?;\n info!(\n \"Listening on {} (SQL) and {} (Raft)\",\n sql_listener.local_addr()?,\n raft_listener.local_addr()?\n );\n\n std::thread::scope(move |s| {\n let id = self.node.id();\n let (raft_request_tx, raft_request_rx) = crossbeam::channel::unbounded();\n let (raft_step_tx, raft_step_rx) = crossbeam::channel::unbounded();\n\n // Serve inbound Raft connections.\n s.spawn(move || Self::raft_accept(raft_listener, raft_step_tx));\n\n // Establish outbound Raft connections to peers.\n let mut raft_peers_tx = HashMap::new();\n for (id, addr) in self.peers.into_iter() {\n let (raft_peer_tx, raft_peer_rx) =\n crossbeam::channel::bounded(RAFT_PEER_CHANNEL_CAPACITY);\n raft_peers_tx.insert(id, raft_peer_tx);\n s.spawn(move || Self::raft_send_peer(addr, raft_peer_rx));\n }\n\n // Route Raft messages between the local node, peers, and clients.\n s.spawn(move || {\n Self::raft_route(\n self.node,\n self.node_rx,\n raft_step_rx,\n raft_peers_tx,\n raft_request_rx,\n )\n });\n\n // Serve inbound SQL connections.\n let sql_engine = sql::engine::Raft::new(raft_request_tx);\n s.spawn(move || Self::sql_accept(id, sql_listener, sql_engine));\n });\n\n Ok(())\n }\n\n /// Accepts new inbound Raft connections from peers and spawns threads\n /// routing inbound messages to the local Raft node.\n fn raft_accept(listener: TcpListener, raft_step_tx: Sender) {\n std::thread::scope(|s| {\n loop {\n let (socket, peer) = match listener.accept() {\n Ok((socket, peer)) => (socket, peer),\n Err(err) => {\n error!(\"Raft peer accept failed: {err}\");\n continue;\n }\n };\n let raft_step_tx = raft_step_tx.clone();\n s.spawn(move || {\n debug!(\"Raft peer {peer} connected\");\n match Self::raft_receive_peer(socket, raft_step_tx) {\n Ok(()) => debug!(\"Raft peer {peer} disconnected\"),\n Err(err) => error!(\"Raft peer {peer} error: {err}\"),\n }\n });\n }\n });\n }\n\n /// Receives inbound messages from a peer via TCP, and queues them for\n /// stepping into the Raft node.\n fn raft_receive_peer(socket: TcpStream, raft_step_tx: Sender) -> Result<()> {\n let mut socket = BufReader::new(socket);\n while let Some(message) = raft::Envelope::maybe_decode_from(&mut socket)? {\n raft_step_tx.send(message)?;\n }\n Ok(())\n }\n\n /// Sends outbound messages to a peer via TCP. Retries indefinitely if the\n /// connection fails.\n fn raft_send_peer(addr: String, raft_node_rx: Receiver) {\n loop {\n let mut socket = match TcpStream::connect(&addr) {\n Ok(socket) => BufWriter::new(socket),\n Err(err) => {\n error!(\"Failed connecting to Raft peer {addr}: {err}\");\n std::thread::sleep(RAFT_PEER_RETRY_INTERVAL);\n continue;\n }\n };\n while let Ok(message) = raft_node_rx.recv() {\n if let Err(err) = message.encode_into(&mut socket).and_then(|_| Ok(socket.flush()?))\n {\n error!(\"Failed sending to Raft peer {addr}: {err}\");\n break;\n }\n }\n debug!(\"Disconnected from Raft peer {addr}\");\n }\n }\n\n /// Routes Raft messages:\n ///\n /// * node_rx: outbound messages from the local Raft node. Routed to peers\n /// via TCP, or to local clients via a response channel.\n ///\n /// * request_rx: inbound requests from local SQL clients. Stepped into\n /// the local Raft node as ClientRequest messages. Responses are returned\n /// via the provided response channel.\n ///\n /// * peers_rx: inbound messages from remote Raft peers. Stepped into the\n /// local Raft node.\n ///\n /// * peers_tx: outbound per-peer channels sent via TCP connections.\n /// Messages from the local node's node_rx are sent here.\n ///\n /// Panics on any errors, since the Raft node can't recover from failed\n /// state transitions.\n fn raft_route(\n mut node: raft::Node,\n node_rx: Receiver,\n peers_rx: Receiver,\n mut peers_tx: HashMap>,\n request_rx: Receiver<(raft::Request, Sender>)>,\n ) {\n // Track response channels by request ID. The Raft node will emit\n // ClientResponse messages that we forward to the response channel.\n let mut response_txs = HashMap::>>::new();\n\n let ticker = crossbeam::channel::tick(raft::TICK_INTERVAL);\n loop {\n crossbeam::select! {\n // Periodically tick the node.\n recv(ticker) -> _ => node = node.tick().expect(\"tick failed\"),\n\n // Step messages from peers into the node.\n recv(peers_rx) -> result => {\n let msg = result.expect(\"peers_rx disconnected\");\n node = node.step(msg).expect(\"step failed\");\n },\n\n // Send outbound messages from the node to the appropriate peer.\n // If we receive a client response addressed to the local node,\n // forward it to the waiting client via the response channel.\n recv(node_rx) -> result => {\n let msg = result.expect(\"node_rx disconnected\");\n if msg.to == node.id()\n && let raft::Message::ClientResponse{ id, response } = msg.message\n {\n if let Some(response_tx) = response_txs.remove(&id) {\n response_tx.send(response).expect(\"response_tx disconnected\");\n }\n continue\n }\n let peer_tx = peers_tx.get_mut(&msg.to).expect(\"unknown peer\");\n match peer_tx.try_send(msg) {\n Ok(()) => {},\n Err(crossbeam::channel::TrySendError::Full(_)) => {\n error!(\"Raft peer channel full, dropping message\");\n },\n Err(crossbeam::channel::TrySendError::Disconnected(_)) => {\n panic!(\"peer_tx disconnected\");\n },\n };\n }\n\n // Track inbound client requests and step them into the node.\n recv(request_rx) -> result => {\n let (request, response_tx) = result.expect(\"request_rx disconnected\");\n let id = Uuid::new_v4();\n let msg = raft::Envelope{\n from: node.id(),\n to: node.id(),\n term: node.term(),\n message: raft::Message::ClientRequest{id, request},\n };\n node = node.step(msg).expect(\"step failed\");\n response_txs.insert(id, response_tx);\n }\n }\n }\n }\n\n /// Accepts new SQL client connections and spawns session threads for them.\n fn sql_accept(id: raft::NodeID, listener: TcpListener, sql_engine: sql::engine::Raft) {\n std::thread::scope(|s| {\n loop {\n let (socket, peer) = match listener.accept() {\n Ok((socket, peer)) => (socket, peer),\n Err(err) => {\n error!(\"Client accept failed: {err}\");\n continue;\n }\n };\n let session = sql_engine.session();\n s.spawn(move || {\n debug!(\"Client {peer} connected\");\n match Self::sql_session(id, socket, session) {\n Ok(()) => debug!(\"Client {peer} disconnected\"),\n Err(err) => error!(\"Client {peer} error: {err}\"),\n }\n });\n }\n })\n }\n\n /// Processes a client SQL session, executing SQL statements against the\n /// Raft node.\n fn sql_session(\n id: raft::NodeID,\n socket: TcpStream,\n mut session: sql::execution::Session,\n ) -> Result<()> {\n let mut reader = BufReader::new(socket.try_clone()?);\n let mut writer = BufWriter::new(socket);\n\n while let Some(request) = Request::maybe_decode_from(&mut reader)? {\n // Execute request.\n debug!(\"Received request {request:?}\");\n let response = match request {\n Request::Execute(query) => session.execute(&query).map(Response::Execute),\n Request::GetTable(table) => {\n session.with_txn(true, |txn| txn.must_get_table(&table)).map(Response::GetTable)\n }\n Request::ListTables => session\n .with_txn(true, |txn| {\n Ok(txn.list_tables()?.into_iter().map(|t| t.name).collect())\n })\n .map(Response::ListTables),\n Request::Status => session\n .status()\n .map(|s| Status { server: id, raft: s.raft, mvcc: s.mvcc })\n .map(Response::Status),\n };\n\n // Process response.\n debug!(\"Returning response {response:?}\");\n response.encode_into(&mut writer)?;\n writer.flush()?;\n }\n Ok(())\n }\n}\n\n/// A SQL client request.\n#[derive(Debug, Serialize, Deserialize)]\npub enum Request {\n /// Executes a SQL statement.\n Execute(String),\n /// Fetches the given table schema.\n GetTable(String),\n /// Lists all tables.\n ListTables,\n /// Returns server status.\n Status,\n}\n\nimpl encoding::Value for Request {}\n\n/// A SQL server response.\n#[derive(Debug, Serialize, Deserialize)]\npub enum Response {\n Execute(StatementResult),\n Row(Option),\n GetTable(Table),\n ListTables(Vec),\n Status(Status),\n}\n\nimpl encoding::Value for Response {}\n\n/// SQL server status.\n#[derive(Debug, PartialEq, Serialize, Deserialize)]\npub struct Status {\n pub server: raft::NodeID,\n pub raft: raft::Status,\n pub mvcc: storage::mvcc::Status,\n}\n" }, { "path": "src/sql/engine/engine.rs", "content": "use std::collections::{BTreeMap, BTreeSet};\n\nuse crate::errinput;\nuse crate::error::Result;\nuse crate::sql::execution::Session;\nuse crate::sql::types::{Expression, Row, Rows, Table, Value};\nuse crate::storage::mvcc;\n\n/// A SQL engine. This provides low-level CRUD (create, read, update, delete)\n/// operations for table rows, a schema catalog for accessing and modifying\n/// table schemas, and interactive SQL sessions that execute client SQL\n/// statements. All engine access is transactional with snapshot isolation.\npub trait Engine<'a>: Sized {\n /// The engine's transaction type. This provides both row-level CRUD operations and\n /// transactional access to the schema catalog.\n type Transaction: Transaction + 'a;\n\n /// Begins a read-write transaction.\n fn begin(&'a self) -> Result;\n /// Begins a read-only transaction.\n fn begin_read_only(&'a self) -> Result;\n /// Begins a read-only transaction as of a historical version.\n fn begin_as_of(&'a self, version: mvcc::Version) -> Result;\n\n /// Creates a client session for executing SQL statements.\n fn session(&'a self) -> Session<'a, Self> {\n Session::new(self)\n }\n}\n\n/// A SQL transaction. Executes transactional CRUD operations on table rows.\n/// Provides snapshot isolation (see `storage::mvcc` module for details).\n///\n/// All methods operate on row batches rather than single rows to amortize the\n/// cost. With the Raft engine, each call results in a Raft roundtrip, and we'd\n/// rather not have to do that for every single row that's modified.\npub trait Transaction: Catalog {\n /// The transaction's internal MVCC state.\n fn state(&self) -> &mvcc::TransactionState;\n\n /// Commits the transaction.\n fn commit(self) -> Result<()>;\n /// Rolls back the transaction.\n fn rollback(self) -> Result<()>;\n\n /// Deletes table rows by primary key, if they exist.\n fn delete(&self, table: &str, ids: &[Value]) -> Result<()>;\n /// Fetches table rows by primary key, if they exist.\n fn get(&self, table: &str, ids: &[Value]) -> Result>;\n /// Inserts new table rows.\n fn insert(&self, table: &str, rows: Vec) -> Result<()>;\n /// Looks up a set of primary keys by index values. BTreeSet for testing.\n fn lookup_index(&self, table: &str, column: &str, values: &[Value]) -> Result>;\n /// Scans a table's rows, optionally applying the given filter.\n fn scan(&self, table: &str, filter: Option) -> Result;\n /// Updates table rows by primary key. BTreeMap for testing.\n fn update(&self, table: &str, rows: BTreeMap) -> Result<()>;\n}\n\n/// The catalog stores table schema information. It must be implemented for\n/// Transaction, and is thus fully transactional. For simplicity, it only\n/// supports creating and dropping tables -- there are no ALTER TABLE schema\n/// changes, nor CREATE INDEX.\npub trait Catalog {\n /// Creates a new table. Errors if it already exists.\n fn create_table(&self, table: Table) -> Result<()>;\n /// Drops a table. Errors if it does not exist, unless if_exists is true.\n /// Returns true if the table existed and was deleted.\n fn drop_table(&self, table: &str, if_exists: bool) -> Result;\n /// Fetches a table schema, or None if it doesn't exist.\n fn get_table(&self, table: &str) -> Result>;\n /// Returns a list of all table schemas.\n fn list_tables(&self) -> Result>;\n\n /// Fetches a table schema, or errors if it does not exist.\n fn must_get_table(&self, table: &str) -> Result
{\n self.get_table(table)?.ok_or_else(|| errinput!(\"table {table} does not exist\"))\n }\n}\n" }, { "path": "src/sql/engine/local.rs", "content": "use std::borrow::Cow;\nuse std::collections::{BTreeMap, BTreeSet};\nuse std::slice;\n\nuse itertools::Itertools as _;\nuse serde::{Deserialize, Serialize};\n\nuse super::Catalog;\nuse crate::encoding::{self, Key as _, Value as _};\nuse crate::errinput;\nuse crate::error::Result;\nuse crate::sql::types::{Expression, Row, Rows, Table, Value};\nuse crate::storage::{self, mvcc};\n\n/// SQL engine keys, using the Keycode order-preserving encoding. For\n/// simplicity, table and column names are used directly as identifiers, instead\n/// of e.g. numeric IDs. It is not possible to change table/column names, so\n/// this is fine, if somewhat inefficient.\n///\n/// Uses Cow to allow encoding borrowed values but decoding owned values.\n#[derive(Debug, Deserialize, Serialize)]\npub enum Key<'a> {\n /// A table schema, keyed by table name. The value is a `sql::types::Table`.\n Table(Cow<'a, str>),\n /// A column index entry, keyed by table name, column name, and index value.\n /// The value is a `BTreeSet` of `sql::types::Value` primary key values.\n Index(Cow<'a, str>, Cow<'a, str>, Cow<'a, Value>),\n /// A table row, keyed by table name and primary key value. The value is a\n /// `sql::types::Row`.\n Row(Cow<'a, str>, Cow<'a, Value>),\n}\n\nimpl<'a> encoding::Key<'a> for Key<'a> {}\n\n/// Key prefixes, allowing prefix scans of specific parts of the keyspace. These\n/// must match the keys -- in particular, the enum variant indexes must match,\n/// since it's part of the encoded key.\n#[derive(Deserialize, Serialize)]\nenum KeyPrefix<'a> {\n /// All table schemas.\n Table,\n /// All column index entries, keyed by table and column name.\n Index(Cow<'a, str>, Cow<'a, str>),\n /// All table rows, keyed by table name.\n Row(Cow<'a, str>),\n}\n\nimpl<'a> encoding::Key<'a> for KeyPrefix<'a> {}\n\n/// A SQL engine using local storage. This provides the main SQL storage logic.\n/// The Raft SQL engine dispatches to this for node-local SQL storage, executing\n/// the same writes across each nodes' instance of `Local`.\npub struct Local {\n /// The local MVCC storage engine.\n pub mvcc: mvcc::MVCC,\n}\n\nimpl Local {\n /// Creates a new local SQL engine using the given storage engine.\n pub fn new(engine: E) -> Self {\n Self { mvcc: mvcc::MVCC::new(engine) }\n }\n\n /// Resumes a transaction from the given state. This is usually kept within\n /// `mvcc::Transaction`, but the Raft-based engine can't retain the MVCC\n /// transaction across requests since it may be executed on different leader\n /// nodes, so it instead keeps the state client-side in the session.\n pub fn resume(&self, state: mvcc::TransactionState) -> Result> {\n Ok(Transaction::new(self.mvcc.resume(state)?))\n }\n\n /// Gets an unversioned key, or None if it doesn't exist.\n pub fn get_unversioned(&self, key: &[u8]) -> Result>> {\n self.mvcc.get_unversioned(key)\n }\n\n /// Sets an unversioned key.\n pub fn set_unversioned(&self, key: &[u8], value: Vec) -> Result<()> {\n self.mvcc.set_unversioned(key, value)\n }\n}\n\nimpl super::Engine<'_> for Local {\n type Transaction = Transaction;\n\n fn begin(&self) -> Result {\n Ok(Self::Transaction::new(self.mvcc.begin()?))\n }\n\n fn begin_read_only(&self) -> Result {\n Ok(Self::Transaction::new(self.mvcc.begin_read_only()?))\n }\n\n fn begin_as_of(&self, version: mvcc::Version) -> Result {\n Ok(Self::Transaction::new(self.mvcc.begin_as_of(version)?))\n }\n}\n\n/// A SQL transaction, wrapping an MVCC transaction.\npub struct Transaction {\n txn: mvcc::Transaction,\n}\n\nimpl Transaction {\n /// Creates a new SQL transaction using the given MVCC transaction.\n fn new(txn: mvcc::Transaction) -> Self {\n Self { txn }\n }\n\n /// Returns the transaction's internal state.\n pub fn state(&self) -> &mvcc::TransactionState {\n self.txn.state()\n }\n\n /// Fetches the matching primary keys for the given secondary index value,\n /// or an empty set if there is none.\n fn get_index(&self, table: &str, column: &str, value: &Value) -> Result> {\n debug_assert!(self.has_index(table, column)?, \"no index on {table}.{column}\");\n Ok(self\n .txn\n .get(&Key::Index(table.into(), column.into(), value.into()).encode())?\n .map(|v| BTreeSet::decode(&v))\n .transpose()?\n .unwrap_or_default())\n }\n\n /// Fetches a single row by primary key, or None if it doesn't exist.\n fn get_row(&self, table: &str, id: &Value) -> Result> {\n self.txn\n .get(&Key::Row(table.into(), id.into()).encode())?\n .map(|v| Row::decode(&v))\n .transpose()\n }\n\n /// Returns true if a secondary index exists for the given column.\n fn has_index(&self, table: &str, column: &str) -> Result {\n let table = self.must_get_table(table)?;\n Ok(table.columns.iter().find(|c| c.name == column).map(|c| c.index).unwrap_or(false))\n }\n\n /// Stores a secondary index entry for the given column value, replacing the\n /// existing entry if any.\n fn set_index(\n &self,\n table: &str,\n column: &str,\n value: &Value,\n ids: BTreeSet,\n ) -> Result<()> {\n debug_assert!(self.has_index(table, column)?, \"no index on {table}.{column}\");\n let key = Key::Index(table.into(), column.into(), value.into()).encode();\n if ids.is_empty() {\n self.txn.delete(&key)?;\n } else {\n self.txn.set(&key, ids.encode())?;\n }\n Ok(())\n }\n\n /// Returns all tables referencing a table, as (table, column index) pairs.\n /// This includes any references from the table itself.\n fn table_references(&self, table: &str) -> Result)>> {\n Ok(self\n .list_tables()?\n .into_iter()\n .map(|t| {\n let references = t\n .columns\n .iter()\n .enumerate()\n .filter(|(_, c)| c.references.as_deref() == Some(table))\n .map(|(i, _)| i)\n .collect_vec();\n (t, references)\n })\n .filter(|(_, references)| !references.is_empty())\n .collect())\n }\n}\n\nimpl super::Transaction for Transaction {\n fn state(&self) -> &mvcc::TransactionState {\n self.txn.state()\n }\n\n fn commit(self) -> Result<()> {\n self.txn.commit()\n }\n\n fn rollback(self) -> Result<()> {\n self.txn.rollback()\n }\n\n fn delete(&self, table: &str, ids: &[Value]) -> Result<()> {\n let table = self.must_get_table(table)?;\n let indexes = table.columns.iter().enumerate().filter(|(_, c)| c.index).collect_vec();\n\n // Check for foreign key references to the deleted rows.\n for (source, refs) in self.table_references(&table.name)? {\n let self_reference = source.name == table.name;\n for i in refs {\n let column = &source.columns[i];\n let mut source_ids = if i == source.primary_key {\n // If the reference is from a primary key column, do a lookup.\n self.get(&source.name, ids)?\n .into_iter()\n .map(|row| row.into_iter().nth(i).expect(\"short row\"))\n .collect()\n } else {\n // Otherwise (commonly), do a secondary index lookup.\n // All foreign keys have a secondary index.\n self.lookup_index(&source.name, &column.name, ids)?\n };\n // We can ignore any references between the deleted rows,\n // including a row referencing itself.\n if self_reference {\n for id in ids {\n source_ids.remove(id);\n }\n }\n // Error if the delete would violate referential integrity.\n if let Some(source_id) = source_ids.first() {\n let table = source.name;\n let column = &source.columns[source.primary_key].name;\n return errinput!(\"row referenced by {table}.{column}={source_id}\");\n }\n }\n }\n\n for id in ids {\n // Update any secondary index entries.\n if !indexes.is_empty()\n && let Some(row) = self.get_row(&table.name, id)?\n {\n for (i, column) in indexes.iter().copied() {\n let mut ids = self.get_index(&table.name, &column.name, &row[i])?;\n ids.remove(id);\n self.set_index(&table.name, &column.name, &row[i], ids)?;\n }\n }\n\n // Delete the row.\n self.txn.delete(&Key::Row((&table.name).into(), id.into()).encode())?;\n }\n Ok(())\n }\n\n fn get(&self, table: &str, ids: &[Value]) -> Result> {\n ids.iter().filter_map(|id| self.get_row(table, id).transpose()).collect()\n }\n\n fn insert(&self, table: &str, rows: Vec) -> Result<()> {\n let table = self.must_get_table(table)?;\n for row in rows {\n // Insert the row.\n table.validate_row(&row, false, self)?;\n let id = &row[table.primary_key];\n self.txn.set(&Key::Row((&table.name).into(), id.into()).encode(), row.encode())?;\n\n // Update any secondary index entries.\n for (i, column) in table.columns.iter().enumerate().filter(|(_, c)| c.index) {\n let mut ids = self.get_index(&table.name, &column.name, &row[i])?;\n ids.insert(id.clone());\n self.set_index(&table.name, &column.name, &row[i], ids)?;\n }\n }\n Ok(())\n }\n\n fn lookup_index(&self, table: &str, column: &str, values: &[Value]) -> Result> {\n debug_assert!(self.has_index(table, column)?, \"no index on {table}.{column}\");\n values.iter().map(|v| self.get_index(table, column, v)).flatten_ok().collect()\n }\n\n fn scan(&self, table: &str, filter: Option) -> Result {\n // TODO: this could be simpler if process_results() implemented Clone.\n let rows = self\n .txn\n .scan_prefix(&KeyPrefix::Row(table.into()).encode())\n .map(|result| result.and_then(|(_, value)| Row::decode(&value)));\n let Some(filter) = filter else {\n return Ok(Box::new(rows));\n };\n let rows = rows.filter_map(move |result| {\n result\n .and_then(|row| match filter.evaluate(Some(&row))? {\n Value::Boolean(true) => Ok(Some(row)),\n Value::Boolean(false) | Value::Null => Ok(None),\n value => errinput!(\"filter returned {value}, expected boolean\"),\n })\n .transpose()\n });\n Ok(Box::new(rows))\n }\n\n fn update(&self, table: &str, rows: BTreeMap) -> Result<()> {\n let table = self.must_get_table(table)?;\n for (id, row) in rows {\n // If the primary key changes, we simply do a delete and insert.\n // This simplifies constraint validation.\n if id != row[table.primary_key] {\n self.delete(&table.name, &[id])?;\n self.insert(&table.name, vec![row])?;\n continue;\n }\n\n // Validate the row, but don't write it yet since we may need to\n // read the existing value to update secondary indexes.\n table.validate_row(&row, true, self)?;\n\n // Update indexes, knowing that the primary key has not changed.\n let indexes = table.columns.iter().enumerate().filter(|(_, c)| c.index).collect_vec();\n if !indexes.is_empty() {\n let old = self.get(&table.name, slice::from_ref(&id))?.remove(0);\n for (i, column) in indexes {\n // If the value didn't change, we don't have to do anything.\n if old[i] == row[i] {\n continue;\n }\n\n // Remove the old value from the index entry.\n let mut ids = self.get_index(&table.name, &column.name, &old[i])?;\n ids.remove(&id);\n self.set_index(&table.name, &column.name, &old[i], ids)?;\n\n // Insert the new value into the index entry.\n let mut ids = self.get_index(&table.name, &column.name, &row[i])?;\n ids.insert(id.clone());\n self.set_index(&table.name, &column.name, &row[i], ids)?;\n }\n }\n\n // Update the row.\n self.txn.set(&Key::Row((&table.name).into(), (&id).into()).encode(), row.encode())?;\n }\n Ok(())\n }\n}\n\nimpl Catalog for Transaction {\n fn create_table(&self, table: Table) -> Result<()> {\n if self.get_table(&table.name)?.is_some() {\n return errinput!(\"table {} already exists\", table.name);\n }\n table.validate(self)?;\n self.txn.set(&Key::Table((&table.name).into()).encode(), table.encode())\n }\n\n fn drop_table(&self, table: &str, if_exists: bool) -> Result {\n let Some(table) = self.get_table(table)? else {\n if if_exists {\n return Ok(false);\n }\n return errinput!(\"table {table} does not exist\");\n };\n\n // Check for foreign key references.\n if let Some((source, refs)) =\n self.table_references(&table.name)?.iter().find(|(t, _)| t.name != table.name)\n {\n return errinput!(\n \"table {} is referenced from {}.{}\",\n table.name,\n source.name,\n source.columns[refs[0]].name\n );\n }\n\n // Delete the table schema entry.\n self.txn.delete(&Key::Table((&table.name).into()).encode())?;\n\n // Delete the table rows.\n let prefix = &KeyPrefix::Row((&table.name).into()).encode();\n let mut keys = self.txn.scan_prefix(prefix).map_ok(|(key, _)| key);\n while let Some(key) = keys.next().transpose()? {\n self.txn.delete(&key)?;\n }\n\n // Delete any secondary index entries.\n for column in table.columns.iter().filter(|c| c.index) {\n let prefix = &KeyPrefix::Index((&table.name).into(), (&column.name).into()).encode();\n let mut keys = self.txn.scan_prefix(prefix).map_ok(|(key, _)| key);\n while let Some(key) = keys.next().transpose()? {\n self.txn.delete(&key)?;\n }\n }\n Ok(true)\n }\n\n fn get_table(&self, table: &str) -> Result> {\n self.txn.get(&Key::Table(table.into()).encode())?.map(|v| Table::decode(&v)).transpose()\n }\n\n fn list_tables(&self) -> Result> {\n self.txn\n .scan_prefix(&KeyPrefix::Table.encode())\n .map(|r| r.and_then(|(_, v)| Table::decode(&v)))\n .collect()\n }\n}\n" }, { "path": "src/sql/engine/mod.rs", "content": "//! The SQL engine provides SQL data storage and access, as well as session and\n//! transaction management. The `Local` engine provides node-local on-disk\n//! storage, while the `Raft` engine submits commands through Raft consensus\n//! before dispatching to the `Local` engine on each node.\n\nmod engine;\nmod local;\nmod raft;\n\npub use engine::{Catalog, Engine, Transaction};\npub use local::{Key, Local};\npub use raft::{Raft, Status, Write};\n" }, { "path": "src/sql/engine/raft.rs", "content": "use std::borrow::Cow;\nuse std::collections::{BTreeMap, BTreeSet};\n\nuse crossbeam::channel::Sender;\nuse serde::de::DeserializeOwned;\nuse serde::{Deserialize, Serialize};\n\nuse super::{Catalog, Engine as _, Transaction as _};\nuse crate::encoding::{self, Value as _, bincode};\nuse crate::errdata;\nuse crate::error::Result;\nuse crate::raft;\nuse crate::sql::types::{Expression, Row, Rows, Table, Value};\nuse crate::storage::{self, mvcc};\n\n/// A read command, submitted via Raft and executed on the leader. Each command\n/// corresponds to a SQL engine method and parameters. Uses Cows to allow\n/// borrowed encoding and owned decoding.\n#[derive(Debug, Serialize, Deserialize)]\npub enum Read<'a> {\n BeginReadOnly {\n as_of: Option,\n },\n Status,\n\n Get {\n txn: Cow<'a, mvcc::TransactionState>,\n table: Cow<'a, str>,\n ids: Cow<'a, [Value]>,\n },\n LookupIndex {\n txn: Cow<'a, mvcc::TransactionState>,\n table: Cow<'a, str>,\n column: Cow<'a, str>,\n values: Cow<'a, [Value]>,\n },\n Scan {\n txn: Cow<'a, mvcc::TransactionState>,\n table: Cow<'a, str>,\n filter: Option,\n },\n\n GetTable {\n txn: Cow<'a, mvcc::TransactionState>,\n table: Cow<'a, str>,\n },\n ListTables {\n txn: Cow<'a, mvcc::TransactionState>,\n },\n}\n\nimpl encoding::Value for Read<'_> {}\n\n/// A write command, submitted via Raft and executed on all nodes. Each command\n/// corresponds to a SQL engine method and parameters. Uses Cows to allow\n/// borrowed encoding and owned decoding.\n#[derive(Debug, Serialize, Deserialize)]\npub enum Write<'a> {\n Begin,\n Commit(Cow<'a, mvcc::TransactionState>),\n Rollback(Cow<'a, mvcc::TransactionState>),\n\n Delete { txn: Cow<'a, mvcc::TransactionState>, table: Cow<'a, str>, ids: Cow<'a, [Value]> },\n Insert { txn: Cow<'a, mvcc::TransactionState>, table: Cow<'a, str>, rows: Vec },\n Update { txn: Cow<'a, mvcc::TransactionState>, table: Cow<'a, str>, rows: BTreeMap },\n\n CreateTable { txn: Cow<'a, mvcc::TransactionState>, schema: Table },\n DropTable { txn: Cow<'a, mvcc::TransactionState>, table: Cow<'a, str>, if_exists: bool },\n}\n\nimpl encoding::Value for Write<'_> {}\n\n/// Raft SQL engine status.\n#[derive(Serialize, Deserialize)]\npub struct Status {\n pub raft: raft::Status,\n pub mvcc: mvcc::Status,\n}\n\n/// A Raft-based SQL engine. This dispatches to the `Local` engine for local\n/// storage and processing on each node, but sends read and write commands\n/// through Raft for distributed consensus.\n///\n/// The `Raft` engine itself is simply a Raft client which sends `raft::Request`\n/// to the local Raft node for processing. These requests are applied to the\n/// Raft SQL engine's `State` state machine running below Raft on each node,\n/// which executes the commands on a `Local` SQL engine using a\n/// `storage::Engine` for local storage.\n///\n/// For more details on how SQL statements flow through the engine, see the\n/// `sql` module documentation.\npub struct Raft {\n /// Sends requests to the local Raft node, along with a response channel.\n tx: Sender<(raft::Request, Sender>)>,\n}\n\nimpl Raft {\n /// The unversioned key used to store the applied index. Just uses a string\n /// for simplicity.\n pub const APPLIED_INDEX_KEY: &'static [u8] = b\"applied_index\";\n\n /// Creates a new Raft-based SQL engine, with a channel to send requests to\n /// the local Raft node.\n pub fn new(tx: Sender<(raft::Request, Sender>)>) -> Self {\n Self { tx }\n }\n\n /// Creates the Raft-managed state machine for the Raft engine. Receives\n /// commands from the Raft engine and executes them on a `Local` engine.\n pub fn new_state(engine: E) -> Result> {\n State::new(engine)\n }\n\n /// Executes a request against the Raft cluster, waiting for the response.\n fn request(&self, request: raft::Request) -> Result {\n let (response_tx, response_rx) = crossbeam::channel::bounded(1);\n self.tx.send((request, response_tx))?;\n response_rx.recv()?\n }\n\n /// Writes through Raft, deserializing the response into the return type.\n fn write(&self, write: Write) -> Result {\n match self.request(raft::Request::Write(write.encode()))? {\n raft::Response::Write(response) => bincode::deserialize(&response),\n response => errdata!(\"unexpected Raft write response {response:?}\"),\n }\n }\n\n /// Reads from Raft, deserializing the response into the return type.\n fn read(&self, read: Read) -> Result {\n match self.request(raft::Request::Read(read.encode()))? {\n raft::Response::Read(response) => bincode::deserialize(&response),\n response => errdata!(\"unexpected Raft read response {response:?}\"),\n }\n }\n\n /// Raft SQL engine status.\n pub fn status(&self) -> Result {\n let raft = match self.request(raft::Request::Status)? {\n raft::Response::Status(status) => status,\n response => return errdata!(\"unexpected Raft status response {response:?}\"),\n };\n let mvcc = self.read(Read::Status)?;\n Ok(Status { raft, mvcc })\n }\n}\n\nimpl<'a> super::Engine<'a> for Raft {\n type Transaction = Transaction<'a>;\n\n fn begin(&'a self) -> Result {\n Transaction::begin(self, false, None)\n }\n\n fn begin_read_only(&'a self) -> Result {\n Transaction::begin(self, true, None)\n }\n\n fn begin_as_of(&'a self, version: mvcc::Version) -> Result {\n Transaction::begin(self, true, Some(version))\n }\n}\n\n/// A Raft SQL engine transaction.\n///\n/// This keeps track of the transaction state in memory. An `mvcc::Transaction`\n/// normally manages this, but since `mvcc::Transaction` runs below Raft, it\n/// can't maintain this state between individual requests (which could execute\n/// on different leaders). Instead, it uses `mvcc::Transaction::resume` to\n/// resume the transaction from the provided transaction state for each request.\npub struct Transaction<'a> {\n /// The Raft SQL engine client, used to communicate with Raft.\n raft: &'a Raft,\n /// The MVCC transaction state.\n state: mvcc::TransactionState,\n}\n\nimpl<'a> Transaction<'a> {\n /// Starts a transaction in the given mode.\n fn begin(raft: &'a Raft, read_only: bool, as_of: Option) -> Result {\n assert!(as_of.is_none() || read_only, \"can't use as_of without read_only\");\n // Read-only transactions don't allocate a new MVCC version, so they\n // don't write anything -- they just grab the current transaction state.\n // Submit them as reads to avoid a replication roundtrip.\n let state = if read_only || as_of.is_some() {\n raft.read(Read::BeginReadOnly { as_of })?\n } else {\n raft.write(Write::Begin)?\n };\n Ok(Self { raft, state })\n }\n}\n\nimpl super::Transaction for Transaction<'_> {\n fn state(&self) -> &mvcc::TransactionState {\n &self.state\n }\n\n fn commit(self) -> Result<()> {\n if self.state.read_only {\n return Ok(()); // noop\n }\n self.raft.write(Write::Commit(self.state.into()))\n }\n\n fn rollback(self) -> Result<()> {\n if self.state.read_only {\n return Ok(()); // noop\n }\n self.raft.write(Write::Rollback(self.state.into()))\n }\n\n fn delete(&self, table: &str, ids: &[Value]) -> Result<()> {\n self.raft.write(Write::Delete {\n txn: (&self.state).into(),\n table: table.into(),\n ids: ids.into(),\n })\n }\n\n fn get(&self, table: &str, ids: &[Value]) -> Result> {\n self.raft.read(Read::Get {\n txn: (&self.state).into(),\n table: table.into(),\n ids: ids.into(),\n })\n }\n\n fn insert(&self, table: &str, rows: Vec) -> Result<()> {\n self.raft.write(Write::Insert { txn: (&self.state).into(), table: table.into(), rows })\n }\n\n fn lookup_index(&self, table: &str, column: &str, values: &[Value]) -> Result> {\n self.raft.read(Read::LookupIndex {\n txn: (&self.state).into(),\n table: table.into(),\n column: column.into(),\n values: values.into(),\n })\n }\n\n fn scan(&self, table: &str, filter: Option) -> Result {\n let scan: Vec = self.raft.read(Read::Scan {\n txn: (&self.state).into(),\n table: table.into(),\n filter,\n })?;\n Ok(Box::new(scan.into_iter().map(Ok)))\n }\n\n fn update(&self, table: &str, rows: BTreeMap) -> Result<()> {\n self.raft.write(Write::Update { txn: (&self.state).into(), table: table.into(), rows })\n }\n}\n\nimpl Catalog for Transaction<'_> {\n fn create_table(&self, schema: Table) -> Result<()> {\n self.raft.write(Write::CreateTable { txn: (&self.state).into(), schema })\n }\n\n fn drop_table(&self, table: &str, if_exists: bool) -> Result {\n self.raft.write(Write::DropTable {\n txn: (&self.state).into(),\n table: table.into(),\n if_exists,\n })\n }\n\n fn get_table(&self, table: &str) -> Result> {\n self.raft.read(Read::GetTable { txn: (&self.state).into(), table: table.into() })\n }\n\n fn list_tables(&self) -> Result> {\n self.raft.read(Read::ListTables { txn: (&self.state).into() })\n }\n}\n\n/// The state machine for the Raft SQL engine. Receives commands via Raft and\n/// dispatches to a `Local` SQL engine which does the actual work, using a\n/// `storage::Engine` for storage.\n///\n/// For simplicity, we don't attempt to stream large requests or responses,\n/// instead just delivering them as one large chunk. This means that e.g. a full\n/// table scan will pull the entire table into memory, serialize it, and send it\n/// across the network as one message, but that's fine for toyDB.\npub struct State {\n /// The local SQL engine, used for actual storage.\n local: super::Local,\n /// The last applied index. This tells Raft which command to apply next.\n applied_index: raft::Index,\n}\n\nimpl State {\n /// Creates a new Raft state maching using the given storage engine for\n /// local storage.\n pub fn new(engine: E) -> Result {\n let local = super::Local::new(engine);\n let applied_index = local\n .get_unversioned(Raft::APPLIED_INDEX_KEY)?\n .map(|b| bincode::deserialize(&b))\n .transpose()?\n .unwrap_or_default();\n Ok(State { local, applied_index })\n }\n\n /// Executes a write command. This is executed on all nodes, but the\n /// response is returned from the Raft leader.\n ///\n /// The response is encoded using Bincode. The caller will know what\n /// response type to expect for each command and deserialize into it.\n fn write(&self, command: Write) -> Result> {\n Ok(match command {\n Write::Begin => self.local.begin()?.state().encode(),\n Write::Commit(txn) => {\n bincode::serialize(&self.local.resume(txn.into_owned())?.commit()?)\n }\n Write::Rollback(txn) => {\n bincode::serialize(&self.local.resume(txn.into_owned())?.rollback()?)\n }\n\n Write::Delete { txn, table, ids } => {\n bincode::serialize(&self.local.resume(txn.into_owned())?.delete(&table, &ids)?)\n }\n Write::Insert { txn, table, rows } => {\n bincode::serialize(&self.local.resume(txn.into_owned())?.insert(&table, rows)?)\n }\n Write::Update { txn, table, rows } => {\n bincode::serialize(&self.local.resume(txn.into_owned())?.update(&table, rows)?)\n }\n\n Write::CreateTable { txn, schema } => {\n bincode::serialize(&self.local.resume(txn.into_owned())?.create_table(schema)?)\n }\n Write::DropTable { txn, table, if_exists } => bincode::serialize(\n &self.local.resume(txn.into_owned())?.drop_table(&table, if_exists)?,\n ),\n })\n }\n}\n\nimpl raft::State for State {\n fn get_applied_index(&self) -> raft::Index {\n self.applied_index\n }\n\n fn apply(&mut self, entry: raft::Entry) -> Result> {\n assert_eq!(entry.index, self.applied_index + 1, \"entry index not after applied index\");\n\n let result = match &entry.command {\n Some(command) => match self.write(Write::decode(command)?) {\n // Panic on non-deterministic apply failures, to prevent node\n // state divergence. See `raft::State` docs for details.\n Err(e) if !e.is_deterministic() => panic!(\"non-deterministic apply failure: {e}\"),\n result => result,\n },\n // Raft submits noop commands on leader changes. Ignore them, but\n // record the applied index below.\n None => Ok(Vec::new()),\n };\n\n // Persist the applied index. We don't have to flush, because it's ok to\n // lose a tail of the state machine writes (e.g. if the machine\n // crashes). Raft will replay the log from the last known applied index.\n self.applied_index = entry.index;\n self.local.set_unversioned(Raft::APPLIED_INDEX_KEY, bincode::serialize(&entry.index))?;\n result\n }\n\n fn read(&self, command: Vec) -> Result> {\n Ok(match Read::decode(&command)? {\n Read::BeginReadOnly { as_of } => {\n let txn = match as_of {\n Some(version) => self.local.begin_as_of(version)?,\n None => self.local.begin_read_only()?,\n };\n txn.state().encode()\n }\n Read::Status => self.local.mvcc.status()?.encode(),\n\n Read::Get { txn, table, ids } => {\n self.local.resume(txn.into_owned())?.get(&table, &ids)?.encode()\n }\n Read::LookupIndex { txn, table, column, values } => self\n .local\n .resume(txn.into_owned())?\n .lookup_index(&table, &column, &values)?\n .encode(),\n Read::Scan { txn, table, filter } => {\n // For simplicity, buffer the entire scan. See `State` comment.\n self.local\n .resume(txn.into_owned())?\n .scan(&table, filter)?\n .collect::>>()?\n .encode()\n }\n\n Read::GetTable { txn, table } => {\n self.local.resume(txn.into_owned())?.get_table(&table)?.encode()\n }\n Read::ListTables { txn } => {\n self.local.resume(txn.into_owned())?.list_tables()?.encode()\n }\n })\n }\n}\n" }, { "path": "src/sql/execution/aggregator.rs", "content": "use std::collections::BTreeMap;\n\nuse itertools::Itertools as _;\n\nuse crate::error::Result;\nuse crate::sql::planner::Aggregate;\nuse crate::sql::types::{Expression, Row, Rows, Value};\n\n/// Computes bucketed aggregates for input rows. For example, this query would\n/// compute COUNT and SUM aggregates bucketed by category and brand:\n///\n/// SELECT COUNT(*), SUM(price) FROM products GROUP BY category, brand\npub struct Aggregator {\n /// GROUP BY expressions.\n group_by: Vec,\n /// Aggregates to compute.\n aggregates: Vec,\n /// Accumulators indexed by group_by bucket.\n buckets: BTreeMap, Vec>,\n}\n\nimpl Aggregator {\n /// Creates a new aggregator for the given GROUP BY buckets and aggregates.\n pub fn new(group_by: Vec, aggregates: Vec) -> Self {\n Self { group_by, aggregates, buckets: BTreeMap::new() }\n }\n\n /// Adds a row to the aggregator.\n pub fn add(&mut self, row: &Row) -> Result<()> {\n // Compute the bucket values.\n let bucket = self.group_by.iter().map(|expr| expr.evaluate(Some(row))).try_collect()?;\n\n // Look up the bucket accumulators, or create a new bucket.\n let accumulators = self\n .buckets\n .entry(bucket)\n .or_insert_with(|| self.aggregates.iter().map(Accumulator::new).collect())\n .iter_mut();\n\n // Collect expressions to evaluate.\n let exprs = self.aggregates.iter().map(|a| a.expr());\n\n // Accumulate the evaluated values.\n for (accumulator, expr) in accumulators.zip_eq(exprs) {\n accumulator.add(expr.evaluate(Some(row))?)?;\n }\n Ok(())\n }\n\n /// Adds rows to the aggregator.\n pub fn add_rows(&mut self, rows: Rows) -> Result<()> {\n for row in rows {\n self.add(&row?)?;\n }\n Ok(())\n }\n\n /// Returns a row iterator over the aggregate result.\n pub fn into_rows(self) -> Rows {\n // If there were no rows and no group_by expressions, return a row of\n // empty accumulators (e.g. SELECT COUNT(*) FROM t WHERE FALSE).\n if self.buckets.is_empty() && self.group_by.is_empty() {\n let result =\n self.aggregates.iter().map(Accumulator::new).map(|acc| acc.value()).try_collect();\n return Box::new(std::iter::once(result));\n }\n\n // Emit the group_by and aggregate values for each bucket. We use an\n // intermediate vec since btree_map::IntoIter doesn't implement Clone\n // (required by Rows).\n let buckets = self.buckets.into_iter().collect_vec();\n Box::new(buckets.into_iter().map(|(bucket, accumulators)| {\n bucket\n .into_iter()\n .map(Ok)\n .chain(accumulators.into_iter().map(|acc| acc.value()))\n .collect()\n }))\n }\n}\n\n/// Accumulates aggregate values. Uses an enum rather than a trait since we need\n/// to keep these in a vector (could use boxed trait objects too).\n#[derive(Clone)]\nenum Accumulator {\n Average { count: i64, sum: Value },\n Count(i64),\n Max(Option),\n Min(Option),\n Sum(Option),\n}\n\nimpl Accumulator {\n /// Creates a new accumulator from an aggregate kind.\n fn new(aggregate: &Aggregate) -> Self {\n match aggregate {\n Aggregate::Average(_) => Self::Average { count: 0, sum: Value::Integer(0) },\n Aggregate::Count(_) => Self::Count(0),\n Aggregate::Max(_) => Self::Max(None),\n Aggregate::Min(_) => Self::Min(None),\n Aggregate::Sum(_) => Self::Sum(None),\n }\n }\n\n /// Adds a value to the accumulator.\n fn add(&mut self, value: Value) -> Result<()> {\n // Aggregates ignore NULL values.\n if value == Value::Null {\n return Ok(());\n }\n match self {\n Self::Average { sum, count } => (*sum, *count) = (sum.checked_add(&value)?, *count + 1),\n Self::Count(count) => *count += 1,\n Self::Max(max @ None) => *max = Some(value),\n Self::Max(Some(max)) if value > *max => *max = value,\n Self::Max(Some(_)) => {}\n Self::Min(min @ None) => *min = Some(value),\n Self::Min(Some(min)) if value < *min => *min = value,\n Self::Min(Some(_)) => {}\n Self::Sum(sum @ None) => *sum = Some(Value::Integer(0).checked_add(&value)?),\n Self::Sum(Some(sum)) => *sum = sum.checked_add(&value)?,\n }\n Ok(())\n }\n\n /// Returns the aggregate value.\n fn value(self) -> Result {\n Ok(match self {\n Self::Average { count: 0, sum: _ } => Value::Null,\n Self::Average { count, sum } => sum.checked_div(&Value::Integer(count))?,\n Self::Count(count) => count.into(),\n Self::Max(Some(value)) | Self::Min(Some(value)) | Self::Sum(Some(value)) => value,\n Self::Max(None) | Self::Min(None) | Self::Sum(None) => Value::Null,\n })\n }\n}\n" }, { "path": "src/sql/execution/executor.rs", "content": "use std::cmp::Ordering;\nuse std::collections::{BTreeMap, HashMap};\n\nuse itertools::{Itertools as _, izip};\n\nuse super::aggregator::Aggregator;\nuse super::join::{HashJoiner, NestedLoopJoiner};\nuse crate::errinput;\nuse crate::error::Result;\nuse crate::sql::engine::Transaction;\nuse crate::sql::planner::{Direction, Node, Plan};\nuse crate::sql::types::{Expression, Label, Row, Rows, Table, Value};\n\n/// Executes statement plans.\n///\n/// The plan root specifies the action to take (e.g. SELECT, INSERT, UPDATE,\n/// etc). It has a nested tree of child nodes that process rows.\n///\n/// Nodes are executed recursively, and return row iterators. Parent nodes\n/// recursively pull input rows from their child nodes, process them, and pass\n/// them on to their parent node.\n///\n/// Below is an example of an (unoptimized) query plan:\n///\n/// SELECT title, released, genres.name AS genre\n/// FROM movies INNER JOIN genres ON movies.genre_id = genres.id\n/// WHERE released >= 2000\n/// ORDER BY released\n///\n/// Select\n/// └─ Order: movies.released desc\n/// └─ Projection: movies.title, movies.released, genres.name as genre\n/// └─ Filter: movies.released >= 2000\n/// └─ NestedLoopJoin: inner on movies.genre_id = genres.id\n/// ├─ Scan: movies\n/// └─ Scan: genres\n///\n/// Rows flow from the tree leaves to the root:\n///\n/// 1. Scan nodes read rows from movies and genres.\n/// 2. NestedLoopJoin joins the rows from movies and genres.\n/// 3. Filter discards rows with release dates older than 2000.\n/// 4. Projection picks out the requested column values from the rows.\n/// 5. Order sorts the rows by release date.\n/// 6. Select returns the final rows to the client.\npub struct Executor<'a, T: Transaction> {\n /// The transaction used to execute the plan.\n txn: &'a T,\n}\n\nimpl<'a, T: Transaction> Executor<'a, T> {\n /// Creates a new executor.\n pub fn new(txn: &'a T) -> Self {\n Self { txn }\n }\n\n /// Executes a plan, returning an execution result.\n pub fn execute(&mut self, plan: Plan) -> Result {\n Ok(match plan {\n // CREATE TABLE\n Plan::CreateTable { schema } => {\n let name = schema.name.clone();\n self.txn.create_table(schema)?;\n ExecutionResult::CreateTable { name }\n }\n\n // DROP TABLE\n Plan::DropTable { name, if_exists } => {\n let existed = self.txn.drop_table(&name, if_exists)?;\n ExecutionResult::DropTable { name, existed }\n }\n\n // DELETE\n Plan::Delete { table, primary_key, source } => {\n let source = self.execute_node(source)?;\n let count = self.delete(&table, primary_key, source)?;\n ExecutionResult::Delete { count }\n }\n\n // INSERT\n Plan::Insert { table, column_map, source } => {\n let source = self.execute_node(source)?;\n let count = self.insert(table, column_map, source)?;\n ExecutionResult::Insert { count }\n }\n\n // SELECT\n Plan::Select(root) => {\n let columns = (0..root.columns()).map(|i| root.column_label(i)).collect();\n let rows = self.execute_node(root)?;\n ExecutionResult::Select { columns, rows }\n }\n\n // UPDATE\n Plan::Update { table, primary_key, source, expressions } => {\n let source = self.execute_node(source)?;\n let count = self.update(&table.name, primary_key, source, expressions)?;\n ExecutionResult::Update { count }\n }\n })\n }\n\n /// Recursively executes a query plan node, returning a row iterator.\n fn execute_node(&mut self, node: Node) -> Result {\n Ok(match node {\n // GROUP BY and aggregate functions.\n Node::Aggregate { source, group_by, aggregates } => {\n let source = self.execute_node(*source)?;\n let mut aggregator = Aggregator::new(group_by, aggregates);\n aggregator.add_rows(source)?;\n aggregator.into_rows()\n }\n\n // WHERE and similar filtering.\n Node::Filter { source, predicate } => {\n let source = self.execute_node(*source)?;\n Box::new(source.filter_map(move |result| {\n result\n .and_then(|row| match predicate.evaluate(Some(&row))? {\n Value::Boolean(true) => Ok(Some(row)),\n Value::Boolean(false) | Value::Null => Ok(None),\n value => errinput!(\"filter returned {value}, expected boolean\",),\n })\n .transpose()\n }))\n }\n\n // JOIN using a hash join.\n Node::HashJoin { left, left_column, right, right_column, outer } => {\n let right_columns = right.columns();\n let left = self.execute_node(*left)?;\n let right = self.execute_node(*right)?;\n Box::new(HashJoiner::new(\n left,\n left_column,\n right,\n right_column,\n right_columns,\n outer,\n )?)\n }\n\n // Looks up primary keys by secondary index values.\n Node::IndexLookup { table, column, values, alias: _ } => {\n let column = table.columns.into_iter().nth(column).expect(\"invalid column\").name;\n let ids =\n self.txn.lookup_index(&table.name, &column, &values)?.into_iter().collect_vec();\n Box::new(self.txn.get(&table.name, &ids)?.into_iter().map(Ok))\n }\n\n // Looks up rows by primary key.\n Node::KeyLookup { table, keys, alias: _ } => {\n Box::new(self.txn.get(&table.name, &keys)?.into_iter().map(Ok))\n }\n\n // LIMIT\n Node::Limit { source, limit } => Box::new(self.execute_node(*source)?.take(limit)),\n\n // JOIN using a nested loop join.\n Node::NestedLoopJoin { left, right, predicate, outer } => {\n let right_columns = right.columns();\n let left = self.execute_node(*left)?;\n let right = self.execute_node(*right)?;\n Box::new(NestedLoopJoiner::new(left, right, right_columns, predicate, outer))\n }\n\n // An empty row iterator.\n Node::Nothing { .. } => Box::new(std::iter::empty()),\n\n // OFFSET\n Node::Offset { source, offset } => Box::new(self.execute_node(*source)?.skip(offset)),\n\n // ORDER BY\n Node::Order { source, key } => {\n let source = self.execute_node(*source)?;\n Box::new(Self::order(source, key)?)\n }\n\n // Projects columns from the source, and evaluates expressions.\n Node::Projection { source, expressions, aliases: _ } => {\n let source = self.execute_node(*source)?;\n Box::new(source.map(move |result| {\n let row = result?;\n expressions.iter().map(|expr| expr.evaluate(Some(&row))).collect()\n }))\n }\n\n // Remaps source column indexes to new target column indexes.\n Node::Remap { source, targets } => {\n let source = self.execute_node(*source)?;\n let size = targets.iter().copied().flatten().map(|i| i + 1).max().unwrap_or(0);\n Box::new(source.map_ok(move |row| {\n let mut remapped = vec![Value::Null; size];\n for (target, value) in targets.iter().copied().zip_eq(row) {\n if let Some(target) = target {\n remapped[target] = value;\n }\n }\n remapped\n }))\n }\n\n // Scans a table, optionally filtering rows.\n Node::Scan { table, filter, alias: _ } => Box::new(self.txn.scan(&table.name, filter)?),\n\n // Emits constant values.\n Node::Values { rows } => Box::new(\n rows.into_iter()\n .map(|row| row.into_iter().map(|expr| expr.evaluate(None)).collect()),\n ),\n })\n }\n\n /// DELETE: deletes rows, taking primary keys from the source at the given\n /// primary_key column index. Returns the number of rows deleted.\n fn delete(&self, table: &str, primary_key: usize, source: Rows) -> Result {\n let ids: Vec = source\n .map_ok(|row| row.into_iter().nth(primary_key).expect(\"short row\"))\n .try_collect()?;\n let count = ids.len() as u64;\n self.txn.delete(table, &ids)?;\n Ok(count)\n }\n\n /// INSERT: inserts rows into a table from the given source.\n ///\n /// If given, column_map contains the mapping of table → source columns for\n /// all columns in source. Otherwise, every column in source corresponds to\n /// those in table, but a tail of source columns may be missing.\n fn insert(\n &self,\n table: Table,\n column_map: Option>,\n mut source: Rows,\n ) -> Result {\n let mut rows = Vec::new();\n while let Some(values) = source.next().transpose()? {\n // Fast path: the row is already complete, with no column mapping.\n if values.len() == table.columns.len() && column_map.is_none() {\n rows.push(values);\n continue;\n }\n if values.len() > table.columns.len() {\n return errinput!(\"too many values for table {}\", table.name);\n }\n if let Some(column_map) = &column_map\n && column_map.len() != values.len()\n {\n return errinput!(\"column and value counts do not match\");\n }\n\n // Map source columns to table columns, and fill in default values.\n let mut row = Vec::with_capacity(table.columns.len());\n for (i, column) in table.columns.iter().enumerate() {\n if column_map.is_none() && i < values.len() {\n // Pass through the source column to the table column.\n row.push(values[i].clone())\n } else if let Some(vi) = column_map.as_ref().and_then(|c| c.get(&i)).copied() {\n // Map the source column to the table column.\n row.push(values[vi].clone())\n } else if let Some(default) = &column.default {\n // Column not given in source, use the default.\n row.push(default.clone())\n } else {\n return errinput!(\"no value given for column {} with no default\", column.name);\n }\n }\n rows.push(row);\n }\n let count = rows.len() as u64;\n self.txn.insert(&table.name, rows)?;\n Ok(count)\n }\n\n /// UPDATE: updates rows passed in from the source. Returns the number of\n /// rows updated.\n fn update(\n &self,\n table: &str,\n primary_key: usize,\n mut source: Rows,\n expressions: Vec<(usize, Expression)>,\n ) -> Result {\n let mut updates = BTreeMap::new();\n while let Some(row) = source.next().transpose()? {\n let mut update = row.clone();\n for (column, expr) in &expressions {\n update[*column] = expr.evaluate(Some(&row))?;\n }\n let id = row.into_iter().nth(primary_key).expect(\"short row\");\n updates.insert(id, update);\n }\n let count = updates.len() as u64;\n self.txn.update(table, updates)?;\n Ok(count)\n }\n\n /// Sorts the input rows.\n fn order(source: Rows, order: Vec<(Expression, Direction)>) -> Result {\n // We can't use sorted_by_cached_key(), since expression evaluation is\n // fallible, and since we may have to vary the sort direction of each\n // expression. Collect the rows and pre-computed sort keys into a vec.\n let mut rows: Vec<(Row, Vec)> = source\n .map(|result| {\n result.and_then(|row| {\n let sort_keys =\n order.iter().map(|(expr, _)| expr.evaluate(Some(&row))).try_collect()?;\n Ok((row, sort_keys))\n })\n })\n .try_collect()?;\n\n rows.sort_by(|(_, a_keys), (_, b_keys)| {\n let dirs = order.iter().map(|(_, dir)| dir).copied();\n for (a_key, b_key, dir) in izip!(a_keys, b_keys, dirs) {\n let mut ordering = a_key.cmp(b_key);\n if dir == Direction::Descending {\n ordering = ordering.reverse();\n }\n if ordering != Ordering::Equal {\n return ordering;\n }\n }\n Ordering::Equal\n });\n\n Ok(Box::new(rows.into_iter().map(|(row, _)| Ok(row))))\n }\n}\n\n/// A plan execution result.\npub enum ExecutionResult {\n CreateTable { name: String },\n DropTable { name: String, existed: bool },\n Delete { count: u64 },\n Insert { count: u64 },\n Update { count: u64 },\n Select { columns: Vec