Showing preview only (1,449K chars total). Download the full file or copy to clipboard to get everything.
Repository: jepsen-io/jepsen
Branch: main
Commit: 07e0ee8887f8
Files: 177
Total size: 1.4 MB
Directory structure:
gitextract_id8dqfc2/
├── .gitignore
├── .travis.yml
├── README.md
├── antithesis/
│ ├── .gitignore
│ ├── CHANGELOG.md
│ ├── LICENSE
│ ├── README.md
│ ├── composer/
│ │ ├── check
│ │ └── op
│ ├── doc/
│ │ └── intro.md
│ ├── project.clj
│ ├── src/
│ │ └── jepsen/
│ │ ├── antithesis/
│ │ │ ├── Random.java
│ │ │ └── composer.clj
│ │ └── antithesis.clj
│ └── test/
│ └── jepsen/
│ ├── antithesis/
│ │ └── composer_test.clj
│ └── antithesis_test.clj
├── charybdefs/
│ ├── .gitignore
│ ├── README.md
│ ├── project.clj
│ ├── src/
│ │ └── jepsen/
│ │ └── charybdefs.clj
│ └── test/
│ ├── .gitignore
│ └── jepsen/
│ └── charybdefs/
│ └── remote_test.clj
├── contributing.md
├── doc/
│ ├── color.md
│ ├── lxc-f36.md
│ ├── lxc.md
│ ├── plan.md
│ ├── tutorial/
│ │ ├── 01-scaffolding.md
│ │ ├── 02-db.md
│ │ ├── 03-client.md
│ │ ├── 04-checker.md
│ │ ├── 05-nemesis.md
│ │ ├── 06-refining.md
│ │ ├── 07-parameters.md
│ │ ├── 08-set.md
│ │ └── index.md
│ └── whats-here.md
├── docker/
│ ├── .gitignore
│ ├── README.md
│ ├── bin/
│ │ ├── build-docker-compose
│ │ ├── console
│ │ ├── up
│ │ └── web
│ ├── control/
│ │ ├── .gitignore
│ │ ├── Dockerfile
│ │ ├── bashrc
│ │ └── init.sh
│ ├── docker-compose.dev.yml
│ ├── node/
│ │ ├── Dockerfile
│ │ └── setup-jepsen.sh
│ ├── secret/
│ │ └── .gitkeep
│ └── template/
│ ├── db.yml
│ ├── depends.yml
│ └── docker-compose.yml
├── generator/
│ ├── .gitignore
│ ├── CHANGELOG.md
│ ├── LICENSE
│ ├── README.md
│ ├── doc/
│ │ └── intro.md
│ ├── project.clj
│ ├── src/
│ │ └── jepsen/
│ │ ├── generator/
│ │ │ ├── context.clj
│ │ │ ├── test.clj
│ │ │ └── translation_table.clj
│ │ ├── generator.clj
│ │ └── random.clj
│ └── test/
│ └── jepsen/
│ ├── generator/
│ │ ├── context_test.clj
│ │ └── translation_table_test.clj
│ ├── generator_test.clj
│ └── random_test.clj
├── jepsen/
│ ├── .eastwood.clj
│ ├── LICENSE.txt
│ ├── project.clj
│ ├── resources/
│ │ ├── bump-time.c
│ │ ├── corrupt-file.c
│ │ ├── log4j.properties
│ │ ├── strobe-time-experiment.c
│ │ └── strobe-time.c
│ ├── src/
│ │ └── jepsen/
│ │ ├── adya.clj
│ │ ├── checker/
│ │ │ ├── clock.clj
│ │ │ ├── perf.clj
│ │ │ ├── plot.clj
│ │ │ └── timeline.clj
│ │ ├── checker.clj
│ │ ├── cli.clj
│ │ ├── client.clj
│ │ ├── codec.clj
│ │ ├── control/
│ │ │ ├── clj_ssh.clj
│ │ │ ├── core.clj
│ │ │ ├── docker.clj
│ │ │ ├── k8s.clj
│ │ │ ├── net.clj
│ │ │ ├── retry.clj
│ │ │ ├── scp.clj
│ │ │ ├── sshj.clj
│ │ │ └── util.clj
│ │ ├── control.clj
│ │ ├── core.clj
│ │ ├── db/
│ │ │ └── watchdog.clj
│ │ ├── db.clj
│ │ ├── faketime.clj
│ │ ├── fs_cache.clj
│ │ ├── generator/
│ │ │ └── interpreter.clj
│ │ ├── independent.clj
│ │ ├── lazyfs.clj
│ │ ├── nemesis/
│ │ │ ├── combined.clj
│ │ │ ├── file.clj
│ │ │ ├── membership/
│ │ │ │ └── state.clj
│ │ │ ├── membership.clj
│ │ │ └── time.clj
│ │ ├── nemesis.clj
│ │ ├── net/
│ │ │ └── proto.clj
│ │ ├── net.clj
│ │ ├── os/
│ │ │ ├── centos.clj
│ │ │ ├── debian.clj
│ │ │ ├── smartos.clj
│ │ │ └── ubuntu.clj
│ │ ├── os.clj
│ │ ├── print.clj
│ │ ├── reconnect.clj
│ │ ├── repl.clj
│ │ ├── report.clj
│ │ ├── role.clj
│ │ ├── store/
│ │ │ ├── FileOffsetOutputStream.java
│ │ │ ├── FressianReader.java
│ │ │ ├── format.clj
│ │ │ └── fressian.clj
│ │ ├── store.clj
│ │ ├── tests/
│ │ │ ├── adya.clj
│ │ │ ├── bank.clj
│ │ │ ├── causal.clj
│ │ │ ├── causal_reverse.clj
│ │ │ ├── cycle/
│ │ │ │ ├── append.clj
│ │ │ │ └── wr.clj
│ │ │ ├── cycle.clj
│ │ │ ├── kafka.clj
│ │ │ ├── linearizable_register.clj
│ │ │ └── long_fork.clj
│ │ ├── tests.clj
│ │ ├── util.clj
│ │ └── web.clj
│ └── test/
│ └── jepsen/
│ ├── checker/
│ │ └── timeline_test.clj
│ ├── checker_test.clj
│ ├── cli_test.clj
│ ├── common_test.clj
│ ├── control/
│ │ ├── net_test.clj
│ │ └── util_test.clj
│ ├── control_test.clj
│ ├── core_test.clj
│ ├── db/
│ │ └── watchdog_test.clj
│ ├── db_test.clj
│ ├── fs_cache_test.clj
│ ├── generator/
│ │ └── interpreter_test.clj
│ ├── generator_test.clj
│ ├── independent_test.clj
│ ├── lazyfs_test.clj
│ ├── nemesis/
│ │ ├── combined_test.clj
│ │ ├── file_test.clj
│ │ └── time_test.clj
│ ├── nemesis_test.clj
│ ├── perf_test.clj
│ ├── print_test.clj
│ ├── role_test.clj
│ ├── store/
│ │ └── format_test.clj
│ ├── store_test.clj
│ ├── tests/
│ │ ├── causal_reverse_test.clj
│ │ ├── kafka_test.clj
│ │ └── long_fork_test.clj
│ └── util_test.clj
└── txn/
├── .gitignore
├── CHANGELOG.md
├── LICENSE
├── README.md
├── doc/
│ └── intro.md
├── project.clj
├── src/
│ └── jepsen/
│ ├── txn/
│ │ └── micro_op.clj
│ └── txn.clj
└── test/
└── jepsen/
└── txn_test.clj
================================================
FILE CONTENTS
================================================
================================================
FILE: .gitignore
================================================
.cake
.lein-repl-history
.rebel_readline_history
pom.xml
pom.xml.asc
repl-port
*.jar
*.tar
*.tar.bz2
*.war
*.deb
*~
.*.swp
*.log
log.txt
lib
classes
build
.lein-deps-sum
.lein-failures
/site
site/**
bench/**
*/target/**
*/checkouts/**
*/store/**
*/report/**
*/timeline.html
*/resources/datomic/download-key
*/resources/datomic/riak-transactor.properties
*/.nrepl-port
.idea
*.iml
jepsen/doc
.eastwood
docker/docker-compose.yml
================================================
FILE: .travis.yml
================================================
language: clojure
lein: lein
dist: trusty
jdk:
- oraclejdk8
branches:
only:
- master
addons:
apt:
packages:
- gnuplot
- gnuplot-x11
before_install:
- cd ${TRAVIS_BUILD_DIR}/jepsen
script:
- lein clean
# Exclude namespaces importing rhizome.viz, because compiling those requires
# X11. For full error message, see:
# https://travis-ci.com/github/jepsen-io/jepsen/builds/161202862
- lein eastwood '{:config-files [".eastwood.clj"], :exclude-namespaces [jepsen.tests.cycle.wr jepsen.tests.cycle.append]}'
- lein test
notifications:
email:
on_success: never
on_failure: never
================================================
FILE: README.md
================================================
# Jepsen
Breaking distributed systems so you don't have to.
Jepsen is a Clojure library. A test is a Clojure program which uses the Jepsen
library to set up a distributed system, run a bunch of operations against that
system, and verify that the history of those operations makes sense. Jepsen has
been used to verify everything from eventually-consistent commutative databases
to linearizable coordination systems to distributed task schedulers. It can
also generate graphs of performance and availability, helping you characterize
how a system responds to different faults. See
[jepsen.io](https://jepsen.io/analyses) for examples of the sorts of analyses
you can carry out with Jepsen.
[](https://clojars.org/jepsen)
[](https://travis-ci.com/jepsen-io/jepsen)
## Design Overview
A Jepsen test runs as a Clojure program on a *control node*. That program uses
SSH to log into a bunch of *db nodes*, where it sets up the distributed system
you're going to test using the test's pluggable *os* and *db*.
Once the system is running, the control node spins up a set of logically
single-threaded *processes*, each with its own *client* for the distributed
system. A *generator* generates new operations for each process to perform.
Processes then apply those operations to the system using their clients. The
start and end of each operation is recorded in a *history*. While performing
operations, a special *nemesis* process introduces faults into the system--also
scheduled by the generator.
Finally, the DB and OS are torn down. Jepsen uses a *checker* to analyze the
test's history for correctness, and to generate reports, graphs, etc. The test,
history, analysis, and any supplementary results are written to the filesystem
under `store/<test-name>/<date>/` for later review. Symlinks to the latest
results are maintained at each level for convenience.
## Documentation
This [tutorial](doc/tutorial/index.md) walks you through writing a Jepsen test
from scratch. An independent translation is available in [Chinese](https://jaydenwen123.gitbook.io/zh_jepsen_doc/).
For reference, see the [API documentation](http://jepsen-io.github.io/jepsen/).
[What's Here](doc/whats-here.md) provides an overview of Jepsen's namespaces
and how they work together.
## Setting up a Jepsen Environment
So, you've got a Jepsen test, and you'd like to run it! Or maybe you'd like to
start learning how to write tests. You've got several options:
### AWS
If you have an AWS account, you can launch a full Jepsen cluster---control and
DB nodes---from the [AWS
Marketplace](https://aws.amazon.com/marketplace/pp/Jepsen-LLC-Jepsen/B01LZ7Y7U0).
Click "Continue to Subscribe", "Continue to Configuration", and choose
"CloudFormation Template". You can choose the number of nodes you'd like to
deploy, adjust the instance types and disk sizes, and so on. These are full
VMs, which means they can test clock skew.
The AWS marketplace clusters come with an hourly fee (generally $1/hr/node),
which helps fund Jepsen development.
### LXC
You can set up your DB nodes as LXC containers, and use your local machine as
the control node. See the [LXC documentation](doc/lxc.md) for guidelines. This
might be the easiest setup for hacking on tests: you'll be able to edit source
code, run profilers, etc on the local node. Containers don't have real clocks,
so you generally can't use them to test clock skew.
### VMs, Real Hardware, etc.
You should be able to run Jepsen against almost any machines which have:
- A TCP network
- An SSH server
- Sudo or root access
Each DB node should be accessible from the control node via SSH: you need to be
able to run `ssh myuser@some-node`, and get a shell. By default, DB nodes are
named n1, n2, n3, n4, and n5, but that (along with SSH username, password,
identity files, etc) is all definable in your test, or at the CLI. The account
you use on those boxes needs sudo access to set up DBs, control firewalls, etc.
BE ADVISED: tests may mess with clocks, add apt repos, run killall -9 on
processes, and generally break things, so you shouldn't, you know, point Jepsen
at your prod machines unless you like to live dangerously, or you wrote the
test and know exactly what it's doing.
NOTE: Most Jepsen tests are written with more specific requirements in
mind---like running on Debian, using `iptables` for network manipulation, etc.
See the specific test code for more details.
### Docker (Unsupported)
There is a [Docker Compose setup](/docker) for running a Jepsen cluster on a
single machine. Sadly the Docker platform has been something of a moving
target; this environment tends to break in new and exciting ways on various
platforms every few months. If you're a Docker whiz and can get this going
reliably on Debian & OS X that's great--pull requests would be a big help.
Like other containers Docker containers don't have real clocks--that means you
generally can't use them to test clock skew.
### Setting Up Control Nodes
For AWS and Docker installs, your control node comes preconfigured with all the
software you'll need to run most Jepsen tests. If you build your own control
node (or if you're using your local machine as a control node), you'll need a
few things:
- A [JVM](https://openjdk.java.net/install/)---version 21 or higher.
- JNA, so the JVM can talk to your SSH.
- [Leiningen](https://leiningen.org/): a Clojure build tool.
- [Gnuplot](http://www.gnuplot.info/): how Jepsen renders performance plots.
- [Graphviz](https://graphviz.org/): how Jepsen renders transactional anomalies.
On Debian, try:
```
sudo apt install default-jdk libjna-java gnuplot graphviz
```
... to get the basic requirements in place. Debian's Leiningen packages are
ancient, so [download lein from the web instead](https://leiningen.org/).
## Running a Test
Once you've got everything set up, you should be able to run `cd aerospike;
lein test`, and it'll spit out something like
```clj
INFO jepsen.core - Analysis invalid! (ﾉಥ益ಥ）ﾉ ┻━┻
{:valid? false,
:counter
{:valid? false,
:reads
[[190 193 194]
[199 200 201]
[253 255 256]
...}}
```
## Working With the REPL
Jepsen tests emit `.jepsen` files in the `store/` directory. You can use these
to investigate a test at the repl. Run `lein repl` in the test directory (which
should contain `store...`, then load a test using `store/test`:
```clj
user=> (def t (store/test -1))
```
-1 is the last test run, -2 is the second-to-last. 0 is the first, 1 is the
second, and so on. You can also load a by the string directory name. As a handy
shortcut, clicking on the title of a test in the web interface will copy its
path to the clipboard.
```clj
user=> (def t (store/test "/home/aphyr/jepsen.etcd/store/etcd append etcdctl kill/20221003T124714.485-0400"))
```
These have the same structure as the test maps you're used to working with in
Jepsen, though without some fields that wouldn't make sense to serialize--no
`:checker`, `:client`, etc.
```clj
jepsen.etcd=> (:name t)
"etcd append etcdctl kill"
jepsen.etcd=> (:ops-per-key t)
200
```
These test maps are also lazy: to speed up working at the REPL, they won't load
the history or results until you ask for them. Then they're loaded from disk
and cached.
```clj
jepsen.etcd=> (count (:history t))
52634
```
You can use all the usual Clojure tricks to introspect results and histories.
Here's an aborted read (G1a) anomaly--we'll pull out the ops which wrote and
read the aborted read:
```clj
jepsen.etcd=> (def writer (-> t :results :workload :anomalies :G1a first :writer))
#'jepsen.etcd/writer
jepsen.etcd=> (def reader (-> t :results :workload :anomalies :G1a first :op))
#'jepsen.etcd/reader
```
The writer appended 11 and 12 to key 559, but failed, returning a duplicate key
error:
```clj
jepsen.etcd=> (:value writer)
[[:r 559 nil] [:r 558 nil] [:append 559 11] [:append 559 12]]
jepsen.etcd=> (:error writer)
[:duplicate-key "rpc error: code = InvalidArgument desc = etcdserver: duplicate key given in txn request"]
```
The reader, however, observed a value for 559 beginning with 12!
```clj
jepsen.etcd=> (:value reader)
[[:r 559 [12]] [:r 557 [1]]]
```
Let's find all successful transactions:
```clj
jepsen.etcd=> (def txns (->> t :history (filter #(and (= :txn (:f %)) (= :ok (:type %)))) (map :value)))
#'jepsen.etcd/txns
```
And restrict those to just operations which affected key 559:
```clj
jepsen.etcd=> (->> txns (filter (partial some (comp #{559} second))) pprint)
([[:r 559 [12]] [:r 557 [1]]]
[[:r 559 [12]] [:append 559 1] [:r 559 [12 1]]]
[[:append 556 32]
[:r 556 [1 18 29 32]]
[:r 556 [1 18 29 32]]
[:r 559 [12 1]]]
[[:r 559 [12 1]]]
[[:append 559 9] [:r 557 [1 5]] [:r 558 [1]] [:r 558 [1]]]
[[:r 559 [12 1 9]] [:r 559 [12 1 9]]]
[[:append 559 17]]
[[:r 559 [12 1 9 17]] [:append 558 5]]
[[:r 559 [12 1 9 17]]
[:append 557 22]
[:append 559 27]
[:r 557 [1 5 12 22]]])
```
Sure enough, no OK appends of 12 to key 559!
You'll find more functions for slicing-and-dicing tests in `jepsen.store`.
## FAQ
### JSCH auth errors
If you see `com.jcraft.jsch.JSchException: Auth fail`, this means something
about your test's `:ssh` map is wrong, or your control node's SSH environment
is a bit weird.
0. Confirm that you can ssh to the node that Jepsen failed to connect to. Try
`ssh -v` for verbose information--pay special attention to whether it uses a
password or private key.
1. If you intend to use a username and password, confirm that they're specified
correctly in your test's `:ssh` map.
2. If you intend to log in with a private key, make sure your SSH agent is
running.
- `ssh-add -l` should show the key you use to log in.
- If your agent isn't running, try launching one with `ssh-agent`.
- If your agent shows no keys, you might need to add it with `ssh-add`.
- If you're SSHing to a control node, SSH might be forwarding your local
agent's keys rather than using those on the control node. Try `ssh -a` to
disable agent forwarding.
If you've SSHed to a DB node already, you might also encounter a jsch bug which
doesn't know how to read hashed known_hosts files. Remove all keys for the DB
hosts from your `known_hosts` file, then:
```sh
ssh-keyscan -t rsa n1 >> ~/.ssh/known_hosts
ssh-keyscan -t rsa n2 >> ~/.ssh/known_hosts
ssh-keyscan -t rsa n3 >> ~/.ssh/known_hosts
ssh-keyscan -t rsa n4 >> ~/.ssh/known_hosts
ssh-keyscan -t rsa n5 >> ~/.ssh/known_hosts
```
to add unhashed versions of each node's hostkey to your `~/.ssh/known_hosts`.
### SSHJ auth errors
If you get an exception like `net.schmizz.sshj.transport.TransportException:
Could not verify 'ssh-ed25519' host key with fingerprint 'bf:4a:...' for 'n1'
on port 22`, but you're sure you've got the keys in your `~/.ssh/known-hosts`,
this is because (I think) SSHJ tries to verify only the ed25519 key and
*ignores* the RSA key. You can add the ed25519 keys explicitly via:
```sh
ssh-keyscan -t ed25519 n1 >> ~/.ssh/known_hosts
...
```
## Other Projects
Additional projects that may be of interest:
- [Jecci](https://github.com/michaelzenz/jecci): A wrapper framework around
Jepsen
- [Porcupine](https://github.com/anishathalye/porcupine): a linearizability
checker written in Go.
- [elle-cli](https://github.com/ligurio/elle-cli): command-line frontend to
transactional consistency checkers for black-box databases.
- [Tickloom](https://github.com/unmeshjoshi/tickloom): A deterministic-simulation framework for building distributed systems, with Jepsen integration for consistency checks.
================================================
FILE: antithesis/.gitignore
================================================
/target
/classes
/checkouts
profiles.clj
pom.xml
pom.xml.asc
*.jar
*.class
/.lein-*
/.nrepl-port
/.prepl-port
.hgignore
.hg/
================================================
FILE: antithesis/CHANGELOG.md
================================================
# Change Log
All notable changes to this project will be documented in this file. This change log follows the conventions of [keepachangelog.com](https://keepachangelog.com/).
## [Unreleased]
### Changed
- Add a new arity to `make-widget-async` to provide a different widget shape.
## [0.1.1] - 2025-10-22
### Changed
- Documentation on how to make the widgets.
### Removed
- `make-widget-sync` - we're all async, all the time.
### Fixed
- Fixed widget maker to keep working when daylight savings switches over.
## 0.1.0 - 2025-10-22
### Added
- Files from the new template.
- Widget maker public API - `make-widget-sync`.
[Unreleased]: https://sourcehost.site/your-name/antithesis/compare/0.1.1...HEAD
[0.1.1]: https://sourcehost.site/your-name/antithesis/compare/0.1.0...0.1.1
================================================
FILE: antithesis/LICENSE
================================================
Eclipse Public License - v 2.0
THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE
PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION
OF THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT.
1. DEFINITIONS
"Contribution" means:
a) in the case of the initial Contributor, the initial content
Distributed under this Agreement, and
b) in the case of each subsequent Contributor:
i) changes to the Program, and
ii) additions to the Program;
where such changes and/or additions to the Program originate from
and are Distributed by that particular Contributor. A Contribution
"originates" from a Contributor if it was added to the Program by
such Contributor itself or anyone acting on such Contributor's behalf.
Contributions do not include changes or additions to the Program that
are not Modified Works.
"Contributor" means any person or entity that Distributes the Program.
"Licensed Patents" mean patent claims licensable by a Contributor which
are necessarily infringed by the use or sale of its Contribution alone
or when combined with the Program.
"Program" means the Contributions Distributed in accordance with this
Agreement.
"Recipient" means anyone who receives the Program under this Agreement
or any Secondary License (as applicable), including Contributors.
"Derivative Works" shall mean any work, whether in Source Code or other
form, that is based on (or derived from) the Program and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship.
"Modified Works" shall mean any work in Source Code or other form that
results from an addition to, deletion from, or modification of the
contents of the Program, including, for purposes of clarity any new file
in Source Code form that contains any contents of the Program. Modified
Works shall not include works that contain only declarations,
interfaces, types, classes, structures, or files of the Program solely
in each case in order to link to, bind by name, or subclass the Program
or Modified Works thereof.
"Distribute" means the acts of a) distributing or b) making available
in any manner that enables the transfer of a copy.
"Source Code" means the form of a Program preferred for making
modifications, including but not limited to software source code,
documentation source, and configuration files.
"Secondary License" means either the GNU General Public License,
Version 2.0, or any later versions of that license, including any
exceptions or additional permissions as identified by the initial
Contributor.
2. GRANT OF RIGHTS
a) Subject to the terms of this Agreement, each Contributor hereby
grants Recipient a non-exclusive, worldwide, royalty-free copyright
license to reproduce, prepare Derivative Works of, publicly display,
publicly perform, Distribute and sublicense the Contribution of such
Contributor, if any, and such Derivative Works.
b) Subject to the terms of this Agreement, each Contributor hereby
grants Recipient a non-exclusive, worldwide, royalty-free patent
license under Licensed Patents to make, use, sell, offer to sell,
import and otherwise transfer the Contribution of such Contributor,
if any, in Source Code or other form. This patent license shall
apply to the combination of the Contribution and the Program if, at
the time the Contribution is added by the Contributor, such addition
of the Contribution causes such combination to be covered by the
Licensed Patents. The patent license shall not apply to any other
combinations which include the Contribution. No hardware per se is
licensed hereunder.
c) Recipient understands that although each Contributor grants the
licenses to its Contributions set forth herein, no assurances are
provided by any Contributor that the Program does not infringe the
patent or other intellectual property rights of any other entity.
Each Contributor disclaims any liability to Recipient for claims
brought by any other entity based on infringement of intellectual
property rights or otherwise. As a condition to exercising the
rights and licenses granted hereunder, each Recipient hereby
assumes sole responsibility to secure any other intellectual
property rights needed, if any. For example, if a third party
patent license is required to allow Recipient to Distribute the
Program, it is Recipient's responsibility to acquire that license
before distributing the Program.
d) Each Contributor represents that to its knowledge it has
sufficient copyright rights in its Contribution, if any, to grant
the copyright license set forth in this Agreement.
e) Notwithstanding the terms of any Secondary License, no
Contributor makes additional grants to any Recipient (other than
those set forth in this Agreement) as a result of such Recipient's
receipt of the Program under the terms of a Secondary License
(if permitted under the terms of Section 3).
3. REQUIREMENTS
3.1 If a Contributor Distributes the Program in any form, then:
a) the Program must also be made available as Source Code, in
accordance with section 3.2, and the Contributor must accompany
the Program with a statement that the Source Code for the Program
is available under this Agreement, and informs Recipients how to
obtain it in a reasonable manner on or through a medium customarily
used for software exchange; and
b) the Contributor may Distribute the Program under a license
different than this Agreement, provided that such license:
i) effectively disclaims on behalf of all other Contributors all
warranties and conditions, express and implied, including
warranties or conditions of title and non-infringement, and
implied warranties or conditions of merchantability and fitness
for a particular purpose;
ii) effectively excludes on behalf of all other Contributors all
liability for damages, including direct, indirect, special,
incidental and consequential damages, such as lost profits;
iii) does not attempt to limit or alter the recipients' rights
in the Source Code under section 3.2; and
iv) requires any subsequent distribution of the Program by any
party to be under a license that satisfies the requirements
of this section 3.
3.2 When the Program is Distributed as Source Code:
a) it must be made available under this Agreement, or if the
Program (i) is combined with other material in a separate file or
files made available under a Secondary License, and (ii) the initial
Contributor attached to the Source Code the notice described in
Exhibit A of this Agreement, then the Program may be made available
under the terms of such Secondary Licenses, and
b) a copy of this Agreement must be included with each copy of
the Program.
3.3 Contributors may not remove or alter any copyright, patent,
trademark, attribution notices, disclaimers of warranty, or limitations
of liability ("notices") contained within the Program from any copy of
the Program which they Distribute, provided that Contributors may add
their own appropriate notices.
4. COMMERCIAL DISTRIBUTION
Commercial distributors of software may accept certain responsibilities
with respect to end users, business partners and the like. While this
license is intended to facilitate the commercial use of the Program,
the Contributor who includes the Program in a commercial product
offering should do so in a manner which does not create potential
liability for other Contributors. Therefore, if a Contributor includes
the Program in a commercial product offering, such Contributor
("Commercial Contributor") hereby agrees to defend and indemnify every
other Contributor ("Indemnified Contributor") against any losses,
damages and costs (collectively "Losses") arising from claims, lawsuits
and other legal actions brought by a third party against the Indemnified
Contributor to the extent caused by the acts or omissions of such
Commercial Contributor in connection with its distribution of the Program
in a commercial product offering. The obligations in this section do not
apply to any claims or Losses relating to any actual or alleged
intellectual property infringement. In order to qualify, an Indemnified
Contributor must: a) promptly notify the Commercial Contributor in
writing of such claim, and b) allow the Commercial Contributor to control,
and cooperate with the Commercial Contributor in, the defense and any
related settlement negotiations. The Indemnified Contributor may
participate in any such claim at its own expense.
For example, a Contributor might include the Program in a commercial
product offering, Product X. That Contributor is then a Commercial
Contributor. If that Commercial Contributor then makes performance
claims, or offers warranties related to Product X, those performance
claims and warranties are such Commercial Contributor's responsibility
alone. Under this section, the Commercial Contributor would have to
defend claims against the other Contributors related to those performance
claims and warranties, and if a court requires any other Contributor to
pay any damages as a result, the Commercial Contributor must pay
those damages.
5. NO WARRANTY
EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, AND TO THE EXTENT
PERMITTED BY APPLICABLE LAW, THE PROGRAM IS PROVIDED ON AN "AS IS"
BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR
IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF
TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR
PURPOSE. Each Recipient is solely responsible for determining the
appropriateness of using and distributing the Program and assumes all
risks associated with its exercise of rights under this Agreement,
including but not limited to the risks and costs of program errors,
compliance with applicable laws, damage to or loss of data, programs
or equipment, and unavailability or interruption of operations.
6. DISCLAIMER OF LIABILITY
EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, AND TO THE EXTENT
PERMITTED BY APPLICABLE LAW, NEITHER RECIPIENT NOR ANY CONTRIBUTORS
SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST
PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE
EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGES.
7. GENERAL
If any provision of this Agreement is invalid or unenforceable under
applicable law, it shall not affect the validity or enforceability of
the remainder of the terms of this Agreement, and without further
action by the parties hereto, such provision shall be reformed to the
minimum extent necessary to make such provision valid and enforceable.
If Recipient institutes patent litigation against any entity
(including a cross-claim or counterclaim in a lawsuit) alleging that the
Program itself (excluding combinations of the Program with other software
or hardware) infringes such Recipient's patent(s), then such Recipient's
rights granted under Section 2(b) shall terminate as of the date such
litigation is filed.
All Recipient's rights under this Agreement shall terminate if it
fails to comply with any of the material terms or conditions of this
Agreement and does not cure such failure in a reasonable period of
time after becoming aware of such noncompliance. If all Recipient's
rights under this Agreement terminate, Recipient agrees to cease use
and distribution of the Program as soon as reasonably practicable.
However, Recipient's obligations under this Agreement and any licenses
granted by Recipient relating to the Program shall continue and survive.
Everyone is permitted to copy and distribute copies of this Agreement,
but in order to avoid inconsistency the Agreement is copyrighted and
may only be modified in the following manner. The Agreement Steward
reserves the right to publish new versions (including revisions) of
this Agreement from time to time. No one other than the Agreement
Steward has the right to modify this Agreement. The Eclipse Foundation
is the initial Agreement Steward. The Eclipse Foundation may assign the
responsibility to serve as the Agreement Steward to a suitable separate
entity. Each new version of the Agreement will be given a distinguishing
version number. The Program (including Contributions) may always be
Distributed subject to the version of the Agreement under which it was
received. In addition, after a new version of the Agreement is published,
Contributor may elect to Distribute the Program (including its
Contributions) under the new version.
Except as expressly stated in Sections 2(a) and 2(b) above, Recipient
receives no rights or licenses to the intellectual property of any
Contributor under this Agreement, whether expressly, by implication,
estoppel or otherwise. All rights in the Program not expressly granted
under this Agreement are reserved. Nothing in this Agreement is intended
to be enforceable by any entity that is not a Contributor or Recipient.
No third-party beneficiary rights are created under this Agreement.
Exhibit A - Form of Secondary Licenses Notice
"This Source Code may also be made available under the following
Secondary Licenses when the conditions for such availability set forth
in the Eclipse Public License, v. 2.0 are satisfied: GNU General Public
License as published by the Free Software Foundation, either version 2
of the License, or (at your option) any later version, with the GNU
Classpath Exception which is available at
https://www.gnu.org/software/classpath/license.html."
Simply including a copy of this Agreement, including this Exhibit A
is not sufficient to license the Source Code under Secondary Licenses.
If it is not possible or desirable to put the notice in a particular
file, then You may include the notice in a location (such as a LICENSE
file in a relevant directory) where a recipient would be likely to
look for such a notice.
You may add additional accurate notices of copyright ownership.
================================================
FILE: antithesis/README.md
================================================
# jepsen.antithesis
This library supports running Jepsen tests inside Antithesis environments. It
provides entropy, lifecycle hooks, and assertions.
## Installation
[](https://clojars.org/io.jepsen/antithesis)
From Clojars, as usual. Note that the Antithesis SDK pulls in an ancient
version of Jackson and *needs it*, so in your `project.clj`, you'll likely want
to prevent other dependencies from relying on Jackson:
```clj
:dependencies [...
[io.jepsen/antithesis "0.1.0"]
[jepsen "0.3.10"
:exclusions [com.fasterxml.jackson.core/jackson-databind
com.fasterxml.jackson.core/jackson-annotations
com.fasterxml.jackson.core/jackson-core]]]
```
## Usage
The main namespace is [`jepsen.antithesis`](src/jepsen/antithesis.clj). There
are several things you can do to integrate your test into Antithesis.
### Randomness
First, wrap the entire program in `(antithesis/with-rng ...)`. This does
nothing in ordinary environments, but in Antithesis, it replaces the
jepsen.random RNG with the Antithesis SDK's entropy source.
### Wrapping Tests
Second, wrap the entire test map with `(antithesis/test test)`. In an
Antithesis run, this disables the OS, DB, and SSH connections.
## Clients
Wrap your client in `(antithesis/client your-client)`. This client informs
Antithesis that the setup is complete, and makes assertions about each
invocation and completion.
## Checker
You can either make assertions (see below) by hand inside your checkers, or you
can wrap an existing checker in `(antithesis/checker "some name" checker)`.
This asserts that the checker's results are always `:valid? true`. You can also
use `antithesis/checker+` to traverse a tree of checkers, wrapping each one
with assertions.
## Generator
Instead of a time limit, you can limit your generator with something like:
```clj
(if (antithesis/antithesis?)
(antithesis/early-termination-generator
{:interval 100}
my-gen)
(gen/time-limit ... my-gen))
```
This early-termination-generator flips a coin every 100 operations, deciding
whether to continue. This allows Antithesis to perform some long runs and some
short ones. I'm not totally sure whether this is a good idea yet, but it does
seem to get us to much shorter reproducible histories.
## Lifecycle
If you'd like to manage the lifecycle manually, you can Call `setup-complete!`
once the test is ready to begin--for instance, at the end of `Client/setup!`.
Call `event!` to signal interesting things have happened.
## Assertions
Assertions begin with `assert-`, and take an expression, a message, and data
to include if the assertion fails. For instance:
```clj
(assert-always! (not (db-corrupted?))
"DB corrupted" {:db "foo"})
```
Ideally, you want to do these *during* the test run, so Antithesis can fail
fast. Many checks can only be done with the full history, by the checker; for
these, assert test validity in the checker itself:
```clj
(defrecord MyChecker []
(check [_ test history opts]
...
(a/assert-always (true? valid?) "checker valid" a)
{:valid? valid? ...}))
```
## License
Copyright © Jepsen, LLC
This program and the accompanying materials are made available under the
terms of the Eclipse Public License 2.0 which is available at
https://www.eclipse.org/legal/epl-2.0.
This Source Code may also be made available under the following Secondary
Licenses when the conditions for such availability set forth in the Eclipse
Public License, v. 2.0 are satisfied: GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or (at your
option) any later version, with the GNU Classpath Exception which is available
at https://www.gnu.org/software/classpath/license.html.
================================================
FILE: antithesis/composer/check
================================================
#!/bin/bash
set -e
DIR="/tmp/jepsen/antithesis"
mkdir -p "$DIR"
FIFO="${DIR}/check"
# Create a FIFO
#echo "$FIFO"
mkfifo "$FIFO"
# And block on it
cat "$FIFO"
================================================
FILE: antithesis/composer/op
================================================
#!/bin/bash
set -e
DIR="/tmp/jepsen/antithesis"
mkdir -p "$DIR"
# Pick a hopefully unique ID for this op
ID="$(date +%s.%N)-${$}"
FIFO="${DIR}/op-${ID}"
# Create a FIFO
#echo "$FIFO"
mkfifo "$FIFO"
# And block on it
cat "$FIFO"
================================================
FILE: antithesis/doc/intro.md
================================================
# Introduction to antithesis
TODO: write [great documentation](https://jacobian.org/writing/what-to-write/)
================================================
FILE: antithesis/project.clj
================================================
(defproject io.jepsen/antithesis "0.1.1-SNAPSHOT"
:description "Support for running Jepsen inside Antithesis"
:url "https://github.com/jepsen-io/jepsen"
:license {:name "EPL-2.0 OR GPL-2.0-or-later WITH Classpath-exception-2.0"
:url "https://www.eclipse.org/legal/epl-2.0/"}
:dependencies [[jepsen "0.3.11"
; Jepsen pulls in jackson 2.16, but the Antithesis SDK
; expects 2.2.3
:exclusions [com.fasterxml.jackson.core/jackson-databind
com.fasterxml.jackson.core/jackson-annotations
com.fasterxml.jackson.core/jackson-core]]
[com.antithesis/sdk "1.4.4"]]
:java-source-paths ["src"]
:javac-options ["--release" "17"]
:repl-options {:init-ns jepsen.antithesis}
:test-selectors {:default (fn [m]
(not (:perf m)))
:focus :focus
:perf :perf})
================================================
FILE: antithesis/src/jepsen/antithesis/Random.java
================================================
// A java.util.random which draws from Antithesis' SDK.
package jepsen.antithesis;
import java.util.Arrays;
import java.util.List;
import java.util.random.RandomGenerator;
public class Random implements RandomGenerator {
public static List<Boolean> booleans = Arrays.asList(true, false);
private final com.antithesis.sdk.Random r;
public Random() {
super();
this.r = new com.antithesis.sdk.Random();
}
public long nextLong() {
return r.getRandom();
}
public double nextDouble() {
// Adapted from https://developer.classpath.org/doc/java/util/Random-source.html nextDouble
// We're generating doubles in the range 0..1, so we have only the 53 bits
// of mantissa to generate
final long mantissa = r.getRandom() >>> (64-53);
return mantissa / (double) (1L << 53);
}
public boolean nextBoolean() {
return r.randomChoice(booleans);
}
public <T> T randomChoice(List<T> list) {
return r.randomChoice(list);
}
}
================================================
FILE: antithesis/src/jepsen/antithesis/composer.clj
================================================
(ns jepsen.antithesis.composer
"Antithesis' Test Composer drives Jepsen by calling shell scripts, which
communicate with the test via a control directory full of FIFOs. Each FIFO
represents a single test composer action. They work like this:
1. Test composer calls a script to do something.
2. Script creates a fifo like /run/jepsen/op-1234, and blocks on it.
3. Jepsen, watching /run/jepsen, detects the new fifo
4. Jepsen fires off an invocation
5. The operation completes
6. Jepsen writes the results of the operation to /run/jepsen/invoke_1234
7. Jepsen closes the FIFO
8. The shell script prints the results out and exits
9. The test composer receives the results
The types of commands are:
- `op-123`: Runs a single operation from the generator
- `check`: Wraps up the test's :generator, runs :final-generator, and
checks the history.
This namespace provides the Jepsen-side infrastructure for watching the FIFO
directory, a test runner like `jepsen.core/run!`, and a CLI command called
`antithesis` which works like `test`, but uses our runner instead."
(:refer-clojure :exclude [run!])
(:require [bifurcan-clj [core :as b]
[map :as bm]]
[clj-commons.slingshot :refer [try+ throw+]]
[clojure [string :as str]]
[clojure.java.io :as io]
[clojure.tools.logging :refer [info warn fatal]]
[dom-top.core :refer []]
[fipp.edn :refer [pprint]]
[jepsen [core :as jepsen]
[checker :as checker]
[cli :as cli]
[client :as client]
[generator :as gen]
[history :as h]
[store :as store]
[util :as util :refer [with-thread-name
relative-time-nanos
real-pmap
fcatch]]]
[jepsen.generator [interpreter :as gi]
[context :as gc]]
[jepsen.store.format :as store.format]
[potemkin :refer [definterface+]])
(:import (java.io IOException)
(java.util.concurrent ArrayBlockingQueue
BlockingQueue
TimeUnit)
(io.lacuna.bifurcan ISet
Set)
(java.nio.file FileSystems
Path
StandardWatchEventKinds
WatchEvent
WatchEvent$Kind
WatchService)))
(def fifo-dir
"The directory where we watch for fifo operations."
"/tmp/jepsen/antithesis")
(definterface+ IFifoOp
(complete-fifo-op! [this ^long status out]
"Completes this operation with the given exit status (ignored) and
stdout, which is printed to the FIFO."))
(defrecord FifoOp [name ^Path path]
IFifoOp
(complete-fifo-op! [this status out]
(try
(spit (.toFile path) out)
:completed
(catch IOException e
(condp re-find (.getMessage e)
; Whoever invoked this op stopped waiting for a response!
#"Broken pipe"
:broken-pipe
(throw e))))))
(defn ^BlockingQueue fifo-queue
"Constructs a queue full of pending FIFO operations. The fifo watcher writes
to this queue, and the interpreter reads from it, completing its promises."
[]
; Fine, whatever.
(ArrayBlockingQueue. 64))
(defn ^WatchService fifo-watcher!
"Takes a test, and launches a future which watches the FIFO directory for new
files. Each new file results in a FifoOp delivered to `(:fifo-queue test)`.
Creates the directory if it does not exist. Always empties directory before
watching."
[test]
; Ensure FIFO dir exists
(.mkdirs (io/file fifo-dir))
; Clear it out
(->> fifo-dir io/file file-seq next (mapv io/delete-file))
(let [fs (FileSystems/getDefault)
fifo-dir-path (.getPath fs fifo-dir (make-array String 0))
^BlockingQueue queue (:fifo-queue test)]
(future
(with-thread-name "Jepsen FIFO watcher"
(with-open [watch-service (.newWatchService fs)]
(let [kinds (into-array [StandardWatchEventKinds/ENTRY_CREATE])
watch-key (.register fifo-dir-path watch-service kinds)]
(info "Waiting for FIFOs in" fifo-dir)
(loop []
(let [key (.take watch-service)]
(doseq [event (.pollEvents key)]
(condp = (.kind event)
StandardWatchEventKinds/OVERFLOW
(do (fatal "Couldn't keep up with FIFOs!")
(System/exit 1))
StandardWatchEventKinds/ENTRY_CREATE
(let [^Path path (.context ^WatchEvent event)
; This seems to come out as just a relative path, but
; ???
short-name (str (.getName path
(dec (.getNameCount path))))
full-path (.resolve fifo-dir-path path)
op (FifoOp. short-name full-path)]
(.put queue op))))
(if (.reset key)
(recur)
; No longer registered
:no-longer-registered)))))))))
(defmacro with-fifo-watcher!
"Opens a fifo watcher for the duration of the body, and terminates it when
the body completes."
[test & body]
`(let [watcher# (fifo-watcher! ~test)]
(try ~@body
(finally (future-cancel watcher#)))))
;; Generator support.
(defrecord MainGen [gen]
gen/Generator
(op [this test ctx]
(if-let [phase (:antithesis-phase test)]
(when (identical? :main @phase)
(when-let [[op gen'] (gen/op gen test ctx)]
[op (MainGen. gen')]))
; Not in a test-composer run.
(gen/op gen test ctx)))
(update [this test ctx event]
(MainGen. (gen/update gen test ctx event))))
(defn main-gen
"We need a way to terminate the regular generator and move to the final
generator. However, the regular generator and final generator may be
*wrapped* in some kind of state-tracking generator which allows context to
pass from one to the other--and we don't have a way to reach inside that
single generator and trigger a flip.
This generator is intended to be called by the test author, and wraps the
main phase generator. It uses an atom, stored in the test: :antithesis-phase.
This generator emits operations so long as the phase `:main`. Otherwise it
emits `nil`, which forces any composed generator using this to proceed to the
next phase--presumably, the final generator before checking.
In non-antithesis environments, this wrapper has no effect."
[gen]
(MainGen. gen))
;; Interpreter. This is adapted from jepsen.generator.interpreter, but uses
;; different control flow at the top level, because we're driven by the
;; *external* FIFO watcher, not by the generator directly.
(def max-pending-interval
; When the generator is :pending, this is how long we'll wait before looking
; at the generator again. In Antithesis, spinning tightly on the generator
; looks busy and can prevent Antithesis from doing actual operations, so we
; wait a full millisecond here instead.
1000)
(defn interpret!
"Borrowed from jepsen.generator.interpreter, with adaptations for our FIFO
queue. Notably, the test now contains a :fifo-queue, which delivers
InterpreterMsgs to the interpreter thread. These messages are of two types:
:op Perform a single operation from the (presumably main phase) of the
generator. Delivers the [invoke, complete] pair to the message's
promise when done.
:check Enters the final phase of the generator. Flips test-phase to
:final, and consumes generator operations as quickly as possible
to perform the final phase. When these operations are exhausted,
the interpreter returns, triggering the final analysis.
Takes a test with a :store :handle open. Causes the test's reference to the
:generator to be forgotten, to avoid retaining the head of infinite seqs.
Opens a writer for the test's history using that handle. Creates an initial
context from test and evaluates all ops from (:gen test). Spawns a thread for
each worker, and hands those workers operations from gen; each thread applies
the operation using (:client test) or (:nemesis test), as appropriate.
Invocations and completions are journaled to a history on disk. Returns a new
test with no :generator and a completed :history.
Generators are automatically wrapped in friendly-exception and validate.
Clients are wrapped in a validator as well.
Automatically initializes the generator system, which, on first invocation,
extends the Generator protocol over some dynamic classes like (promise)."
[test]
(gen/init!)
(with-open [history-writer (store.format/test-history-writer!
(:handle (:store test))
test)]
(let [ctx (gen/context test)
worker-ids (gen/all-threads ctx)
^BlockingQueue fifo-queue (:fifo-queue test)
_ (assert (instance? BlockingQueue fifo-queue))
completions (ArrayBlockingQueue.
(.size ^ISet worker-ids))
workers (mapv (partial gi/spawn-worker test completions
(gi/client-nemesis-worker))
worker-ids)
invocations (into {} (map (juxt :id :in) workers))
gen (->> (:generator test)
deref
gen/friendly-exceptions
gen/validate)
phase (:antithesis-phase test)
; Forget generator
_ (util/forget! (:generator test))
test (dissoc test :generator)]
; HERE
(try+
(loop [ctx ctx
gen gen
op-index 0 ; Index of the next op in the history
outstanding 0 ; Number of in-flight ops
; How long to poll on the completion queue, in micros.
poll-timeout 0
; The FifoOp each thread is processing
fifo-ops (b/linear bm/empty)
; How long to poll on the fifo queue, in micros
fifo-timeout max-pending-interval]
; First, can we complete an operation? We want to get to these first
; because they're latency sensitive--if we wait, we introduce false
; concurrency.
(if-let [op' (.poll completions poll-timeout TimeUnit/MICROSECONDS)]
(let [;_ (prn :completed op')
thread (gen/process->thread ctx (:process op'))
time (util/relative-time-nanos)
; Update op with index and new timestamp
op' (assoc op' :index op-index :time time)
; Update context with new time and thread being free
ctx (gc/free-thread ctx time thread)
; Let generator know about our completion. We use the context
; with the new time and thread free, but *don't* assign a new
; process here, so that thread->process recovers the right
; value for this event.
gen (gen/update gen test ctx op')
; Threads that crash (other than the nemesis), or which
; explicitly request a new process, should be assigned new
; process identifiers.
ctx (if (and (not= :nemesis thread)
(or (= :info (:type op'))
(:end-process? op')))
(gc/with-next-process ctx thread)
ctx)
; The FifoOp to complete
fifo-op (bm/get fifo-ops thread nil)
fifo-ops (bm/remove fifo-ops thread)]
; Log completion and move on
(if (gi/goes-in-history? op')
(do (store.format/append-to-big-vector-block!
history-writer op')
(when fifo-op (complete-fifo-op! fifo-op 0 (pr-str op')))
(recur ctx gen (inc op-index) (dec outstanding) 0 fifo-ops
fifo-timeout))
(do (when fifo-op (complete-fifo-op! fifo-op 0 (pr-str op')))
(recur ctx gen op-index (dec outstanding) 0 fifo-ops
fifo-timeout))))
; There's nothing to complete; let's see what the generator's up to
(let [time (util/relative-time-nanos)
ctx (assoc ctx :time time)
[op gen'] (gen/op gen test ctx)]
;(info :op op)
(cond
; We're exhausted, but workers might still be going.
(nil? op)
(if (pos? outstanding)
; Still waiting on workers
(recur ctx gen op-index outstanding
(long max-pending-interval) fifo-ops
fifo-timeout)
; Good, we're done. Tell workers to exit...
(do (doseq [[thread queue] invocations]
(.put ^ArrayBlockingQueue queue {:type :exit}))
; Wait for exit
(dorun (map (comp deref :future) workers))
; Await completion of writes
(.close history-writer)
; And return history
(let [history-block-id (:block-id history-writer)
history
(-> (:handle (:store test))
(store.format/read-block-by-id
history-block-id)
:data
(h/history {:dense-indices? true
:have-indices? true
:already-ops? true}))]
(assoc test :history history))))
; Nothing we can do right now. Let's try to complete something.
(identical? :pending op)
(recur ctx gen op-index
outstanding (long max-pending-interval) fifo-ops
fifo-timeout)
; Good, the generator has an invocation for us.
; But... it's in the future, so we have to wait.
(< time (:time op))
(recur ctx gen op-index outstanding
; Unless something changes, we don't need to ask
; the generator for another op until it's time.
(long (/ (- (:time op) time) 1000))
fifo-ops
fifo-timeout)
; We have an invocation and it's ready.
true
; We need to wait for a FIFO op to release us.
(let [fifo-op (when (identical? :main @phase)
; Once we're done with the main phase, we don't
; care about FIFO ops any more.
(.poll fifo-queue fifo-timeout
TimeUnit/MICROSECONDS))]
(cond
; As a special case, the fifo op "check" ends the main
; phase.
(= "check" (:name fifo-op))
; Gosh this is a gross hack. We're just going to shove the
; fifo op into the phase, so it can be completed when the
; checker is done.
(do (info "Antithesis requested check; main phase ending...")
(reset! phase fifo-op)
(recur ctx gen op-index outstanding poll-timeout
fifo-ops 0))
; We're in the main phase and are still waiting on the
; FIFO. Let's switch gears and check for a completion op.
(and (identical? :main @phase)
(nil? fifo-op))
(recur ctx gen op-index outstanding
(long max-pending-interval)
fifo-ops
(long max-pending-interval))
; We're either no longer in the main phase, or we are, and
; have a fifo op. Let's invoke!
true
(let [thread (gen/process->thread ctx (:process op))
op (assoc op :index op-index)
; Log the invocation
goes-in-history? (gi/goes-in-history? op)
_ (when goes-in-history?
(store.format/append-to-big-vector-block!
history-writer op))
op-index' (if goes-in-history? (inc op-index) op-index)
; Dispatch it to a worker
_ (.put ^ArrayBlockingQueue (get invocations thread) op)
; Update our context to reflect
ctx (gc/busy-thread ctx
(:time op) ; Use time instead?
thread)
; Let the generator know about the invocation
gen' (gen/update gen' test ctx op)
; Remember that this thread is servicing this fifo op
fifo-ops' (bm/put fifo-ops thread fifo-op)]
(recur ctx gen' op-index' (inc outstanding) 0 fifo-ops'
fifo-timeout))))))))
(catch Throwable t
; We've thrown, but we still need to ensure the workers exit.
(info "Shutting down workers after abnormal exit")
; We only try to cancel each worker *once*--if we try to cancel
; multiple times, we might interrupt a worker while it's in the
; finally block, cleaning up its client.
(dorun (map (comp future-cancel :future) workers))
; If for some reason *that* doesn't work, we ask them all to exit via
; their queue.
(loop [unfinished workers]
(when (seq unfinished)
(let [{:keys [in future] :as worker} (first unfinished)]
(if (future-done? future)
(recur (next unfinished))
(do (.offer ^java.util.Queue in {:type :exit})
(recur unfinished))))))
(throw t))))))
(defn run-case!
"Takes a test with a store handle. Spawns nemesis and clients, runs the
generator, and returns test with no :generator and a completed :history."
[test]
(jepsen/with-client+nemesis-setup-teardown [test test]
(interpret! test)))
(defn notify-test-checked!
"Complets the final check fifo op once the test is complete. Returns test
unchanged."
[test antithesis-phase]
(let [fifo-op @antithesis-phase]
(when (instance? IFifoOp fifo-op)
(complete-fifo-op! fifo-op 0 "checked")))
test)
(defn run!
"Runs a test, taking direction from fifo-watcher. See jepsen.core/run!"
[test]
(with-thread-name "jepsen test runner"
(let [antithesis-phase (atom :main)
test (-> test
jepsen/prepare-test
(assoc :antithesis-phase antithesis-phase
:fifo-queue (fifo-queue))
(update :nonserializable-keys conj
:antithesis-phase
:fifo-queue))]
(jepsen/with-logging test
(store/with-handle [test test]
; Initial save
(let [test (store/save-0! test)]
(util/with-relative-time
(with-fifo-watcher! test
(let [test (-> (run-case! test)
; Remove state
(dissoc :barrier
:sessions
:antithesis-phase))
_ (info "Run complete, writing")]
(-> test
store/save-1!
jepsen/analyze!
jepsen/log-results
(notify-test-checked! antithesis-phase)))))))))))
(defn antithesis-cmd
"A command package for jepsen.cli. Provides a new command, `lein run
antithesis`, which works like `lein run test`, but uses the Antithesis test
composer to drive execution. Options:
{:opt-spec A vector of additional options for tools.cli. Merge into
`test-opt-spec`. Optional.
:opt-fn A function which transforms parsed options. Composed after
`test-opt-fn`. Optional.
:opt-fn* Replaces test-opt-fn, in case you want to override it
altogether.
:usage Defaults to `jc/test-usage`. Optional.
:test-fn A function that receives the option map and constructs a test.}"
[opts]
(let [opt-spec (cli/merge-opt-specs cli/test-opt-spec (:opt-spec opts))
opt-fn (or (:opt-fn* opts) cli/test-opt-fn)
test-fn (:test-fn opts)]
{"antithesis"
{:opt-spec opt-spec
:opt-fn opt-fn
:usage (:usage opts cli/test-usage)
:run (fn [{:keys [options]}]
(info "Test options:\n"
(with-out-str (pprint options)))
(let [test (run! (test-fn options))]
(case (:valid? (:results test))
false (System/exit 1)
:unknown (System/exit 2)
nil)))}}))
================================================
FILE: antithesis/src/jepsen/antithesis.clj
================================================
(ns jepsen.antithesis
"Provides support for running Jepsen tests in Antithesis. Provides an RNG,
lifecycle hooks, and assertions.
## Randomness
You should wrap your entire program in `(with-rng ...)`. This does nothing in
ordinary environments, but in Antithesis, it replaces the jepsen.random RNG
with one powered by Antithesis.
## Lifecycle
Call `setup-complete!` once the test is ready to begin--for instance, at the
end of Client/setup. Call `event!` to signal interesting things have
happened.
## Assertions
Assertions begin with `assert-`, and take an expression, a message, and data
to include if the assertion fails. For instance:
(assert-always! (not (db-corrupted?)) \"DB corrupted\" {:db \"foo\"})
## Wrappers
Wrap your test map in `(a/test test-map)`. This wraps both the client and
checkers with Antithesis instrumentation. You can also do this selectively
using `checker` and `client`."
(:refer-clojure :exclude [test])
(:require [clojure [pprint :refer [pprint]]
[string :as str]]
[clojure.java.io :as io]
[clojure.tools.logging :refer [info]]
[jepsen [checker :as checker]
[client :as client]
[db :as db]
[generator :as gen]
[os :as os]
[random :as rand]])
(:import (com.antithesis.sdk Assert
Lifecycle)
(com.fasterxml.jackson.databind ObjectMapper)
(com.fasterxml.jackson.databind.node ArrayNode
ObjectNode)
(java.io Writer)
(java.util.random RandomGenerator
RandomGenerator$SplittableGenerator)
(jepsen.antithesis Random)))
(def ^ObjectMapper om (ObjectMapper.))
(defprotocol ToJsonNode
(->json-node [x] "Coerces x to a Jackson JsonNode."))
(extend-protocol ToJsonNode
clojure.lang.IPersistentMap
(->json-node [x]
(reduce (fn [^ObjectNode n, [k v]]
; my kingdom for a JSON with non-string keys
(.put n
(if (instance? clojure.lang.Named k)
(name k)
(str k))
(->json-node v))
n)
(.createObjectNode om)
x))
clojure.lang.IPersistentSet
(->json-node [x]
(reduce (fn [^ArrayNode a, e]
(.add a (->json-node e)))
(.createArrayNode om)
x))
clojure.lang.Sequential
(->json-node [x]
(reduce (fn [^ArrayNode a, e]
(.add a (->json-node e)))
(.createArrayNode om)
x))
clojure.lang.Keyword
(->json-node [x]
(name x))
clojure.lang.BigInt
(->json-node [x]
; Weirdly Jackson takes bigdecimals but not bigintegers
(bigdec x))
java.lang.Number
(->json-node [x] x)
java.lang.String
(->json-node [x] x)
java.lang.Boolean
(->json-node [x] x)
nil
(->json-node [x] nil))
(let [d (delay (System/getenv "ANTITHESIS_OUTPUT_DIR"))]
(defn dir
"The Antithesis SDK directory, if present, or nil."
[]
@d)
(defn log-file
"The Antithesis log file we write JSON to, or nil."
[]
(when-let [d @d]
(str d "/sdk.jsonl"))))
(defn antithesis?
"Are we running in an Antithesis environment?"
[]
(boolean (dir)))
;; Randomness
(defn replacement-double-weighted-index
"Antithesis replacement for jepsen.random/double-weighted-index. Takes a
double array, and picks a random index into it."
(^long [^doubles weights]
(replacement-double-weighted-index 0.0 weights))
(^long [^double total-weight ^doubles weights]
(.randomChoice ^Random rand/rng (range (alength weights)))))
(def choice-cardinality
"When selecting long values, we consider something an Antithesis \"choice\"
if it asks for at most this many elements."
16)
(defn replacement-long
"Antithesis replacement for `jepsen.random/long`. Mostly equivalent to the
original, but when there are less than `choice-cardinality` options, hints to
Antithesis that we're making a specific choice."
(^long [] (.nextLong rand/rng))
(^long [^long upper]
(if (< 0 upper choice-cardinality)
(.randomChoice ^Random rand/rng (range upper))
(.nextLong rand/rng upper)))
(^long [^long lower, ^long upper]
(if (< 0 (- upper lower) choice-cardinality)
(.randomChoice ^Random rand/rng (range lower upper))
(.nextLong rand/rng lower upper))))
(defn replacement-bool
"Antithesis replacement for `jepsen.random/bool`. Uses Antithesis choices
when probabilities are closer to chance than choice-cardinality."
([] (.nextBoolean ^Random rand/rng))
([^double p]
(if (< (/ choice-cardinality)
p
(- 1 (/ choice-cardinality)))
(.nextBoolean ^Random rand/rng)
(< (rand/double) p))))
(defmacro with-rng
"When running in an Antithesis environment, replaces Jepsen's random source
with an Antithesis-controlled source. You should wrap your top-level program
in this."
[& body]
`(rand/with-rng (if (antithesis?)
(Random.)
rand/rng)
(with-redefs [jepsen.random/double-weighted-index
(if (antithesis?)
replacement-double-weighted-index
rand/double-weighted-index)
jepsen.random/long
(if (antithesis?)
replacement-long
rand/long)
jepsen.random/bool
(if (antithesis?)
replacement-bool
rand/bool)]
~@body)))
;; Lifecycle
(defn setup-complete!
"Logs that we've started up. Only emits once per JVM run. Optionally takes a
JSON-coerceable structure with details."
([]
(setup-complete! nil))
([details]
(Lifecycle/setupComplete (->json-node details))))
(defn event!
"Logs that we've reached a specific event. Takes a string name and an
optional JSON-coerceable details map."
([name]
(event! name nil))
([name details]
(Lifecycle/sendEvent name (->json-node details))))
;; Assertions
(defmacro assert-always
"Asserts that expr is true every time, and that it's called at least once.
Takes a map of data which is serialized to JSON."
[expr message data]
`(Assert/always (boolean ~expr)
~message
(->json-node ~data)))
(defmacro assert-always-or-unreachable
"Asserts that expr is true every time. Passes even if the assertion never
triggers."
[expr message data]
`(Assert/alwaysOrUnreachable (boolean ~expr)
~message
(->json-node ~data)))
(defmacro assert-sometimes
"Asserts that expr is true at least once, and that this assertion is reached."
[expr message data]
`(Assert/sometimes (boolean ~expr)
~message
(->json-node ~data)))
(defmacro assert-unreachable
"Asserts that this line of code is never reached."
[message data]
`(Assert/unreachable ~message (->json-node ~data)))
(defmacro assert-reachable
"Asserts that this line of code is reached at least once."
[message data]
`(Assert/reachable ~message (->json-node ~data)))
;; Client
(defrecord Client [client]
client/Client
(open! [this test node]
(Client. (client/open! client test node)))
(setup! [this test]
(let [r (client/setup! client test)]
(setup-complete!)
r))
(invoke! [this test op]
; Every Jepsen run should do at *least* one invocation. We emit one of
; these for each kind of :f the client does. TODO: I'm not sure if we
; should provide details or not.
;
; TODO: is it acceptable for us to runtime generate the name for this
; assertion? My guess is that whatever magic instrumentation Antithesis'
; SDK is doing might expect compile-time constants here, but the SDK docs
; don't say.
(assert-reachable (str "invoke " (pr-str (:f op))) {})
(let [op' (client/invoke! client test op)]
; We'd like at least one invocation to succeed.
;
; TODO: is this OK? It feels like Antithesis could trivially generate a
; counterexample by just making the system unavailable, and that's not
; actually a bug. What SHOULD I be doing here?
(assert-sometimes (identical? :ok (:type op'))
(str "ok " (pr-str (:f op)))
; We'll at least log the type it came back with?
(select-keys op' [:type]))
op'))
(teardown! [this test]
(client/teardown! client test))
(close! [this test]
(client/close! client test)))
(defn client
"Wraps a Jepsen Client in one which performs some simple Antithesis
instrumentation. Calls `setup-complete!` after the client's setup is done,
and issues a pair of assertions for every operation--once on invocation, and
once on completion."
[client]
(if (antithesis?)
(Client. client)
client))
; We build these as protocols so that they can be extended *both* over the
; built-in checkers, and also by new checkers.
(defprotocol Checker
"This protocol marks checker types which perform their own Antithesis
assertions. The `checker+` wrapper skips over any checkers which satisfy this
protocol, as well as their children.")
(extend-protocol Checker
; The Stats checker will fail on histories with no :ok operations of some
; type, which Antithesis will sometimes generate.
jepsen.checker.Stats)
(defrecord AlwaysChecker [^String name checker]
Checker
checker/Checker
(check [this test history opts]
(let [r (checker/check checker test history opts)]
(assert-always (true? (:valid? r)) name r)
r)))
(defn checker
"Wraps a Jepsen Checker in one which asserts the results of the underlying
checker are always valid. Takes a string name, which defaults to \"checker\";
this is used for the Antithesis assertion."
([checker-]
(checker "checker" checker-))
([name checker]
(if (antithesis?)
(AlwaysChecker. name checker)
checker)))
(defn checker+
"Rewrites a Jepsen Checker, wrapping every Checker in its own Antithesis
Checker, which asserts validity of that specific checker. Each checker gets a
name derived from the path into that data structure it took to get there.
Ignores any object which satisfies the Checker protocol. You can use this to
flag checkers you don't want to add assertions for, or which implement their
own assertions."
([c]
(checker+ ["checker"] c))
([path c]
; (prn :rewrite path c)
(let [; We often rewrite singleton maps; in these cases, don't bother
; propagating paths. For example, a Compose has only a single key
; :checkers, and we don't need to generate names like "checker
; :checkers :cat"; we can just do "checker :cat".
branch? (and (coll? c) (< 1 (count c)))
; Rewrite child forms
c' (cond ; Already aware of Antithesis checking; don't explore any
; deeper
(satisfies? Checker c)
c
; For maps and records, rewrite each value, using the key as
; our path
(map? c)
(reduce (fn [c [k v]]
(assoc c k (checker+
(if branch?
(conj path (pr-str k))
path)
v)))
c
c)
; For sets, rewrite without changing path
(set? c)
(reduce (fn [c v]
(conj c (checker+ path v)))
(empty c)
c)
; For sequential collections, rewrite without changing path
(sequential? c)
(reduce (fn [c v]
(conj c (checker+ path v)))
(empty c)
c)
; Everything else bottoms out in itself
true
c)]
; Now, consider the rewritten thing.
(cond ; Our Checkers handle their own assertions.
(satisfies? Checker c')
c'
; If it's a Jepsen checker, wrap it, using the current path as our
; name.
(satisfies? checker/Checker c')
(checker (str/join " " path) c')
; Otherwise, return unchanged.
true
c'))))
(defrecord EarlyTerminationGen [^long interval
^double probability
^long remaining
gen]
gen/Generator
(op [this test ctx]
(if (and (= 0 remaining)
; This might be a little awkward if EarlyTerm winds up nested
; inside a branching structure; different calls to op would return
; different results. Probably fine, but might be worth
; stabilizing.
(rand/bool probability))
; Done
nil
; Keep going
(when-let [[op gen'] (gen/op gen test ctx)]
(if (identical? :pending op)
[:pending this]
[op (EarlyTerminationGen. interval
probability
(mod (dec remaining) interval)
gen')]))))
(update [this test ctx event]
(EarlyTerminationGen. interval probability remaining
(gen/update gen test ctx event))))
(defn early-termination-generator
"Wraps a Jepsen generator in one which, every so often, asks Jepsen's
random source whether it ought to terminate. This allows Antithesis to
perform a sort of weak 'online checking', as opposed to always running for
the full (e.g.) time limit. Options are a map of:
:interval How many operations to emit before flipping a coin
:probability The probability of early termination at each interval. This is
only useful for extremely low/high probabilities; everything in the moderate
range (see choice-cardinality) winds up being a coin toss."
[{:keys [interval probability]} gen]
(EarlyTerminationGen. (long interval)
(double (or probability 0.5))
(long interval)
gen))
(defn test
"Prepares a Jepsen test for running in Antithesis. When running inside
Antithesis, this:
1. Replaces the OS with a no-op
2. Repaces the DB with a no-op
3. Replaces the SSH system with a stub.
You likely also want to wrap the client in `client`, parts of the checker in
`checker`, and possibly the generator in `early-termination-generator`."
[test]
(if (antithesis?)
(assoc test
:os os/noop
:db db/noop
:ssh {:dummy? true})
test))
================================================
FILE: antithesis/test/jepsen/antithesis/composer_test.clj
================================================
(ns jepsen.antithesis.composer-test
(:refer-clojure :exclude [run!])
(:require [clojure [edn :as edn]
[pprint :refer [pprint]]
[test :refer :all]]
[clojure.java.shell :refer [sh]]
[clojure.tools.logging :refer [info warn]]
[jepsen [checker :as checker]
[client :as client]
[generator :as gen]
[history :as h]
[tests :as tests]]
[jepsen.antithesis.composer :as composer :refer [run!]]))
(defn op!
"Shells out to the fifo wrapper to perform a single op."
[]
(sh "composer/op"))
(defn check!
"Shells out to the fifo wrapper to perform the final check."
[]
(sh "composer/check"))
(defn make-test
"Constructs a new test map which records side effects in :effects, and which
delivers a promise to :setup. Call ((:await-setup test)) to block until all
clients have completed setup. Call ((:effects! test)) to read and clear side
effects. We use this in run!-test."
[]
(let [setup (promise)
effects (atom [])
; Returns effects and clears them, as a side effect.
effects! (fn effects! []
(let [ret (volatile! [])]
(swap! effects (fn [effects]
(vreset! ret effects)
[]))
@ret))
test tests/noop-test
n (count (:nodes test))
; Waits until effects have received n :setup's
await-setup (fn await-setup []
(loop []
(if (= n (count (filter #{:setup} @effects)))
:setup
(do (Thread/sleep 10)
(recur)))))
client (reify client/Client
(open! [this test node] this)
(setup! [this test]
(info "Setup")
(swap! effects conj :setup))
(invoke! [this test op]
(swap! effects conj [:op op])
(assoc op :type :ok))
(teardown! [this test]
(swap! effects conj :teardown))
(close! [this test]))
main-invokes [{:f :write}
{:f :read}]
final-invokes [{:f :final}]
gen (gen/clients
(gen/phases (composer/main-gen main-invokes)
final-invokes))
checker (reify checker/Checker
(check [this test history opts]
(swap! effects conj :check)
{:valid? (= [:write :read :final]
(->> (h/invokes history)
(map :f)))}))]
(assoc test
:name "composer test"
:client client
:generator gen
:checker checker
:effects effects
:effects! effects!
:await-setup await-setup
:nonserializable-keys [:effects :effects! :await-setup])))
(defn strip-time
"Strips out the :time field of an [:op op] element of an effects vector."
[effect]
(if (and (vector? effect) (= :op (first effect)))
(update effect 1 dissoc :time)
effect))
(defn strip-times
"Strips out :time fields from [:op op] elements of an effects vector."
[effects]
(mapv strip-time effects))
(deftest immediate-check-test
(let [{:keys [await-setup effects!] :as test} (make-test)
n (count (:nodes test))
runner (future (run! test))]
; We should do setup, then pause.
(await-setup)
(is (= (repeat n :setup) (effects!)))
; Immediate check
(is (= {:exit 0, :err "", :out "checked"} (check!)))
; We do to the final op, tear down, then check
(is (= (concat
[[:op {:index 0, :process 0, :type :invoke, :f :final, :value nil}]]
(repeat n :teardown)
[:check])
(strip-times (effects!))))
; And the test is invalid, because we didn't do both ops
(let [test' @runner]
(is (= {:valid? false} (:results test'))))))
(deftest early-check-test
(let [{:keys [await-setup effects!] :as test} (make-test)
n (count (:nodes test))
runner (future (run! test))]
; We should do setup, then pause.
(await-setup)
(is (= (repeat n :setup) (effects!)))
; Do a single op
(let [op' (op!)]
(is (= 0 (:exit op')))
(is (= {:index 1, :type :ok, :process 0, :f :write, :value nil}
(dissoc (edn/read-string (:out op')) :time))))
(is (= [[:op {:index 0, :process 0, :type :invoke, :f :write, :value nil}]]
(strip-times (effects!))))
; Early check
(is (= {:exit 0, :err "", :out "checked"} (check!)))
(is (= (concat
[[:op {:index 2, :process 1, :type :invoke, :f :final, :value nil}]]
(repeat n :teardown)
[:check])
(strip-times (effects!))))
; Test is invalid because we didn't finish the main gen
(let [test' @runner]
(is (= {:valid? false} (:results test'))))))
(deftest full-test
(let [{:keys [await-setup effects!] :as test} (make-test)
n (count (:nodes test))
runner (future (run! test))]
; We should do setup, then pause.
(await-setup)
(is (= (repeat n :setup) (effects!)))
; Do both ops
(let [op' (op!)]
(is (= 0 (:exit op')))
(is (= {:index 1, :type :ok, :process 0, :f :write, :value nil}
(dissoc (edn/read-string (:out op')) :time))))
(is (= [[:op {:index 0, :process 0, :type :invoke, :f :write, :value nil}]]
(strip-times (effects!))))
(let [op' (op!)]
(is (= 0 (:exit op')))
(is (= {:index 3, :type :ok, :process 1, :f :read, :value nil}
(dissoc (edn/read-string (:out op')) :time))))
(is (= [[:op {:index 2, :process 1, :type :invoke, :f :read, :value nil}]]
(strip-times (effects!))))
; Early check
(is (= {:exit 0, :err "", :out "checked"} (check!)))
(is (= (concat
[[:op {:index 4, :process 2, :type :invoke, :f :final, :value nil}]]
(repeat n :teardown)
[:check])
(strip-times (effects!))))
; Test is invalid because we didn't finish the main gen
(let [test' @runner]
(is (= {:valid? true} (:results test'))))))
================================================
FILE: antithesis/test/jepsen/antithesis_test.clj
================================================
(ns jepsen.antithesis-test
(:require [clojure [pprint :refer [pprint]]
[test :refer :all]]
[jepsen [antithesis :as a]
[checker :as checker]
[client :as client]
[generator :as gen]
[random :as rand]]
[jepsen.generator [context :as gen.ctx]
[test :as gen.test]]))
(deftest antithesis?-test
(is (false? (a/antithesis?))))
(deftest random-test
(testing "outside antithesis"
(rand/with-seed 0
(a/with-rng
(is (= 3247545035024278667 (rand/long)))
(is (= 0.07473821244042433 (rand/double))))))
(testing "in antithesis"
(with-redefs [a/antithesis? (constantly true)]
(rand/with-seed 0
(a/with-rng
(testing "nondeterministic"
; That we *don't* get the with-seed 0 values means we're using
; antithesis.Random
(is (not (= 3247545035024278667 (rand/long))))
(is (not (= 0.07473821244042433 (rand/double)))))
(testing "long"
(testing "saturation"
; Make sure we're flipping every possible Long bit
(loop [i 0
sat 0]
(if (= i 1000)
(is (= -1 sat)) ; -1 is 2r111...111
(recur (inc i)
(bit-or sat (rand/long))))))
(testing "upper"
(dotimes [i 1000]
(is (< (rand/long 5403) 5403))))
(testing "lower, upper"
(dotimes [i 1000]
(let [x (rand/long -5 -2)]
(is (and (<= -5 x)
(< x -2)))))))
(testing "double"
(let [xs (vec (take 1000 (repeatedly rand/double)))]
(testing "range"
(is (every? #(<= 0.0 % 1.0) xs)))
(testing "saturation"
(let [bits (map #(Double/doubleToRawLongBits %) xs)]
; Top two bits should be zero, otherwise we should saturate the
; mantissa. The other leftmost bits are all 1.
;(println (Long/toBinaryString (reduce bit-or 0 bits)))
(is (= (unsigned-bit-shift-right -1 2)
(reduce bit-or 0 bits)))))
(testing "upper"
(dotimes [i 1000]
(is (< (rand/double 5403) 5403))))
(testing "lower, upper"
(dotimes [i 1000]
(is (<= -10 (rand/double -5 -2) -2))))))
)))))
(deftest choices-test
; We redefine `jepsen.random`'s `long` and `double-weighted-index` so that they use the
; Antithesis randomChoice (for small choices) instead of weighted entropy.
; This should hopefully make Antithesis more efficient at exploring branches.
; Of course we have no way to TEST this outside Antithesis, since it just
; proxies back to j.u.Random, but we can at least make sure it's uniform
; rather than weighted.
(let [weights (double-array [0.0 1.0])]
(testing "stock"
(dotimes [i 100]
(is (== 1.0 (rand/double-weighted-index weights)))))
(testing "antithesis"
(with-redefs [a/antithesis? (constantly true)]
(a/with-rng
(testing "double-weighted-index"
(let [freqs (->> #(rand/double-weighted-index weights)
repeatedly
(take 1000)
frequencies)]
(is (= #{0 1} (set (keys freqs))))
(is (< (Math/abs (- (freqs 0 0) (freqs 1 0))) 100))))
(testing "long"
; Long's behavior should be uniform random either way, but we can
; at least check the range.
(doseq [i (range 1 100)]
(is (< (rand/long i) i))
; Just adding some offset (41) to check long's lower/upper bounds
(is (< 40 (rand/long 41 (+ i 41)) (+ i 41)))))
(testing "bool"
; Weighted booleans should stop being weighted when Antithesis takes over
(let [freqs (->> #(rand/bool 1/10)
repeatedly
(take 1000)
frequencies)]
(is (< (Math/abs (- (freqs true 0) (freqs false 0))) 100)))
; Unless their probability is extreme
(let [freqs (->> #(rand/bool 1/100)
repeatedly
(take 1000)
frequencies)]
(is (< (freqs true 0) 50)))))))))
(deftest client-test
; Clients should inform us they started, and also make assertions about
; invocations.
(with-redefs [a/antithesis? (constantly true)]
(let [side (atom [])
client (reify client/Client
(open! [this test node]
(swap! side conj :open)
this)
(setup! [this test]
(swap! side conj :setup))
(invoke! [this test op]
(swap! side conj :invoke)
(assoc op :type :ok))
(teardown! [this test]
(swap! side conj :teardown))
(close! [this test]
(swap! side conj :close)))
client' (a/client client)]
; Since assertions are macros, not sure how to test them.
(let [sc! a/setup-complete!]
(with-redefs [a/setup-complete! (fn
([] (a/setup-complete! nil))
([details] (swap! side conj :setup-complete) (sc! details)))]
(let [client (client/open! client' {} "n1")]
(is (= [:open] @side))
(client/setup! client' {})
(is (= [:open :setup :setup-complete] @side))
(is (= :ok (:type (client/invoke! client' {} {:type :invoke}))))
(client/teardown! client' {})
(client/close! client {})
(is (= [:open :setup :setup-complete :invoke :teardown :close]
@side))))))))
(deftest checker-test
; Checkers should assert validity. Again, I have no way to test this, since
; they're macros, but we can at least guarantee they proxy through.
(with-redefs [a/antithesis? (constantly true)]
(let [side (atom [])
checker (reify checker/Checker
(check [this test history opts]
(swap! side conj :check)
{:valid? true}))
checker' (a/checker checker)]
(is (= {:valid? true} (checker/check checker' {} [] {})))
(is (= [:check] @side)))))
(deftest checker+-test
; If we nest one checker in another via e.g. `compose`, each one should get a
; distinct wrapper.
(with-redefs [a/antithesis? (constantly true)]
(let [side (atom [])
cat (reify checker/Checker
(check [this test history opts]
(swap! side conj :cat)
{:type :cat, :valid? true}))
dog (reify checker/Checker
(check [this test history opts]
(swap! side conj :dog)
{:type :dog, :valid? :unknown}))
checker (checker/compose {:cat cat, :dog dog})
checker' (a/checker+ checker)
; We can ask for instance? a.Checker, but that breaks during hot code
; reload
ac? (fn [x]
(is (= "jepsen.antithesis.AlwaysChecker"
(-> x class .getName))))]
;(prn)
;(println "-----")
;(prn)
;(pprint checker')
; Obviously
(is (ac? checker'))
; But also
(is (ac? (-> checker' :checker :checkers :cat)))
(is (ac? (-> checker' :checker :checkers :dog)))
; Paths should be nice and clean
(is (= "checker :cat" (-> checker' :checker :checkers :cat :name)))
(is (= "checker :dog" (-> checker' :checker :checkers :dog :name)))
; This should still work like a checker
(is (= {:valid? :unknown
:cat {:type :cat, :valid? true}
:dog {:type :dog, :valid? :unknown}}
(checker/check checker' {} [] {}))))))
(deftest early-termination-generator-test
; In Antithesis mode, generators terminate randomly.
(with-redefs [a/antithesis? (constantly true)]
(let [gen (->> (range)
(map (fn [x] {:f :write, :value x}))
(a/early-termination-generator {:interval 10 :probability 0.5})
gen/clients)
trials (rand/with-seed 555
(mapv (fn [i]
(->> gen
gen.test/perfect
count))
(range 100)))]
; Frequency distribution of history lengths falls off exponentially
(is (= {10 79, 20 15, 30 5, 40 1}
(frequencies trials))))))
================================================
FILE: charybdefs/.gitignore
================================================
/target
/classes
/checkouts
pom.xml
pom.xml.asc
*.jar
*.class
/.lein-*
/.nrepl-port
.hgignore
.hg/
================================================
FILE: charybdefs/README.md
================================================
# charybdefs
A wrapper around [CharybdeFS](https://github.com/scylladb/charybdefs) for use
in a jepsen nemesis.
## Usage
TODO
## License
Distributed under the Eclipse Public License either version 1.0 or (at
your option) any later version.
================================================
FILE: charybdefs/project.clj
================================================
(defproject jepsen-charybdefs "0.1.0-SNAPSHOT"
:description "charybdefs wrapper for use in jepsen"
:url "https://github.com/jepsen.io/jepsen"
:license {:name "Eclipse Public License"
:url "http://www.eclipse.org/legal/epl-v10.html"}
:dependencies [[org.clojure/clojure "1.8.0"]
[jepsen "0.1.6"]
[yogthos/config "0.8"]])
================================================
FILE: charybdefs/src/jepsen/charybdefs.clj
================================================
(ns jepsen.charybdefs
(:require [clojure.tools.logging :refer [info]]
[jepsen.control :as c]
[jepsen.control.util :as cu]
[jepsen.os.debian :as debian]))
(defn install-thrift!
"Install thrift (compiler, c++, and python libraries) from source.
Ubuntu includes thrift-compiler and python-thrift packages, but
not the c++ library, so we must build it from source. We can't mix
and match versions, so we must install everything from source."
[]
(when-not (cu/exists? "/usr/bin/thrift")
(c/su
(debian/install [:automake
:bison
:flex
:g++
:git
:libboost-all-dev
:libevent-dev
:libssl-dev
:libtool
:make
:pkg-config
:python-setuptools
:libglib2.0-dev])
(info "Building thrift (this takes several minutes)")
(let [thrift-dir "/opt/thrift"]
(cu/install-archive! "http://www-eu.apache.org/dist/thrift/0.10.0/thrift-0.10.0.tar.gz" thrift-dir)
(c/cd thrift-dir
;; charybdefs needs this in /usr/bin
(c/exec "./configure" "--prefix=/usr")
(c/exec :make :-j4)
(c/exec :make :install))
(c/cd (str thrift-dir "/lib/py")
(c/exec :python "setup.py" :install))))))
(defn install!
"Ensure CharybdeFS is installed and the filesystem mounted at /faulty."
[]
(install-thrift!)
(let [charybdefs-dir "/opt/charybdefs"
charybdefs-bin (str charybdefs-dir "/charybdefs")]
(when-not (cu/exists? charybdefs-bin)
(c/su
(debian/install [:build-essential
:cmake
:libfuse-dev
:fuse]))
(c/su
(c/exec :mkdir :-p charybdefs-dir)
(c/exec :chmod "777" charybdefs-dir))
(c/exec :git :clone :--depth 1 "https://github.com/scylladb/charybdefs.git" charybdefs-dir)
(c/cd charybdefs-dir
(c/exec :thrift :-r :--gen :cpp :server.thrift)
(c/exec :cmake :CMakeLists.txt)
(c/exec :make)))
(c/su
(c/exec :modprobe :fuse)
(c/exec :umount "/faulty" "||" "/bin/true")
(c/exec :mkdir :-p "/real" "/faulty")
(c/exec charybdefs-bin "/faulty" "-oallow_other,modules=subdir,subdir=/real")
(c/exec :chmod "777" "/real" "/faulty"))))
(defn- cookbook-command
[flag]
(c/cd "/opt/charybdefs/cookbook"
(c/exec "./recipes" flag)))
(defn break-all
"All operations fail with EIO."
[]
(cookbook-command "--io-error"))
(defn break-one-percent
"1% of disk operations fail."
[]
(cookbook-command "--probability"))
(defn clear
"Clear a previous failure injection."
[]
(cookbook-command "--clear"))
================================================
FILE: charybdefs/test/.gitignore
================================================
config.edn
================================================
FILE: charybdefs/test/jepsen/charybdefs/remote_test.clj
================================================
(ns jepsen.charybdefs.remote-test
(:require [clojure.test :refer :all]
[jepsen.control :as c]
[jepsen.charybdefs :as charybdefs]
[config.core :refer [env]]))
;;; To run these tests, create a file config.edn in ../.. (the 'test' directory)
;;; containing at least:
;;; {:hostname "foo"}
;;; This must name a host you can ssh to without a password, and have passwordless
;;; sudo on. It must run debian or ubuntu (tested with ubuntu 16.04).
;;; Other ssh options may also be used.
(use-fixtures :each
(fn [f]
(if (not (:hostname env))
(throw (RuntimeException. "hostname is required in config.edn")))
(c/with-ssh (select-keys env [:private-key-path :strict-host-key-checking :username])
(c/on (:hostname env)
(charybdefs/install!)
(f)))))
(deftest break-fix
(testing "break the disk and then fix it"
(let [filename "/faulty/foo"]
(c/exec :touch filename)
(charybdefs/break-all)
(is (thrown? RuntimeException (c/exec :cat filename)))
(charybdefs/clear)
(is "" (c/exec :cat filename)))))
================================================
FILE: contributing.md
================================================
# Contributing to Jepsen
Hi there, and thanks for helping make Jepsen better! I've got just one request:
start your commit messages with the *part* of Jepsen you're changing. For
instance, if I made a change to the MongoDB causal consistency tests:
> MongoDB causal: fix a bug when analyzing zero-length histories
Namespaces are cool too!
> jepsen.os.debian: fix libzip package name for debian stretch
If you're making a chance to the core Jepsen library, as opposed to a specific
database test, you can be more concise:
> add test for single nemesis events
Jepsen's a big project with lots of moving parts, and it can be confusing to
read the commit logs. Giving a bit of context makes my life a lot easier.
Thanks!
================================================
FILE: doc/color.md
================================================
Color scheme:
Light:
ok #6DB6FE
info #FFAA26
fail #FEB5DA
Dark:
ok: #81BFFC
info: #FFA400
fail: #FF1E90
================================================
FILE: doc/lxc-f36.md
================================================
# How to set up nodes via LXC
## Fedora 36
As a user can be sudoer:
```
sudo dnf install -y openssh-server
sudo dnf -y install clusterssh
sudo dnf -y install dnsmasq
sudo dnf install lxc lxc-templates lxc-extra debootstrap libvirt perl gpg
sudo dnf -y install bridge-utils libvirt virt-install qemu-kvm
sudo dnf install libvirt-devel virt-top libguestfs-tools guestfs-tools
```
Then, run all ``systemctl start/enable`` as you need:
```
sudo systemctl start sshd.service
sudo systemctl enable sshd.service
sudo systemctl start lxc.service
sudo systemctl enable lxc.service
sudo systemctl start libvirtd.service
sudo systemctl enable libvirtd.service
sudo systemctl start dnsmasq.service
sudo systemctl enable dnsmasq.service
<...> so on <..>
```
Watch out libvirtd and kvm messages...
Apply google and kernel parameters until checkconfig passes:
```
lxc-checkconfig
```
Well, here we go, problems are comming..
```
[root@fedora ~]# lxc-checkconfig
LXC version 4.0.12
Kernel configuration not found at /proc/config.gz; searching...
Kernel configuration found at /boot/config-5.18.16-200.fc36.x86_64
--- Namespaces ---
Namespaces: enabled
Utsname namespace: enabled
Ipc namespace: enabled
Pid namespace: enabled
User namespace: enabled
Warning: newuidmap is not setuid-root
Warning: newgidmap is not setuid-root
Network namespace: enabled
--- Control groups ---
Cgroups: enabled
Cgroup namespace: enabled
Cgroup v1 mount points:
Cgroup v2 mount points:
/sys/fs/cgroup
Cgroup v1 systemd controller: missing
Cgroup v1 freezer controller: missing
Cgroup ns_cgroup: required
Cgroup device: enabled
Cgroup sched: enabled
Cgroup cpu account: enabled
Cgroup memory controller: enabled
Cgroup cpuset: enabled
--- Misc ---
Veth pair device: enabled, loaded
Macvlan: enabled, not loaded
Vlan: enabled, not loaded
Bridges: enabled, loaded
Advanced netfilter: enabled, not loaded
CONFIG_IP_NF_TARGET_MASQUERADE: enabled, not loaded
CONFIG_IP6_NF_TARGET_MASQUERADE: enabled, not loaded
CONFIG_NETFILTER_XT_TARGET_CHECKSUM: enabled, loaded
CONFIG_NETFILTER_XT_MATCH_COMMENT: enabled, not loaded
FUSE (for use with lxcfs): enabled, loaded
--- Checkpoint/Restore ---
checkpoint restore: enabled
CONFIG_FHANDLE: enabled
CONFIG_EVENTFD: enabled
CONFIG_EPOLL: enabled
CONFIG_UNIX_DIAG: enabled
CONFIG_INET_DIAG: enabled
CONFIG_PACKET_DIAG: enabled
CONFIG_NETLINK_DIAG: enabled
File capabilities:
Note : Before booting a new kernel, you can check its configuration
usage : CONFIG=/path/to/config /usr/bin/lxc-checkconfig
```
Ok,
``Cgroup ns_cgroup: required`` forget about it, it is problem about versions...
Fedora 36 use as default cgroups2, if you are running v1, sorry, and disable selinux, it is just testing...
``cgroup_no_v1=all selinux=0``
Create VMs arch amd64, 3 of them :
```
for i in {1..3}; do sudo lxc-create -t download -n n$i -- -d fedora -r 36 -a amd64; done
```
Ok, add network configuration to each node. We assign each a sequential MAC
address.
Right, but you need known where you are, ``brctl show ``
```
bridge name bridge id STP enabled interfaces
docker0 8000.02425aa0dc72 no
lxcbr0 8000.00163e000000 no
virbr0 8000.525400bb5910 yes vethDcihoW
```
ok, ``virbr0`` seems our winner ...
Then
```
for i in {1..3}; do
sudo cat >>/var/lib/lxc/n${i}/config <<EOF
# Network config
lxc.net.0.type = veth
lxc.net.0.flags = up
lxc.net.0.link = virbr0
lxc.net.0.hwaddr = 00:1E:62:AA:AA:$(printf "%02x" $i)
EOF
done
```
Set up the virsh network bindings mapping those MAC addresses to hostnames and
IP addresses:
```
for i in {1..3}; do
virsh net-update --current default add-last ip-dhcp-host "<host mac=\"00:1E:62:AA:AA:$(printf "%02x" $i)\" name=\"n${i}\" ip=\"192.168.122.1$(printf "%02d" $i)\"/>"
done
```
Start the network, and set it to start at boot so the dnsmasq will be
available.
Well, we need know what libvirtd service is doing ...
```
[root@fedora ~]# systemctl status libvirtd
○ libvirtd.service - Virtualization daemon
Loaded: loaded (/usr/lib/systemd/system/libvirtd.service; disabled; vendor preset: disabled)
Active: inactive (dead) since Mon 2022-08-08 09:10:49 UTC; 1h 19min ago
TriggeredBy: ● libvirtd-ro.socket
○ libvirtd-tls.socket
○ libvirtd-tcp.socket
● libvirtd-admin.socket
● libvirtd.socket
Docs: man:libvirtd(8)
https://libvirt.org
Process: 2678 ExecStart=/usr/sbin/libvirtd $LIBVIRTD_ARGS (code=exited, s
```
Then, next steps
```
[root@fedora ~]# virsh net-autostart default;
Network default marked as autostarted
```
So, continue,
```
virsh net-start default
```
you get,
```
error: Failed to start network default
error: Requested operation is not valid: network is already active
```
fedora 36, libvirtd.socket is managing libvirtd-tls.socket and libvirtd-tcp.socket ...
If you configure resolv.conf by hand, add the libvirt local dnsmasq to
resolv.conf:
```
echo -e "nameserver 192.168.122.1\n$(cat /etc/resolv.conf)" > /etc/resolv.conf
```
If you're letting dhclient manage it, then:
```
echo "prepend domain-name-servers 192.168.122.1;" >>/etc/dhcp/dhclient.conf
systemctl restart NetworkManager
```
And update ``/etc/hosts`` if you don't wanna lose your mind seeing ``ping`` fails...
```
192.168.122.101 n1
192.168.122.102 n2
192.168.122.103 n3
```
Slip your preferred SSH key into each node's `.authorized-keys`:
```
for i in {1..3}; do
mkdir -p /var/lib/lxc/n${i}/rootfs/root/.ssh
chmod 700 /var/lib/lxc/n${i}/rootfs/root/.ssh/
cp ~/.ssh/id_rsa.pub /var/lib/lxc/n${i}/rootfs/root/.ssh/authorized_keys
chmod 644 /var/lib/lxc/n${i}/rootfs/root/.ssh/authorized_keys
done
```
And start the nodes:
```
for i in {1..3}; do
lxc-start -d -n n$i
done
```
To stop them:
```
for i in {1..3}; do
lxc-stop -n n$i
done
```
To check them:
```
lxc-ls -f
NAME STATE AUTOSTART GROUPS IPV4 IPV6 UNPRIVILEGED
n1 RUNNING 0 - 192.168.122.101 - false
n2 RUNNING 0 - 192.168.122.102 - false
n3 RUNNING 0 - 192.168.122.103 - false
```
Reset the root passwords to whatever you like. Jepsen uses `root` by default,
and allow root logins with passwords on each container. If you've got an SSH
agent set up, Jepsen can use that instead.
<b>Warning !!!</b>
Be sure that sshd service is running -ok- in {n1,n2,n3} without problems ...
``lxc-attach -n {n1, n2, n3} ``
Be sure that you change <u> "sshd_config" </u>
```
for i in {1..3}; do
lxc-attach -n n${i} -- bash -c 'echo -e "root\nroot\n" | passwd root';
lxc-attach -n n${i} -- sed -i 's,^#\?PermitRootLogin .*,PermitRootLogin yes,g' /etc/ssh/sshd_config;
lxc-attach -n n${i} -- systemctl restart sshd;
done
```
Store the host keys unencrypted so that jsch can use them. If you already have
the host keys, they may be unreadable to Jepsen--remove them from .known_hosts
and rescan.
```
for n in {1..3}; do ssh-keyscan -vvv -t rsa n$n; done >> ~/.ssh/known_hosts
```
Check it:
```
[root@fedora ~]# cat .ssh/known_hosts
n1 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCzqHvG5GEmX8RaALxTZT22fX1hDsxljAPH/m/6=
n2 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC+taCDJSyJbu5oaRK/zTFu+CvqOx0MRkvzXfoM=
n3 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDcpQcG9GUjYZqM5dA5LwSoCH6Qot42mbsdXwva=
```
If you get some problems as "write failed or similars", check again sshd, you got some wrong...
And check that you can SSH to the nodes:
fedora 36 needs,
```
dnf -y install clusterssh
```
If you don't have X mode, graphical mode... running
```
cssh n1 n2 n3
```
you should be sometime similar :
```
[root@fedora ~]# cssh n1
Can't connect to display `localhost:0': Connection refused at /usr/share/perl5/vendor_perl/X11/Protocol.pm line 2269.
```
Not use cssh, use ssh in a loop.
--- O ---
================================================
FILE: doc/lxc.md
================================================
# How to set up nodes via LXC
#### [Mint 22.3 Zena](#mint-22.3-zena-1)
#### [Debian 13/trixie - Incus](#debian-13trixie---incus-1)
For further information, [LXD - Debian Wiki](https://wiki.debian.org/LXD).
----
## Mint 22.3 Zena
Install LXC and DNSMasq:
```sh
sudo apt install lxc lxc-templates libvirt-clients dnsmasq
```
Update the old GPG keys for debian releases
```sh
cd /tmp
wget "https://ftp-master.debian.org/keys/archive-key-13.asc"
sudo gpg --no-default-keyring --keyring=/etc/apt/trusted.gpg.d/debian-archive-trixie-stable.gpg --import archive-key-13.asc
```
Set up a ZFS filesystem for containers. These are throwaway so I don't bother
with sync or atime.
```sh
sudo zfs create -o acltype=posix -o atime=off -o sync=disabled -o mountpoint=/var/lib/lxc rpool/lxc
```
If you've got Docker installed, it creates a whole bunch of firewall gunk that
totally breaks the LXC bridge. Make a script to let LXC talk:
```sh
sudo bash -c "cat >/usr/local/bin/post-docker.sh <<EOF
#!/usr/bin/env bash
date > /var/log/post-docker-timestamp
iptables -I DOCKER-USER -i lxcbr0 -j ACCEPT
iptables -I DOCKER-USER -o lxcbr0 -j ACCEPT
EOF"
sudo chmod +x /usr/local/bin/post-docker.sh
```
And call it after Docker loads:
```sh
sudo bash -c "cat >/etc/systemd/system/post-docker.service <<EOF
[Unit]
Description=Post Docker
After=docker.service
BindsTo=docker.service
ReloadPropagatedFrom=docker.service
[Service]
Type=oneshot
ExecStart=/usr/local/bin/post-docker.sh
ExecReload=/usr/local/bin/post-docker.sh
RemainAfterExit=yes
[Install]
WantedBy=multi-user.target
EOF"
sudo systemctl daemon-reload
sudo systemctl enable post-docker.service
```
I think UFW might also interfere? See https://linuxcontainers.org/incus/docs/main/howto/network_bridge_firewalld/#prevent-connectivity-issues-with-incus-and-docker
```
sudo ufw allow in on lxcbr0
sudo ufw route allow in on lxcbr0
sudo ufw route allow out on lxcbr0
```
Create containers.
```sh
# To destroy: for i in {1..10}; do sudo lxc-destroy --force -n n$i; done
for i in {1..10}; do sudo lxc-create -n n$i -t debian -- --release trixie; done
```
Uncomment this line in `/etc/default/lxc-net` to allow DHCP address reservations:
```sh
LXC_DHCP_CONFILE=/etc/dnsmasq.conf
```
Uncomment `bind-interfaces` in `/etc/dnsmasq.conf`, because otherwise
systemd-resolved will fight it. Also uncomment `conf-dir`:
```
bind-interfaces
conf-dir=/etc/dnsmasq.d
```
But disable the actual dnsmasq service, because lxc-net will run it.
```
sudo systemctl stop dnsmasq
sudo systemctl disable dnsmasq
```
Disable the systemd-resolved stub listener in `/etc/systemd/resolved.conf`, and
search the .lxc domain. Ignore resolv.conf, and instead hardcode your preferred
upstream DNS resolver (mine is 10.0.0.1).
```
...
DNSStubListener=no
DOMAINS=lxc,...
no-resolv
server=10.0.0.1
```
Add DHCP reservations for the nodes. The 10.0.3.xxx here should line up with
the network address on `lxcbr0`; check `ifconfig lxcbr0`.
```
for i in {1..10}; do sudo bash -c "echo 'dhcp-host=n$i,10.0.3.1$(printf "%02d" $i)' >> /etc/dnsmasq.d/jepsen.conf"; done
```
Restart the resolver and LXC networking
```
sudo systemctl restart systemd-resolved
sudo systemctl restart lxc-net
```
Start up a container and confirm that it's assigned the right address:
```
sudo lxc-start n1
sudo lxc-ls --fancy
sudo lxc-stop n1
```
And add the local dnsmasq to networkmanager
```sh
sudo bash -c "cat >/etc/NetworkManager/dnsmasq.d/lxc.conf <<EOF
server=/lxc/10.0.3.1
EOF"
```
Insist that NetworkManager use dnsmasq, *not* the public DNS. Get the long UUID
here from `nmcli con`.
```sh
sudo nmcli con mod d41034c4-48d0-3867-922c-73480603ff2e ipv4.ignore-auto-dns yes
sudo nmcli con mod d41034c4-48d0-3867-922c-73480603ff2e ipv4.dns "10.0.3.1"
```
Restart networking so that takes effect, and/or bounce the interface
```sh
sudo systemctl restart NetworkManager
sudo nmcli con down d41034c4-48d0-3867-922c-73480603ff2e
sudo nmcli con up d41034c4-48d0-3867-922c-73480603ff2e
```
At this juncture `cat /etc/resolv.conf` should show only 10.0.3.1, the local
dnsmasq. Dig `google.com` should still resolve using your upstream resolver.
Set up resolved to reference the nodes. Check your interface name and address
in `ip addr`; they may vary. This miiiight be optional with the above nmcli
settings.
```sh
sudo bash -c "cat >/etc/systemd/system/lxc-dns-lxcbr0.service <<EOF
[Unit]
Description=LXC DNS configuration for lxcbr0
After=lxc-net.service
[Service]
Type=oneshot
ExecStart=/usr/bin/resolvectl dns lxcbr0 10.0.3.1
ExecStart=/usr/bin/resolvectl domain lxcbr0 '~lxc'
ExecStopPost=/usr/bin/resolvectl revert lxcbr0
RemainAfterExit=yes
[Install]
WantedBy=lxc-net.service
EOF"
sudo systemctl daemon-reload
sudo systemctl enable lxc-dns-lxcbr0.service
sudo systemctl start lxc-dns-lxcbr0.service
```
Start nodes
```sh
for i in {1..10}; do
sudo lxc-start -d -n n$i
done
```
[This](https://thelinuxcode.com/how-to-configure-dns-on-linux-2026-systemd-resolved-networkmanager-resolvconf-and-bind/)
is a helpful guide to fixing resolv.conf/resolved/dnsmasq/NetworkManager
issues. You may need to fully restart too.
Copy your SSH key to nodes and set their passwords to something trivial
```sh
export YOUR_SSH_KEY=~/.ssh/id_rsa.pub
for i in {1..10}; do
sudo mkdir -p /var/lib/lxc/n${i}/rootfs/root/.ssh &&
sudo chmod 700 /var/lib/lxc/n${i}/rootfs/root/.ssh/ &&
sudo cp $YOUR_SSH_KEY /var/lib/lxc/n${i}/rootfs/root/.ssh/authorized_keys &&
sudo chmod 644 /var/lib/lxc/n${i}/rootfs/root/.ssh/authorized_keys &&
## Set root password
sudo lxc-attach -n n${i} -- bash -c 'echo -e "root\nroot\n" | passwd root';
sudo lxc-attach -n n${i} -- sed -i 's,^#\?PermitRootLogin .*,PermitRootLogin yes,g' /etc/ssh/sshd_config;
sudo lxc-attach -n n${i} -- systemctl restart sshd;
done
```
Install sudo on nodes; this is a base Jepsen dependency
```sh
for i in {1..10}; do
sudo lxc-attach -n n${i} -- apt install -y sudo
done
```
Scan node SSH keys (as whatever user you'll run Jepsen as)
```sh
for n in {1..10}; do
ssh-keyscan -t rsa n$n &&
ssh-keyscan -t ed25519 n$n
done >> ~/.ssh/known_hosts
```
At this point you should be able to `ssh n1` without a password.
----
## Debian 13/trixie - Incus
Due to Canonical's re-licensing and imposing of a CLA, the last version of Debian to include LXD will be trixie. Users are encouraged to migrate to Incus after upgrading from bookworm to trixie.
### Install Host Packages
```bash
sudo apt update && sudo apt install incus systemd-resolved
```
### Initialize Incus
```bash
# add yourself to the incus-admin group to avoid having to be root or sudo
# you will need to logout/login for new group to be active
sudo adduser $USER incus-admin
# initialize Incus with default config, defaults are usually OK
incus admin init --minimal
# try creating a sample container if you want
incus launch images:debian/13 scratch
incus list
incus shell scratch
incus stop scratch
incus delete scratch
```
### Create and Start Jepsen's Node Containers
```bash
for i in {1..10}; do
incus launch images:debian/13 n${i};
done
```
### Confirm Incus' Bridge Network
`incus init` automatically created the bridge network, and `incus launch` automatically configured the containers for it:
```bash
incus network list
+----------+----------+---------+----------------+---+-------------+---------+---------+
| NAME | TYPE | MANAGED | IPV4 |...| DESCRIPTION | USED BY | STATE |
+----------+----------+---------+----------------+---+-------------+---------+---------+
| incusbr0 | bridge | YES | 10.242.68.1/24 |...| | 11 | CREATED |
+----------+----------+---------+----------------+---+-------------+---------+---------+
# confirm your settings
incus network get incusbr0 ipv4.address
incus network get incusbr0 ipv6.address
incus network get incusbr0 dns.domain # will be blank if default incus config is used
# confirm containers are reachable
ping n1
PING n1 (10.242.68.40) 56(84) bytes of data.
64 bytes from n1 (10.242.68.40): icmp_seq=1 ttl=64 time=0.030 ms
...
```
If you want to install and run Docker, it may mess up your networking and firewall. If your Incus containers are not reachable from the host, see the Incus documentation:
- [Prevent connectivity issues with Incus and Docker](https://linuxcontainers.org/incus/docs/main/howto/network_bridge_firewalld/#prevent-connectivity-issues-with-incus-and-docker)
#### Add Required Packages to Node Containers
```bash
for i in {1..10}; do
incus exec n${i} -- sh -c "apt-get -qy update && apt-get -qy install openssh-server sudo";
done
```
#### Configure SSH
Slip your preferred SSH key into each node's `.ssh/.authorized-keys`:
```bash
for i in {1..10}; do
incus exec n${i} -- sh -c "mkdir -p /root/.ssh && chmod 700 /root/.ssh/";
incus file push ~/.ssh/id_rsa.pub n${i}/root/.ssh/authorized_keys --uid 0 --gid 0 --mode 644;
done
```
Reset the root password to root, and allow root logins with passwords on each container.
If you've got an SSH agent set up, Jepsen can use that instead.
```bash
for i in {1..10}; do
incus exec n${i} -- bash -c 'echo -e "root\nroot\n" | passwd root';
incus exec n${i} -- sed -i 's,^#\?PermitRootLogin .*,PermitRootLogin yes,g' /etc/ssh/sshd_config;
incus exec n${i} -- systemctl restart sshd;
done
```
Store the node keys unencrypted so that jsch can use them.
If you already have the node keys, they may be unreadable to Jepsen -- remove them from `~/.ssh/known_hosts` and rescan:
```bash
for n in {1..10}; do
ssh-keyscan -t rsa n${n} >> ~/.ssh/known_hosts;
done
```
#### Confirm You Can `ssh` Into Nodes
```bash
ssh root@n1
```
#### Stopping and Deleting Containers
```bash
for i in {1..10}; do
incus stop n${i} --force;
incus delete n${i} --force;
done
```
----
#### Real VMs w/Real Clocks
```bash
# VM's use QEMU
sudo apt update && sudo apt install qemu-system
# note --vm flag
incus launch images:debian/13 n1 --vm
```
Allows the clock nemesis to bump, skew, and scramble time in a Jepsen node as it's a real vm with a real clock.
----
#### Misc
The `incus` command's \<Tab\> completion works well, even autocompletes container names.
================================================
FILE: doc/plan.md
================================================
# Stuff to improve!
## Error handling
- Knossos: Better error messages when users pass models that fail on the
first op (I think there's a ticket about this? Null pointer exception for i?)
- When users enter a node multiple times into :nodes, complain early
## Visualizations
- Add a plot for counters, showing the upper and lower bounds, and the observed
value
- Rework latency plot color scheme to use colors that hint at a continuum
- Adaptive temporal resolution for rate and latency plots, based on point
density
- Where plots are dense, make points somewhat transparent to better show
density?
## Web
- Use utf-8 for transferring files; I think we're doing latin-1 or ascii or
8859-1 or something now.
- Add search for tests
- Add sorting
- Add filtering
## Performance
- Knossos: let's make the memoization threshold configurable via options passed
to the checker.
## Core
- Macro like (synchronize-nodes test), which enforces a synchronization
barrier where (count nodes threads) must come to sync on the test map.
- Generator/each works on each *process*, not each *thread*, but almost always,
what people intend is for each thread--and that's how concat, independent,
etc work. This leads to weird scenarios like tests looping on a final read
forever and ever, as each process crashes, a new one comes in and gets a
fresh generator. Let's make it by thread?
## Extensions
- Reusable packet capture utility (take from cockroach)
## New tests
- Port bank test from Galera into core (alongside G2)
- Port query-across-tables-and-insert from Cockroach into core
- Port pure-insert from Cockroach into core
- Port comments from Cockroach into core (better name?)
- Port other Hermitage tests to Jepsen?
## Tests
- Clean up causal test. Drop model and port to workload
================================================
FILE: doc/tutorial/01-scaffolding.md
================================================
# Test scaffolding
In this tutorial, we're going to write a test for etcd: a distributed consensus
system. I want to encourage you to *type* the code yourself, even if you don't
understand everything yet. It'll help you learn faster, and you won't get as lost when we start updating small pieces of larger functions.
We'll begin by creating a new Leiningen project in any directory.
```bash
$ lein new jepsen.etcdemo
Generating a project called jepsen.etcdemo based on the 'default' template.
The default template is intended for library projects, not applications.
To see other templates (app, plugin, etc), try `lein help new`.
$ cd jepsen.etcdemo
$ ls
CHANGELOG.md doc/ LICENSE project.clj README.md resources/ src/ test/
```
Like any fresh Clojure project, we have a blank changelog, a directory for
documentation, a copy of the Eclipse Public License, a `project.clj` file,
which tells `leiningen` how to build and run our code, and a README. The
`resources` directory is a place for us to data files--for instance, config
files for a database we want to test. `src` has our source code, organized into
directories and files which match the namespace structure of our code. `test`
is for testing our code. Note that this *whole directory* is a "Jepsen test";
the `test` directory is a convention for most Clojure libraries, and we won't
be using it here.
We'll start by editing `project.clj`, which specifies the project's
dependencies and other metadata. We'll add a `:main` namespace, which is how
we'll run the test from the command line. In addition to depending on the
Clojure language itself, we'll pull in the Jepsen library, and
Verschlimmbesserung: a library for talking to etcd.
```clj
(defproject jepsen.etcdemo "0.1.0-SNAPSHOT"
:description "A Jepsen test for etcd"
:license {:name "Eclipse Public License"
:url "http://www.eclipse.org/legal/epl-v10.html"}
:main jepsen.etcdemo
:dependencies [[org.clojure/clojure "1.10.0"]
[jepsen "0.2.1-SNAPSHOT"]
[verschlimmbesserung "0.1.3"]])
```
Let's try running this program with `lein run`.
```bash
$ lein run
Exception in thread "main" java.lang.Exception: Cannot find anything to run for: jepsen.etcdemo, compiling:(/tmp/form-init6673004597601163646.clj:1:73)
...
```
Ah, yes. We haven't written anything to run yet. We need a main function in the `jepsen.etcdemo` namespace, which will receive our command line args and run the test. In `src/jepsen/etcdemo.clj`:
```clj
(ns jepsen.etcdemo)
(defn -main
"Handles command line arguments. Can either run a test, or a web server for
browsing results."
[& args]
(prn "Hello, world!" args))
```
Clojure, by default, calls our `-main` function with any arguments we passed on
the command line--whatever we type after `lein run`. It takes a variable number
of arguments (that's the `&` symbol), and calls that argument list `args`. We
print that argument list after "Hello World":
```bash
$ lein run hi there
"Hello, world!" ("hi" "there")
```
Jepsen includes some scaffolding for argument handling, running tests, handling
errors, logging, etc. Let's pull in the `jepsen.cli` namespace, call it `cli` for short, and turn our main function into a Jepsen test runner:
```clj
(ns jepsen.etcdemo
(:require [jepsen.cli :as cli]
[jepsen.tests :as tests]))
(defn etcd-test
"Given an options map from the command line runner (e.g. :nodes, :ssh,
:concurrency, ...), constructs a test map."
[opts]
(merge tests/noop-test
{:pure-generators true}
opts))
(defn -main
"Handles command line arguments. Can either run a test, or a web server for
browsing results."
[& args]
(cli/run! (cli/single-test-cmd {:test-fn etcd-test})
args))
```
`cli/single-test-cmd` is provided by `jepsen.cli`: it parses command line
arguments for a test and calls the provided `:test-fn`, which should return a
map containing all the information Jepsen needs to run a test. In this case,
our test function is `etcd-test`, which takes options from the command line
runner, and uses them to fill in position in an empty test that does nothing:
`noop-test`.
```bash
$ lein run
Usage: lein run -- COMMAND [OPTIONS ...]
Commands: test
```
With no args, `cli/run!` provides a basic help message, informing us it takes a
command as its first argument. Let's try the test command we added:
Let's give it a shot!
```bash
$ lein run test
13:04:30.927 [main] INFO jepsen.cli - Test options:
{:concurrency 5,
:test-count 1,
:time-limit 60,
:nodes ["n1" "n2" "n3" "n4" "n5"],
:ssh
{:username "root",
:password "root",
:strict-host-key-checking false,
:private-key-path nil}}
INFO [2018-02-02 13:04:30,994] jepsen test runner - jepsen.core Running test:
{:concurrency 5,
:db
#object[jepsen.db$reify__1259 0x6dcf7b6a "jepsen.db$reify__1259@6dcf7b6a"],
:name "noop",
:start-time
#object[org.joda.time.DateTime 0x79d4ff58 "2018-02-02T13:04:30.000-06:00"],
:net
#object[jepsen.net$reify__3493 0xae3c140 "jepsen.net$reify__3493@ae3c140"],
:client
#object[jepsen.client$reify__3380 0x20027c44 "jepsen.client$reify__3380@20027c44"],
:barrier
#object[java.util.concurrent.CyclicBarrier 0x2bf3ec4 "java.util.concurrent.CyclicBarrier@2bf3ec4"],
:ssh
{:username "root",
:password "root",
:strict-host-key-checking false,
:private-key-path nil},
:checker
#object[jepsen.checker$unbridled_optimism$reify__3146 0x1410d645 "jepsen.checker$unbridled_optimism$reify__3146@1410d645"],
:nemesis
#object[jepsen.nemesis$reify__3574 0x4e6cbdf1 "jepsen.nemesis$reify__3574@4e6cbdf1"],
:active-histories #<Atom@210a26b: #{}>,
:nodes ["n1" "n2" "n3" "n4" "n5"],
:test-count 1,
:generator
#object[jepsen.generator$reify__1936 0x1aac0a47 "jepsen.generator$reify__1936@1aac0a47"],
:os
#object[jepsen.os$reify__1176 0x438aaa9f "jepsen.os$reify__1176@438aaa9f"],
:time-limit 60,
:model {}}
INFO [2018-02-02 13:04:35,389] jepsen nemesis - jepsen.core Starting nemesis
INFO [2018-02-02 13:04:35,389] jepsen worker 1 - jepsen.core Starting worker 1
INFO [2018-02-02 13:04:35,389] jepsen worker 2 - jepsen.core Starting worker 2
INFO [2018-02-02 13:04:35,389] jepsen worker 0 - jepsen.core Starting worker 0
INFO [2018-02-02 13:04:35,390] jepsen worker 3 - jepsen.core Starting worker 3
INFO [2018-02-02 13:04:35,390] jepsen worker 4 - jepsen.core Starting worker 4
INFO [2018-02-02 13:04:35,391] jepsen nemesis - jepsen.core Running nemesis
INFO [2018-02-02 13:04:35,391] jepsen worker 1 - jepsen.core Running worker 1
INFO [2018-02-02 13:04:35,391] jepsen worker 2 - jepsen.core Running worker 2
INFO [2018-02-02 13:04:35,391] jepsen worker 0 - jepsen.core Running worker 0
INFO [2018-02-02 13:04:35,391] jepsen worker 3 - jepsen.core Running worker 3
INFO [2018-02-02 13:04:35,391] jepsen worker 4 - jepsen.core Running worker 4
INFO [2018-02-02 13:04:35,391] jepsen nemesis - jepsen.core Stopping nemesis
INFO [2018-02-02 13:04:35,391] jepsen worker 1 - jepsen.core Stopping worker 1
INFO [2018-02-02 13:04:35,391] jepsen worker 2 - jepsen.core Stopping worker 2
INFO [2018-02-02 13:04:35,391] jepsen worker 0 - jepsen.core Stopping worker 0
INFO [2018-02-02 13:04:35,391] jepsen worker 3 - jepsen.core Stopping worker 3
INFO [2018-02-02 13:04:35,391] jepsen worker 4 - jepsen.core Stopping worker 4
INFO [2018-02-02 13:04:35,397] jepsen test runner - jepsen.core Run complete, writing
INFO [2018-02-02 13:04:35,434] jepsen test runner - jepsen.core Analyzing
INFO [2018-02-02 13:04:35,435] jepsen test runner - jepsen.core Analysis complete
INFO [2018-02-02 13:04:35,438] jepsen results - jepsen.store Wrote /home/aphyr/jepsen/jepsen.etcdemo/store/noop/20180202T130430.000-0600/results.edn
INFO [2018-02-02 13:04:35,440] main - jepsen.core {:valid? true}
Everything looks good! ヽ(‘ー`)ノ
```
We can see Jepsen start a series of workers--each one responsible for executing
operations against the database--and a nemesis, which causes failures. We
haven't given them anything to do, so they shut down immediately. Jepsen writes
out the result of this (trivial) test to the `store` directory, and prints out
a brief analysis.
`noop-test` uses nodes named `n1`, `n2`, ... `n5` by default. If your nodes
have different names, this test will fail to connect to them. That's OK! You can change that by passing node names on the command line:
```bash
$ lein run test --node foo.mycluster --node 1.2.3.4
```
... or by passing a filename that has a list of nodes in it, one per line. If
you're using the AWS Marketplace cluster, you've already got a file called
`nodes` in your home directory, ready to go.
```bash
$ lein run test --nodes-file ~/nodes
```
If you're still hitting SSH errors at this point, you should check that your
SSH agent is running and has keys for all your nodes loaded. `ssh some-db-node`
should work without a password. You can override the username, password, and
identity file at the command line as well; see `lein run test --help` for
details.
```bash
$ lein run test --help
#object[jepsen.cli$test_usage 0x7ddd84b5 jepsen.cli$test_usage@7ddd84b5]
-h, --help Print out this message and exit
-n, --node HOSTNAME ["n1" "n2" "n3" "n4" "n5"] Node(s) to run test on
--nodes-file FILENAME File containing node hostnames, one per line.
--username USER root Username for logins
--password PASS root Password for sudo access
--strict-host-key-checking Whether to check host keys
--ssh-private-key FILE Path to an SSH identity file
--concurrency NUMBER 1n How many workers should we run? Must be an integer, optionally followed by n (e.g. 3n) to multiply by the number of nodes.
--test-count NUMBER 1 How many times should we repeat a test?
--time-limit SECONDS 60 Excluding setup and teardown, how long should a test run for, in seconds?
```
We'll use `lein run test ...` throughout this guide to re-run our Jepsen test. Each time we run a test, Jepsen will create a new directory in `store/`. You can see the latest results in `store/latest`:
```bash
$ ls store/latest/
history.txt jepsen.log results.edn test.fressian
```
`history.txt` shows the operations the test performed--ours is empty, since the
noop test doesn't perform any ops. `jepsen.log` has a copy of the console log
for that test. `results.edn` shows the analysis of the test, which we see at
the end of each run. Finally, `test.fressian` has the raw data for the test,
including the full machine-readable history and analysis, if you need to
perform post-hoc analysis.
Jepsen also comes with a built-in web browser for browsing these results. Let's add it to our `main` function:
```clj
(defn -main
"Handles command line arguments. Can either run a test, or a web server for
browsing results."
[& args]
(cli/run! (merge (cli/single-test-cmd {:test-fn etcd-test})
(cli/serve-cmd))
args))
```
This works because `cli/run!` takes a map of command names to specifications of
how to run those commands. We're merging those maps together with `merge`.
```bash
$ lein run serve
13:29:21.425 [main] INFO jepsen.web - Web server running.
13:29:21.428 [main] INFO jepsen.cli - Listening on http://0.0.0.0:8080/
```
We can open `http://localhost:8080` in a web browser to explore the history of
our test results. Of course, the serve command comes with its own options and
help message:
```bash
$ lein run serve --help
Usage: lein run -- serve [OPTIONS ...]
-h, --help Print out this message and exit
-b, --host HOST 0.0.0.0 Hostname to bind to
-p, --port NUMBER 8080 Port number to bind to
```
Open up a new terminal window, and leave the web server running there. That way
we can see the results of our tests without having to start and stop it
repeatedly.
With this groundwork in place, we'll write the code to [set up and tear down the database](02-db.md).
================================================
FILE: doc/tutorial/02-db.md
================================================
# Database automation
In a Jepsen test, a `DB` encapsulates code for setting up and tearing down a
database, queue, or other distributed system we want to test. We could perform
the setup and teardown by hand, but letting Jepsen handle it lets us run tests
in a CI system, parameterize database configuration, run multiple tests
back-to-back with a clean slate, and so on.
In `src/jepsen/etcdemo.clj`, we'll require the `jepsen.db`, `jepsen.control`,
`jepsen.control.util`, and `jepsen.os.debian` namespaces, aliasing each to a
short name. `clojure.string` will help us build configuration strings for etcd.
We'll also pull in every function from `clojure.tools.logging`, giving us log
functions like `info`, `warn`, etc.
```clj
(ns jepsen.etcdemo
(:require [clojure.tools.logging :refer :all]
[clojure.string :as str]
[jepsen [cli :as cli]
[control :as c]
[db :as db]
[tests :as tests]]
[jepsen.control.util :as cu]
[jepsen.os.debian :as debian]))
```
Then, we'll write a function that constructs a Jepsen DB, given a particular
version string.
```clj
(defn db
"Etcd DB for a particular version."
[version]
(reify db/DB
(setup! [_ test node]
(info node "installing etcd" version))
(teardown! [_ test node]
(info node "tearing down etcd"))))
```
The string after `(defn db ...` is a *docstring*, documenting the function's
behavior. When given a `version`, the `db` function uses `reify` to construct a
new object satisfying jepsen's `DB` protocol (from the `db` namespace). That
protocol specifies two functions all databases must support: `(setup! db test
node)`, and `(teardown! db test node)`. We provide stub implementations
here, which simply log an informational message.
Now, we'll extend the default `noop-test` by adding an `:os`, which tells
Jepsen how to handle operating system setup, and a `:db`, which we can
construct using the `db` function we just wrote. We'll test etcd version
`v3.1.5`.
```clj
(defn etcd-test
"Given an options map from the command line runner (e.g. :nodes, :ssh,
:concurrency ...), constructs a test map."
[opts]
(merge tests/noop-test
opts
{:name "etcd"
:os debian/os
:db (db "v3.1.5")
:pure-generators true}))
```
`noop-test`, like all Jepsen tests, is a map with keys like `:os`, `:name`,
`:db`, etc. See [jepsen.core](../../jepsen/src/jepsen/core.clj) for an
overview of test structure, and `jepsen.core/run` for the full definition of a
test.
Right now `noop-test` has stub implementations for those keys. But we can
use `merge` to build a copy of the `noop-test` map with *new* values for some
keys.
If we run this test, we'll see Jepsen using our code to set up debian,
pretend to tear down and install etcd, then start its workers.
```bash
$ lein run test
...
INFO [2017-03-30 10:08:30,852] jepsen node n2 - jepsen.os.debian :n2 setting up debian
INFO [2017-03-30 10:08:30,852] jepsen node n3 - jepsen.os.debian :n3 setting up debian
INFO [2017-03-30 10:08:30,852] jepsen node n4 - jepsen.os.debian :n4 setting up debian
INFO [2017-03-30 10:08:30,852] jepsen node n5 - jepsen.os.debian :n5 setting up debian
INFO [2017-03-30 10:08:30,852] jepsen node n1 - jepsen.os.debian :n1 setting up debian
INFO [2017-03-30 10:08:52,385] jepsen node n1 - jepsen.etcdemo :n1 tearing down etcd
INFO [2017-03-30 10:08:52,385] jepsen node n4 - jepsen.etcdemo :n4 tearing down etcd
INFO [2017-03-30 10:08:52,385] jepsen node n2 - jepsen.etcdemo :n2 tearing down etcd
INFO [2017-03-30 10:08:52,385] jepsen node n3 - jepsen.etcdemo :n3 tearing down etcd
INFO [2017-03-30 10:08:52,385] jepsen node n5 - jepsen.etcdemo :n5 tearing down etcd
INFO [2017-03-30 10:08:52,386] jepsen node n1 - jepsen.etcdemo :n1 installing etcd v3.1.5
INFO [2017-03-30 10:08:52,386] jepsen node n4 - jepsen.etcdemo :n4 installing etcd v3.1.5
INFO [2017-03-30 10:08:52,386] jepsen node n2 - jepsen.etcdemo :n2 installing etcd v3.1.5
INFO [2017-03-30 10:08:52,386] jepsen node n3 - jepsen.etcdemo :n3 installing etcd v3.1.5
INFO [2017-03-30 10:08:52,386] jepsen node n5 - jepsen.etcdemo :n5 installing etcd v3.1.5
...
```
See how the version string `"v3.1.5"` was passed from `etcd-test` to `db`,
where it was captured by the `reify` expression? This is how we *parameterize*
Jepsen tests, so the same code can test multiple versions or options. Note also
that the object `reify` returns closes over its lexical scope, *remembering*
the value of `version`.
## Installing the DB
With the skeleton of the DB in place, it's time to actually install something. Let's take a quick look at the [installation instructions for etcd](https://github.com/coreos/etcd/releases/tag/v3.1.5). It looks like we'll need to download a tarball, unpack it into a directory, set an environment variable for the API version, and run the `etcd` binary with it.
We'll have to be root to install packages, so we'll use `jepsen.control/su` to
assume root privileges. Note that `su` (and its companions `sudo`, `cd`, etc)
establish *dynamic*, not *lexical* scope--their effects apply not only to the
code they enclose, but down the call stack to any functions called. Then we'll
use `jepsen.control.util/install-archive!` to download the etcd tarball and
install it to `/opt/etcd`.
```clj
(def dir "/opt/etcd")
(defn db
"Etcd DB for a particular version."
[version]
(reify db/DB
(setup! [_ test node]
(info node "installing etcd" version)
(c/su
(let [url (str "https://storage.googleapis.com/etcd/" version
"/etcd-" version "-linux-amd64.tar.gz")]
(cu/install-archive! url dir))))
(teardown! [_ test node]
(info node "tearing down etcd"))))
```
We're using `jepsen.control/su` to become root (via sudo) while we remove the
etcd directory. `jepsen.control` provides a comprehensive DSL for executing
arbitrary shell commands on remote nodes.
Now `lein run test` will go and install etcd itself. Note that Jepsen runs
`setup` and `teardown` concurrently across all nodes. This might take a little
while because each node has to download the tarball, but on future runs, Jepsen
will re-use the cached tarball on disk.
## Starting the DB
Per the [clustering instructions](https://coreos.com/etcd/docs/latest/v2/clustering.html), we'll need to generate a string like `"ETCD_INITIAL_CLUSTER="infra0=http://10.0.1.10:2380,infra1=http://10.0.1.11:2380,infra2=http://10.0.1.12:2380"`, so that our nodes know which nodes are a part of the cluster. Let's write a few small functions to build those strings:
```clj
(defn node-url
"An HTTP url for connecting to a node on a particular port."
[node port]
(str "http://" node ":" port))
(defn peer-url
"The HTTP url for other peers to talk to a node."
[node]
(node-url node 2380))
(defn client-url
"The HTTP url clients use to talk to a node."
[node]
(node-url node 2379))
(defn initial-cluster
"Constructs an initial cluster string for a test, like
\"foo=foo:2380,bar=bar:2380,...\""
[test]
(->> (:nodes test)
(map (fn [node]
(str node "=" (peer-url node))))
(str/join ",")))
```
The `->>` threading macro takes each form and inserts it into the next form as
a final argument. So `(->> test :nodes)` becomes `(:nodes test)`, and `(->> test
:nodes (map-indexed (fn ...)))` becomes `(map-indexed (fn ...) (:nodes test))`,
and so on. Normal function calls often look "inside out", but the `->>` macro
lets us write a chain of operations "in order"--like an object-oriented
language's `foo.bar().baz()` notation.
So in `initial-cluster`, we take the nodes from a test map, and map each node
to a string: its name, the "=" string, and its peer url. Then we join all those
strings together with commas.
With this in place, we'll tell the DB how to start up etcd as a daemon. We
could use an init script or service to start and stop the program, but since
we're working with a plain binary here, we'll use Debian's `start-stop-daemon`
command to run `etcd` in the background.
We'll need a few more constants: the etcd binary, where to log output to, and
where to store a pidfile.
```clj
(def dir "/opt/etcd")
(def binary "etcd")
(def logfile (str dir "/etcd.log"))
(def pidfile (str dir "/etcd.pid"))
```
Now we'll use `jepsen.control.util`'s functions for starting and stopping
daemons to spin up etcd. Per the
[documentation](https://coreos.com/etcd/docs/latest/v2/clustering.html), we'll
need to provide a node name, urls to listen on for clients and peers, and some
initial cluster state.
```clj
(setup! [_ test node]
(info node "installing etcd" version)
(c/su
(let [url (str "https://storage.googleapis.com/etcd/" version
"/etcd-" version "-linux-amd64.tar.gz")]
(cu/install-archive! url dir))
(cu/start-daemon!
{:logfile logfile
:pidfile pidfile
:chdir dir}
binary
:--log-output :stderr
:--name (name node)
:--listen-peer-urls (peer-url node)
:--listen-client-urls (client-url node)
:--advertise-client-urls (client-url node)
:--initial-cluster-state :new
:--initial-advertise-peer-urls (peer-url node)
:--initial-cluster (initial-cluster test))
(Thread/sleep 10000)))
```
We'll sleep for a little bit after starting the cluster, so it has a chance to boot up and perform its initial handshakes.
## Tearing down
To make sure that every run starts fresh, even if a previous run crashed,
Jepsen performs a DB teardown at the start of the test, before setup. Then it
tears the DB down again at the conclusion of the test. To tear down, we'll use
`stop-daemon!`, and then delete the etcd directory, so that future runs won't
accidentally read data from our current run.
```clj
(teardown! [_ test node]
(info node "tearing down etcd")
(cu/stop-daemon! binary pidfile)
(c/su (c/exec :rm :-rf dir)))))
```
We use `jepsen.control/exec` to run a shell command: `rm -rf`.
Jepsen automatically binds `exec` to operate on the `node` being set up
during `db/setup!`, but we can connect to arbitrary nodes if we need to. Note
that `exec` can take any mixture of strings, numbers, keywords--it'll convert
them to strings and perform appropriate shell escaping. You can use
`jepsen.control/lit` for an unescaped literal string, if need be. `:>` and
`:>>` perform shell redirection as you'd expect. That's an easy way to write
out configuration files to disk, for databases that need config.
Let's try it out.
```bash
$ lein run test
NFO [2017-03-30 12:08:19,755] jepsen node n5 - jepsen.etcdemo :n5 installing etcd v3.1.5
INFO [2017-03-30 12:08:19,755] jepsen node n1 - jepsen.etcdemo :n1 installing etcd v3.1.5
INFO [2017-03-30 12:08:19,755] jepsen node n2 - jepsen.etcdemo :n2 installing etcd v3.1.5
INFO [2017-03-30 12:08:19,755] jepsen node n4 - jepsen.etcdemo :n4 installing etcd v3.1.5
INFO [2017-03-30 12:08:19,855] jepsen node n3 - jepsen.etcdemo :n3 installing etcd v3.1.5
INFO [2017-03-30 12:08:20,866] jepsen node n4 - jepsen.control.util starting etcd
INFO [2017-03-30 12:08:20,866] jepsen node n1 - jepsen.control.util starting etcd
INFO [2017-03-30 12:08:20,866] jepsen node n5 - jepsen.control.util starting etcd
INFO [2017-03-30 12:08:20,866] jepsen node n2 - jepsen.control.util starting etcd
INFO [2017-03-30 12:08:20,963] jepsen node n3 - jepsen.control.util starting etcd
...
```
Looks good. We can confirm that `teardown` did its job by checking that the etcd directory is empty after the test.
```bash
$ ssh n1 ls /opt/etcd
ls: cannot access /opt/etcd: No such file or directory
```
## Log files
Hang on--if we delete etcd's files after every run, how do we figure out
what the database did? It'd be nice if we could download a copies of the
database's logs before cleaning up. For that, we'll use the `db/LogFiles`
protocol, and return a list of log file paths to download.
```clj
(defn db
"Etcd DB for a particular version."
[version]
(reify db/DB
(setup! [_ test node]
...)
(teardown! [_ test node]
...)
db/LogFiles
(log-files [_ test node]
[logfile])))
```
Now, when we run a test, we'll find a copy of the log for each node, stored in the local directory `store/latest/<node-name>/`. If we get stuck setting up the DB, we can check those logs to see what's gone wrong.
```bash
$ less store/latest/n1/etcd.log
...
2018-02-02 11:36:51.848330 I | raft: 5440ff22fe632778 became leader at term 2
2018-02-02 11:36:51.848360 I | raft: raft.node: 5440ff22fe632778 elected leader 5440ff22fe632778 at term 2
2018-02-02 11:36:51.860295 I | etcdserver: setting up the initial cluster version to 3.1
2018-02-02 11:36:51.864532 I | embed: ready to serve client requests
...
```
Look for the line that says "elected leader"--that demonstrates that our nodes
formed a cluster successfully. If your etcd nodes can't see each other, make
sure that you're using the right port names, that you used `http://` instead of
`https://` in `node-url`, and that nodes can ping one another.
With the database ready, it's time to [write a client](03-client.md).
================================================
FILE: doc/tutorial/03-client.md
================================================
# Writing a client
A Jepsen *client* takes *invocation operations* and applies them to the system
being tested, returning corresponding *completion operations*. For our
etcd test, we might model the system as a single register: a particular key
holding an integer. The operations against that register might be `read`,
`write`, and `compare-and-set`, which we could model like so:
```clj
(defn r [_ _] {:type :invoke, :f :read, :value nil})
(defn w [_ _] {:type :invoke, :f :write, :value (rand-int 5)})
(defn cas [_ _] {:type :invoke, :f :cas, :value [(rand-int 5) (rand-int 5)]})
```
These are functions that construct Jepsen *operations*: an abstract
representation of things you can do to a database. `:invoke` means that we're
going to *try* an operation--when it completes, we'll use a type like `:ok` or
`:fail` to tell what happened. The `:f` tells us what function we're applying
to the database--for instance, that we want to perform a read or a write. These
can be *any* values--Jepsen doesn't know what they mean.
Function calls are parameterized by their arguments and return values. Jepsen
operations are parameterized by `:value`, which can be anything we like--Jepsen
doesn't inspect them. We use a write's value to specify the value that we
wrote, and a read's value to specify the value that we (eventually) read. When
we invoke a read, we don't know *what* we're going to read yet, so we'll leave
it `nil`.
These functions can be used by `jepsen.generator` to construct new invocations
for reads, writes, and compare-and-set ops, respectively. Note that the read
value is `nil`--we don't know what value is being read until we actually
perform the read. When the client reads a particular value it'll fill that
value in for the completion op.
## Connecting to the database
Now we need to take these operations and *apply* them to etcd. We'll use
the [Verschlimmbessergung](https://github.com/aphyr/verschlimmbesserung)
library to talk to etcd. We'll start by requiring Verschlimmbesserung, and
writing an empty implementation of Jepsen's Client protocol:
```clj
(ns jepsen.etcdemo
(:require [clojure.tools.logging :refer :all]
[clojure.string :as str]
[verschlimmbesserung.core :as v]
[jepsen [cli :as cli]
[client :as client]
[control :as c]
[db :as db]
[tests :as tests]]
[jepsen.control.util :as cu]
[jepsen.os.debian :as debian]))
...
(defrecord Client [conn]
client/Client
(open! [this test node]
this)
(setup! [this test])
(invoke! [_ test op])
(teardown! [this test])
(close! [_ test]))
```
`defrecord` defines a new type of data structure, which we're calling `Client`.
Each Client has a single field, called `conn`, which will hold our connection
to a particular network server. Clients support Jepsen's Client protocol, and
just like a `reify`, we'll provide implementations for Client functions.
Clients have a five-part lifecycle. We begin with a single *seed* client
`(client)`. When we call `open!` on that client, we get a *copy* of the client
bound to a particular node. The `setup!` function initializes any data
structures the test needs--for instance, creating tables or setting up
fixtures. `invoke!` applies operations to the system and returns corresponding
completion operations. `teardown!` cleans up any tables `setup!` may have
created. `close!` closes network connections and completes the lifecycle for
the client.
When it comes time to add the client to the test, we use `(Client.)` to
construct a new Client, and pass `nil` as the value for `conn`. Remember, our
initial seed client doesn't have a connection; Jepsen will call `open!` to get
connected clients later.
```clj
(defn etcd-test
"Given an options map from the command line runner (e.g. :nodes, :ssh,
:concurrency ...), constructs a test map."
[opts]
(merge tests/noop-test
opts
{:pure-generators true
:name "etcd"
:os debian/os
:db (db "v3.1.5")
:client (Client. nil)}))
```
Now, let's complete our `open!` function by connecting to etcd. The
[Verschlimmbesserung docs](https://github.com/aphyr/verschlimmbesserung#usage)
tell us that every function takes a Verschlimmbesserung client, created with
`(connect url)`. That client is what we'll store in `conn`. Let's have
Verschlimmbesserung time out requests after 5 seconds, too.
```clj
(defrecord Client [conn]
client/Client
(open! [this test node]
(assoc this :conn (v/connect (client-url node)
{:timeout 5000})))
(setup! [this test])
(invoke! [_ test op])
(teardown! [this test])
(close! [_ test]
; If our connection were stateful, we'd close it here. Verschlimmmbesserung
; doesn't actually hold connections, so there's nothing to close.
))
(defn etcd-test
"Given an options map from the command-line runner (e.g. :nodes, :ssh,
:concurrency, ...), constructs a test map."
[opts]
(merge tests/noop-test
opts
{:name "etcd"
:os debian/os
:db (db "v3.1.5")
:client (Client. nil)}))
```
Remember, the initial client *has no connections*--like a stem cell, it has the
*potential* to become an active client but doesn't do any work directly. We
call `(Client. nil)` to construct that initial client--its conn will be filled
in when Jepsen calls `open!`.
## Reads
Now we have to actually *do* something with the client. Let's start with fifteen
seconds of reads, randomly staggered about a second apart. We'll pull in `jepsen.generator` to schedule operations.
```clj
(ns jepsen.etcdemo
(:require [clojure.tools.logging :refer :all]
[clojure.string :as str]
[verschlimmbesserung.core :as v]
[jepsen [cli :as cli]
[client :as client]
[control :as c]
[db :as db]
[generator :as gen]
[tests :as tests]]
[jepsen.control.util :as cu]
[jepsen.os.debian :as debian]))
```
... and write a simple generator: take reads, stagger them by about a second,
give those operations to clients only (not the nemesis, which has other
duties), and stop after 15 seconds.
```clj
(defn etcd-test
"Given an options map from the command line runner (e.g. :nodes, :ssh,
:concurrency ...), constructs a test map."
[opts]
(merge tests/noop-test
opts
{:pure-generators true
:name "etcd"
:os debian/os
:db (db "v3.1.5")
:client (Client. nil)
:generator (->> r
(gen/stagger 1)
(gen/nemesis nil)
(gen/time-limit 15))}))
```
This throws a bunch of errors, because we haven't told the client *how* to
intepret these reads yet.
```bash
$ lein run test
...
WARN [2020-09-21 20:16:33,150] jepsen worker 0 - jepsen.generator.interpreter Process 0 crashed
clojure.lang.ExceptionInfo: throw+: {:type :jepsen.client/invalid-completion, :op {:type :invoke, :f :read, :value nil, :time 26387538, :process 0}, :op' nil, :problems ["should be a map" ":type should be :ok, :info, or :fail" ":process should be the same" ":f should be the same"]}
```
The client's `invoke!` function takes an invocation operation, and right now,
does nothing with it, returning `nil`. Jepsen is telling us that it should be a
map, and specifically one with a `:type` field, a matching `:process`, and a
matching `:f`. In short, we have to construct a completion operation which
*concludes* the invocation operation. We'll construct this completion op with
type `:ok` if the operation succeeded, `:fail` if it didn't take place, or
`:info` if we're not sure. `invoke!` can also throw an exception, which is
automatically converted to an `:info`.
Let's start by handling reads. We'll use `v/get` to read the value of a single
key. We can pick any name we like--let's call it "foo" for now.
```clj
(invoke! [this test op]
(case (:f op)
:read (assoc op :type :ok, :value (v/get conn "foo"))))
```
We dispatch based on the `:f` field of the operation, and when it's a
`:read`, we take the invoke op and return a copy of it, with `:type` `:ok` and
a value obtained by reading the register "foo".
```bash
$ lein run test
...
INFO [2017-03-30 15:28:17,423] jepsen worker 2 - jepsen.util 2 :invoke :read nil
INFO [2017-03-30 15:28:17,427] jepsen worker 2 - jepsen.util 2 :ok :read nil
INFO [2017-03-30 15:28:18,315] jepsen worker 0 - jepsen.util 0 :invoke :read nil
INFO [2017-03-30 15:28:18,320] jepsen worker 0 - jepsen.util 0 :ok :read nil
INFO [2017-03-30 15:28:18,437] jepsen worker 4 - jepsen.util 4 :invoke :read nil
INFO [2017-03-30 15:28:18,441] jepsen worker 4 - jepsen.util 4 :ok :read nil
```
Much better! We haven't created the key yet, so its value is `nil`. In order to
change the value, we'll add some some writes to the generator.
## Writes
We'll change our generator to take a random mixture of reads and writes, using
`(gen/mix [r w])`.
```clj
(defn etcd-test
"Given an options map from the command line runner (e.g. :nodes, :ssh,
:concurrency ...), constructs a test map."
[opts]
(merge tests/noop-test
opts
{:pure-generators true
:name "etcd"
:os debian/os
:db (db "v3.1.5")
:client (Client. nil)
:generator (->> (gen/mix [r w])
(gen/stagger 1)
(gen/nemesis nil)
(gen/time-limit 15))}))
```
To handle those writes, we'll use `v/reset!`, and return the op with
`:type` `:ok`. If `reset!` fails it'll throw, and Jepsen's machinery will
automatically convert it to an `:info` crash.
```clj
(invoke! [this test op]
(case (:f op)
:read (assoc op :type :ok, :value (v/get conn "foo"))
:write (do (v/reset! conn "foo" (:value op))
(assoc op :type :ok))))
```
We'll confirm writes work by watching the test:
```bash
$ lein run test
INFO [2017-03-30 22:14:25,428] jepsen worker 4 - jepsen.util 4 :invoke :write 0
INFO [2017-03-30 22:14:25,439] jepsen worker 4 - jepsen.util 4 :ok :write 0
INFO [2017-03-30 22:14:25,628] jepsen worker 0 - jepsen.util 0 :invoke :read nil
INFO [2017-03-30 22:14:25,633] jepsen worker 0 - jepsen.util 0 :ok :read "0"
```
Ah, we've got a bit of a snag here. etcd thinks in terms of strings, but we'd
like to work with numbers. We could pull in a serialization library (jepsen
includes a simple one in `jepsen.codec`), but since we're only dealing with
integers and nil, we can get away with using Clojure's built-in `parse-long`
function. It doesn't like being passed `nil`, though, so we'll write a small
wrapper:
```clj
(defn parse-long-nil
"Parses a string to a Long. Passes through `nil`."
[s]
(when s (parse-long s)))
...
(invoke! [_ test op]
(case (:f op)
:read (assoc op :type :ok, :value (parse-long-nil (v/get conn "foo")))
:write (do (v/reset! conn "foo" (:value op))
(assoc op :type :ok))))
```
Note that we only call `parse-long` when our string is truthy--using `(when s
…)`. If `when` doesn't match, it'll return `nil`, which lets us pass through
`nil` values transparently.
```bash
$ lein run test
...
INFO [2017-03-30 22:26:45,322] jepsen worker 4 - jepsen.util 4 :invoke :write 1
INFO [2017-03-30 22:26:45,341] jepsen worker 4 - jepsen.util 4 :ok :write 1
INFO [2017-03-30 22:26:45,434] jepsen worker 2 - jepsen.util 2 :invoke :read nil
INFO [2017-03-30 22:26:45,439] jepsen worker 2 - jepsen.util 2 :ok :read 1
```
Seems reasonable! Only one type of operation left to implement: compare-and-set.
## Compare and set
We'll finish the client by adding compare-and-set to the mix:
```clj
(gen/mix [r w cas])
```
Handling CaS is a little trickier. Verschlimmbesserung gives us a `cas!`
function, which takes a connection, key, old value, and new value. `cas!` sets
the key to the new value if and only if the old value matches what's currently
there, and returns a detailed response map. If the CaS fails, it returns false.
We can use that to determine the `:type` of the CaS operation.
```clj
(invoke! [_ test op]
(case (:f op)
:read (assoc op :type :ok, :value (parse-long-nil (v/get conn "foo")))
:write (do (v/reset! conn "foo" (:value op))
(assoc op :type :ok))
:cas (let [[old new] (:value op)]
(assoc op :type (if (v/cas! conn "foo" old new)
:ok
:fail)))))
```
The `let` binding here uses *destructuring*: it breaks apart the `[old-value
new-value]` pair from the operation's `:value` field into `old` and `new`.
Since all values except `false` and `nil` are logically true, we can use the
result of the `cas!` call as our predicate in `if`.
## Handling exceptions
If you run this a few times, you might see:
```bash
$ lein run test
...
INFO [2017-03-30 22:38:51,892] jepsen worker 1 - jepsen.util 1 :invoke :cas [3 1]
WARN [2017-03-30 22:38:51,936] jepsen worker 1 - jepsen.core Process 1 indeterminate
clojure.lang.ExceptionInfo: throw+: {:errorCode 100, :message "Key not found", :cause "/foo", :index 11, :status 404}
at slingshot.support$stack_trace.invoke(support.clj:201) ~[na:na]
...
```
If we try to CaS the key before it's written, Verschlimmbesserung will throw an
exception complaining (quite sensibly!) that we can't alter something that
doesn't exist. This won't cause our test to return false positives--Jepsen will
interpret the exception as an indeterminate `:info` result, and allow that it
might or might not have taken place. However, we *know* the value didn't change
when we get this exception, so we can convert it to a known failure. We'll pull
in the `slingshot` exception handling library, so we can catch that particular
error code.
```clj
(ns jepsen.etcdemo
(:require ...
[slingshot.slingshot :refer [try+]]))
```
... and wrap our `cas` in a try/catch block.
```clj
(invoke! [this test op]
(case (:f op)
:read (assoc op :type :ok, :value (parse-long-nil (v/get conn "foo")))
:write (do (v/reset! conn "foo" (:value op))
(assoc op :type :ok))
:cas (try+
(let [[old new] (:value op)]
(assoc op :type (if (v/cas! conn "foo" old new)
:ok
:fail)))
(catch [:errorCode 100] ex
(assoc op :type :fail, :error :not-found)))))
```
This [:errorCode 100] form tells Slingshot to catch only exceptions which have
that particular error code, and bind them to `ex`. We've added an extra
`:error` field to our operation. This doesn't matter as far as correctness is
concerned, but it helps us understand what happened when we read the logs.
Jepsen prints errors at the end of log lines.
```bash
$ lein run test
...
INFO [2017-03-30 23:00:50,978] jepsen worker 0 - jepsen.util 0 :invoke :cas [1 4]
INFO [2017-03-30 23:00:51,065] jepsen worker 0 - jepsen.util 0 :fail :cas [1 4] :not-found
```
Much neater. In general, we'll start by writing the simplest code we can, and
allow Jepsen to handle exceptions for us. Once we have a feeling for how things
can go wrong, we can introduce special error handlers and semantics for those
failure cases.
```bash
...
INFO [2017-03-30 22:38:59,278] jepsen worker 1 - jepsen.util 11 :invoke :write 4
INFO [2017-03-30 22:38:59,286] jepsen worker 1 - jepsen.util 11 :ok :write 4
INFO [2017-03-30 22:38:59,289] jepsen worker 4 - jepsen.util 4 :invoke :cas [2 2]
INFO [2017-03-30 22:38:59,294] jepsen worker 1 - jepsen.util 11 :invoke :read nil
INFO [2017-03-30 22:38:59,297] jepsen worker 1 - jepsen.util 11 :ok :read 4
INFO [2017-03-30 22:38:59,298] jepsen worker 4 - jepsen.util 4 :fail :cas [2 2]
INFO [2017-03-30 22:38:59,818] jepsen worker 4 - jepsen.util 4 :invoke :write 1
INFO [2017-03-30 22:38:59,826] jepsen worker 4 - jepsen.util 4 :ok :write 1
INFO [2017-03-30 22:38:59,917] jepsen worker 1 - jepsen.util 11 :invoke :cas [1 2]
INFO [2017-03-30 22:38:59,926] jepsen worker 1 - jepsen.util 11 :ok :cas [1 2]
```
Notice that some CaS operations fail, and other succeed? It's OK for them to
fail, and in fact, it's desirable. We expect some CaS ops to fail because their
predicate value doesn't match the current value, but a few (~1/5, since there
are 5 possible values for the register at any given point) should succeed. In
addition, it's worth trying operations we think should be impossible, because
if they *do* succeed, that can point to a consistency violation.
With our client performing operations, it's time to analyze results using a
[checker](04-checker.md).
================================================
FILE: doc/tutorial/04-checker.md
================================================
# Checking Correctness
With our generator and clients performing operations, we've got a history to
analyze for correctness. Jepsen uses a *model* to represent the abstract
behavior of a system, and a *checker* to verify whether the history conforms to
that model. We'll require `knossos.model` and `jepsen.checker`:
```clj
(ns jepsen.etcdemo
(:require [clojure.tools.logging :refer :all]
[clojure.string :as str]
[jepsen [checker :as checker]
[cli :as cli]
[client :as client]
[control :as c]
[db :as db]
[generator :as gen]
[tests :as tests]]
[jepsen.control.util :as cu]
[jepsen.os.debian :as debian]
[knossos.model :as model]
[slingshot.slingshot :refer [try+]]
[verschlimmbesserung.core :as v])
(:import (knossos.model Model)))
```
Remember how we chose to model our operations as reads, writes, and cas operations?
```clj
(defn r [_ _] {:type :invoke, :f :read, :value nil})
(defn w [_ _] {:type :invoke, :f :write, :value (rand-int 5)})
(defn cas [_ _] {:type :invoke, :f :cas, :value [(rand-int 5) (rand-int 5)]})
```
Jepsen doesn't know what `:f :read` or `:f :cas` mean. As far as it's
concerned, they're arbitrary values. However, our *client* understands how to
interpret those operations, when it dispatches based on `(case (:f op) :read
...)`. Now we need a *model* of the system which understands those same
operations. Knossos defines a Model data type for us, which takes a model and an
operation to apply, and returns a new model resulting from that operation. Here's that code, inside `knossos.model`:
```clj
(definterface+ Model
(step [model op]
"The job of a model is to *validate* that a sequence of operations
applied to it is consistent. Each invocation of (step model op)
returns a new state of the model, or, if the operation was
inconsistent with the model's state, returns a (knossos/inconsistent
msg). (reduce step model history) then validates that a particular
history is valid, and returns the final state of the model.
Models should be a pure, deterministic function of their state and an
operation's :f and :value."))
```
It turns out that the Knossos checker defines some common models for things
like locks and registers. Here's one for [a compare-and-set register](https://github.com/jepsen-io/knossos/blob/443a5a081c76be315eb01c7990cc7f1d9e41ed9b/src/knossos/model.clj#L66-L80)--exactly the datatype we're modeling.
```clj
(defrecord CASRegister [value]
Model
(step [r op]
(condp = (:f op)
:write (CASRegister. (:value op))
:cas (let [[cur new] (:value op)]
(if (= cur value)
(CASRegister. new)
(inconsistent (str "can't CAS " value " from " cur
" to " new))))
:read (if (or (nil? (:value op))
(= value (:value op)))
r
(inconsistent (str "can't read " (:value op)
" from register " value))))))
```
We don't need to write these in our tests, as long as `knossos` provides a
model for the type of thing we're checking. This is just so you can see how
things work under the hood.
This defrecord defines a new data type called `CASRegister`, which has a
single, immutable field, called `value`. It satisfies the `Model` interface we
discussed earlier, and its `step` function takes a current register `r`, and an
operation `op`. When we want to write a new value, we simply return a new
`CASRegister` with that value assigned. To compare-and-set from one value to
another, we pull apart the current and new values from the operation, and if
the current and new values match, construct a fresh register with the new
value. If they don't match, we return a special type of model with
`inconsistent`, which indicates that that operation could not be applied to the
register. Reads are similar, except that we always allow reads of `nil` to pass
through. This allows us to satisfy histories which include reads that never
returned.
To analyze the history, we'll specify a `:checker` for the test, and provide a
`:model` to specify how the system *should* behave. `checker/linearizable` uses
the Knossos linearizability checker to verify that every operation appears to
take place atomically between its invocation and completion. The linearizable
checker requires a model and to specify a particular algorithm which we pass to
it in an options map.
```clj
(defn etcd-test
"Given an options map from the command line runner (e.g. :nodes, :ssh,
:concurrency ...), constructs a test map."
[opts]
(merge tests/noop-test
opts
{:pure-generators true
:name "etcd"
:os debian/os
:db (db "v3.1.5")
:client (Client. nil)
:checker (checker/linearizable
{:model (model/cas-register)
:algorithm :linear})
:generator (->> (gen/mix [r w cas])
(gen/stagger 1)
(gen/nemesis nil)
(gen/time-limit 15))}))
```
Running the test, we can confirm the checker's results:
```bash
$ lein run test
...
INFO [2019-04-17 17:38:16,855] jepsen worker 0 - jepsen.util 0 :invoke :write 1
INFO [2019-04-17 17:38:16,861] jepsen worker 0 - jepsen.util 0 :ok :write 1
...
INFO [2019-04-18 03:53:32,714] jepsen test runner - jepsen.core {:valid? true,
:configs
({:model #knossos.model.CASRegister{:value 3},
:last-op
{:process 1,
:type :ok,
:f :write,
:value 3,
:index 29,
:time 14105346871},
:pending []}),
:analyzer :linear,
:final-paths ()}
Everything looks good! ヽ(‘ー`)ノ
```
The last operation in this history was a write of `1`, and sure enough, the
checker's final value is also `1`. This history was linearizable.
## Multiple checkers
Checkers can render all kinds of output--as data structures, images, or
interactive visualizations. For instance, if we have `gnuplot` installed,
Jepsen can generate throughput and latency graphs for us. Let's use
`checker/compose` to run both a linearizability analysis, and generate
performance graphs.
```clj
:checker (checker/compose
{:perf (checker/perf)
:linear (checker/linearizable {:model (model/cas-register)
:algorithm :linear})})
```
```bash
$ lein run test
...
$ open store/latest/latency-raw.png
```
We can also generate HTML visualizations of the history. Let's add the `jepsen.checker.timeline` namespace:
```clj
(ns jepsen.etcdemo
(:require ...
[jepsen.checker.timeline :as timeline]
...))
```
And add that checker to the test:
```clj
:checker (checker/compose
{:perf (checker/perf)
:linear (checker/linearizable
{:model (model/cas-register)
:algorithm :linear})
:timeline (timeline/html)})
```
Now we can plot how different processes performed operations over time--which
ones were concurrent, which ones succeeded, failed, or crashed, and so on.
```bash
$ lein run test
...
$ open store/latest/timeline.html
```
Now that we've got a passing test, it's time to [introduce
failures](05-nemesis.md) into the system.
================================================
FILE: doc/tutorial/05-nemesis.md
================================================
# Introducing Faults
The nemesis is a special client, not bound to any particular node, which
introduces failures across the cluster. We'll require `jepsen.nemesis`, which
provides several built-in failure modes.
```clj
(ns jepsen.etcdemo
(:require [clojure.tools.logging :refer :all]
[clojure.string :as str]
[jepsen [checker :as checker]
[cli :as cli]
[client :as client]
[control :as c]
[db :as db]
[generator :as gen]
[nemesis :as nemesis]
[tests :as tests]]
[jepsen.checker.timeline :as timeline]
[jepsen.control.util :as cu]
[jepsen.os.debian :as debian]
[knossos.model :as model]
[slingshot.slingshot :refer [try+]]
[verschlimmbesserung.core :as v]))
```
We'll pick a simple nemesis to start, and add it to the `:nemesis` key for the
test. This one partitions the network into two halves, selected randomly, when
it receives a `:start` op, and heals the network when it receives a `:stop`.
```clj
(defn etcd-test
"Given an options map from the command line runner (e.g. :nodes, :ssh,
:concurrency ...), constructs a test map."
[opts]
(merge tests/noop-test
opts
{:pure-generators true
:name "etcd"
:os debian/os
:db (db "v3.1.5")
:client (Client. nil)
:nemesis (nemesis/partition-random-halves)
:checker (checker/compose
{:perf (checker/perf)
:linear (checker/linearizable
{:model (model/cas-register)
:algorithm :linear})
:timeline (timeline/html)})
:generator (->> (gen/mix [r w cas])
(gen/stagger 1)
(gen/nemesis nil)
(gen/time-limit 15))}))
```
Like regular clients, the nemesis draws operations from the generator. Right
now our generator only emits ops to regular clients--the nemesis just gets
`nil`, which tells it there's nothing to do. We'll replace that with a
dedicated generator for nemesis operations. We're also going to increase the time limit, so we have enough time to see the nemesis take effect.
```clj
:generator (->> (gen/mix [r w cas])
(gen/stagger 1)
(gen/nemesis
(cycle [(gen/sleep 5)
{:type :info, :f :start}
(gen/sleep 5)
{:type :info, :f :stop}]))
(gen/time-limit 30))}
```
Clojure sequences can act as generators, so we can use regular Clojure
functions to construct them. Here, we use `cycle` to construct an infinite loop
of sleep, start, sleep, stop, ..., which ends once the time limit is up.
The network partition causes some operations to crash:
```clj
WARN [2018-02-02 15:54:53,380] jepsen worker 1 - jepsen.core Process 1 crashed
java.net.SocketTimeoutException: Read timed out
```
... and so on. If we *know* an operation didn't take place we can make the
checker more efficient (and detect more bugs!) by returning ops with `:type
:fail` instead of letting `client/invoke!` throw exceptions, but letting every
error crash the process is still safe: jepsen's checkers understand that a
crashed operation may or may not take place.
## Finding a bug
We've hardcoded a 30 second time limit into our test, but it'd be nice if we
could control that at the command line. Jepsen's CLI kit provides a
`--time-limit` switch, which is passed to `etcd-test` as `:time-limit`, in the
options map. Let's hook that up now:
```clj
:generator (->> (gen/mix [r w cas])
(gen/stagger 1)
(gen/nemesis
(gen/seq (cycle [(gen/sleep 5)
{:type :info, :f :start}
(gen/sleep 5)
{:type :info, :f :stop}])))
(gen/time-limit (:time-limit opts)))}
```
```bash
$ lein run test --time-limit 60
...
```
Now that we can run tests for shorter or longer, let's speed up the request rate. If we take too long between requests, we won't have a chance to see interesting behaviors. Let's try a tenth of a second between requests:
```clj
:generator (->> (gen/mix [r w cas])
(gen/stagger 1/50)
(gen/nemesis
(cycle [(gen/sleep 5)
{:type :info, :f :start}
(gen/sleep 5)
{:type :info, :f :stop}]))
(gen/time-limit (:time-limit opts)))}
```
If you run this test a few times, you might notice an interesting result.
Sometimes, it fails!
```clj
$ lein run test --test-count 10
...
:model {:msg "can't read 3 from register 4"}}]
...
Analysis invalid! (ﾉಥ益ಥ）ﾉ ┻━┻
```
Knossos ran out of options: it thought the only legal value for the register
was 4, but a process successfully read 3. When a linearizability failure
occurs, Knossos will emit an SVG diagram showing the problem--and we can read
the history to see the op in more detail.
```clj
$ open store/latest/linear.svg
$ open store/latest/history.txt
```
This is a case of a stale read: we saw a value from the *past*, despite more
recent writes having completed. This occurs because etcd allows us to read the
local state of any replica, without going through consensus to ensure we have
the most recent state.
## Read consistency
The etcd docs claim "etcd ensures linearizability for all [operations other
than watches] by default." This is clearly not the case--and indeed, buried in
the [v2 API docs](https://coreos.com/etcd/docs/latest/v2/api.html) is this
unobtrusive note:
> If you want a read that is fully linearized you can use a quorum=true GET. The read will take a very similar path to a write and will have a similar speed. If you are unsure if you need this feature feel free to email etcd-dev for advice.
Aha! So we need to use *quorum* reads. Verschlimmbesserung has an option for
that:
```clj
(invoke! [this test op]
(case (:f op)
:read (let [value (-> conn
(v/get "foo" {:quorum? true})
parse-long-nil)]
(assoc op :type :ok, :value value))
...
```
Introducing quorum reads makes our tests pass!
```bash
$ lein run test
...
Everything looks good! ヽ(‘ー`)ノ
```
Congratulations! You've written your first successful Jepsen test. This is the
same issue I identified in
[2014](https://aphyr.com/posts/316-jepsen-etcd-and-consul), and dialogue with
the etcd team led them to introduce the *quorum* read option.
Take a quick break! You've earned it! Then, if you like, we can move onto [refining the test](06-refining.md).
================================================
FILE: doc/tutorial/06-refining.md
================================================
# Refining Tests
Our test identified a fault, but it took some luck and clever guessing to
stumble upon it. It's time to refine our test, to make it faster, easier to
understand, and more powerful.
In order to analyze the history of a single key, Jepsen searches through every
permutation of concurrent operations, looking for a history that follows the
rules of a compare-and-set register. This means our search is exponential in
the number of concurrent operations at any given point in time.
Jepsen runs with a fixed number of worker threads, which would ordinarily limit
the number of concurrent operations too. However, when operations *crash*
(either by returning an `:info` result or throwing an exception), we abandon
that operation and let the thread move on to something new. It might be the
case that the crashed process' operation is still in-flight, and might be
executed by the database at some later time. This implies that crashed
operations are *concurrent for the entire remainder of the history.*
The more crashed operations, the more operations are concurrent by the end of
the history. That linear increase in concurrency is accompanied by an
exponential increase in verification time. Our first order of business is
reducing the number of crashed operations. We'll start with reads.
## Crashed reads
When an operation times out, we get a long stacktrace like
```
WARN [2018-02-02 16:14:37,588] jepsen worker 1 - jepsen.core Process 11 crashed
java.net.SocketTimeoutException: Read timed out
at java.net.SocketInputStream.socketRead0(Native Method) ~[na:1.8.0_40]
...
```
... and that process' operation is converted to an `:info` message, because we
can't tell if it succeeded or failed. However, *idempotent* operations, like
reads, leave the state of the system unchanged. It doesn't *matter* whether
they succeed or fail, because the effects are equivalent. We can therefore
safely convert crashed reads to failed reads, and improve checker performance.
```clj
(invoke! [_ test op]
(case (:f op)
:read (try (let [value (-> conn
(v/get "foo" {:quorum? true})
parse-long-nil)]
(assoc op :type :ok, :value value))
(catch java.net.SocketTimeoutException ex
(assoc op :type :fail, :error :timeout)))
:write (do (v/reset! conn "foo" (:value op))
(assoc op :type :ok))
:cas (try+
(let [[old new] (:value op)]
(assoc op :type (if (v/cas! conn "foo" old new)
:ok
:fail)))
(catch [:errorCode 100] ex
(assoc op :type :fail, :error :not-found)))))
```
Better yet--we can get rid of all the exception stacktrace noise in the logs if
we catch socket timeouts on all three paths at once. We'll handle not-found
errors there too, even though they only happen on `:cas` ops--it keeps the code
a little cleaner.
```clj
(invoke! [_ test op]
(try+
(case (:f op)
:read (let [value (-> conn
(v/get "foo" {:quorum? true})
parse-long-nil)]
(assoc op :type :ok, :value value))
:write (do (v/reset! conn "foo" (:value op))
(assoc op :type :ok))
:cas (let [[old new] (:value op)]
(assoc op :type (if (v/cas! conn "foo" old new)
:ok
:fail))))
(catch java.net.SocketTimeoutException e
(assoc op
:type (if (= :read (:f op)) :fail :info)
:error :timeout))
(catch [:errorCode 100] e
(assoc op :type :fail, :error :not-found))))
```
Now we get nice short timeout errors on *all* operations, not just reads.
```bash
INFO [2017-03-31 19:34:47,351] jepsen worker 4 - jepsen.util 4 :info :cas [4 4] :timeout
```
## Independent keys
We have a working test for a single linearizable key. However, sooner or later
processes *are* going to crash, and our concurrency will rise, slowing down the
analysis. We need a way to *bound* the length of a history on a particular key,
while still performing enough operations to observe concurrency errors.
Since operations on independent keys linearize independently, we can *lift* our
single-key test into one which operates on multiple keys. The
`jepsen.independent` namespace provides support.
```clj
(ns jepsen.etcdemo
(:require [clojure.tools.logging :refer :all]
[clojure.string :as str]
[jepsen [checker :as checker]
[cli :as cli]
[client :as client]
[control :as c]
[db :as db]
[generator :as gen]
[independent :as independent]
[nemesis :as nemesis]
[tests :as tests]]
[jepsen.checker.timeline :as timeline]
[jepsen.control.util :as cu]
[jepsen.os.debian :as debian]
[knossos.model :as model]
[slingshot.slingshot :refer [try+]]
[verschlimmbesserung.core :as v]))
```
We have a generator that emits operations on a single key, like `{:type :invoke,
:f :write, :value 3}`. We want to lift that to an operation that writes
*multiple* keys. Instead of `:value v`, we want `:value [key v]`.
```clj
:generator (->> (independent/concurrent-generator
10
(range)
(fn [k]
(->> (gen/mix [r w cas])
(gen/stagger 1/50)
(gen/limit 100))))
(gen/nemesis
(cycle [(gen/sleep 5)
{:type :info, :f :start}
(gen/sleep 5)
{:type :info, :f :stop}]))
(gen/time-limit (:time-limit opts)))}))
```
Our mix of reads, writes, and cas ops is still in there, but it's been wrapped
up in a *function*, which takes a key and returns a generator of values for
that particular key. We're using `concurrent-generator` to have 10 threads per
key, with keys taken from the infinite sequence of integers `(range)`, and
generators for those keys derived from `(fn [k] ...)`.
`concurrent-generator` changes the shape of our values from `v` to `[k v]`, so
we need to modify our client to understand how to read and write to different
keys.
```clj
(invoke! [_ test op]
(let [[k v] (:value op)]
(try+
(case (:f op)
:read (let [value (-> conn
(v/get k {:quorum? true})
parse-long-nil)]
(assoc op :type :ok, :value (independent/tuple k value)))
:write (do (v/reset! conn k v)
(assoc op :type :ok))
:cas (let [[old new] v]
(assoc op :type (if (v/cas! conn k old new)
:ok
:fail))))
(catch java.net.SocketTimeoutException e
(assoc op
:type (if (= :read (:f op)) :fail :info)
:error :timeout))
(catch [:errorCode 100] e
(assoc op :type :fail, :error :not-found)))))
```
See how our hardcoded key `"foo"` is gone? Now every key is parameterized by the
operation itself. Also note that where we modify the value--for instance, in
`:f :read`--we have to construct a special `independent/tuple` for the
key/value pair. Having a special datatype for tuples allows
`jepsen.independent` to split up the history later.
Finally, our checker thinks in terms of a single value--but we can turn that
into a checker that reasons about *independent* values, identified by keys.
```clj
:checker (checker/compose
{:perf (checker/perf)
:indep (independent/checker
(checker/compose
{:linear (checker/linearizable {:model (model/cas-register)
:algorithm :linear})
:timeline (timeline/html)}))})
```
Write one checker, get a family of n checkers for free! Maaaaagic!
```bash
$ lein run test --time-limit 30
...
ERROR [2017-03-31 19:51:28,300] main - jepsen.cli Oh jeez, I'm sorry, Jepsen broke. Here's why:
java.util.concurrent.ExecutionException: java.lang.AssertionError: Assert failed: This jepsen.independent/concurrent-generator has 5 threads to work with, but can only use 0 of those threads to run 0 concurrent keys with 10 threads apiece. Consider raising or lowering the test's :concurrency to a multiple of 10.
```
Aha. Our default concurrency is 5 threads, but we're asking for at least 10 in order to run a single key. Let's run 10 keys, using 100 threads.
```bash
$ lein run test --time-limit 30 --concurrency 100
...
142 :invoke :read [134 nil]
67 :invoke :read [133 nil]
66 :ok :read [133 1]
101 :ok :read [137 3]
181 :ok :write [135 3]
116 :ok :read [131 3]
111 :fail :cas [131 [0 0]]
151 :invoke :read [138 nil]
129 :ok :write [130 2]
159 :ok :read [138 1]
64 :ok :write [133 0]
69 :ok :cas [133 [0 0]]
109 :ok :cas [137 [4 3]]
89 :ok :read [135 1]
139 :ok :read [139 4]
19 :fail :cas [131 [2 1]]
124 :fail :cas [130 [4 4]]
```
Look at that! We can perform far more operations in a limited time window now. This helps us find bugs faster.
We've hardcoded a lot so far. Let's make some of those choices [configurable](07-parameters.md) at the command line.
================================================
FILE: doc/tutorial/07-parameters.md
================================================
# Tuning with Parameters
We allowed our last test to pass by including a `quorum` flag on reads, but in order to see the original stale-reads bug, we have to edit the source code again, flipping the flag to `false`. It'd be nice if we could adjust that from the command line. Jepsen provides several command-line options by default in [jepsen.cli](https://github.com/jepsen-io/jepsen/blob/0.1.7/jepsen/src/jepsen/cli.clj#L52-L87), but we can add our own options by passing an `:opt-spec` to `cli/single-test-cmd`.
```clj
(def cli-opts
"Additional command line options."
[["-q" "--quorum" "Use quorum reads, instead of reading from any primary."]])
```
CLI options are a collection of vectors, giving a short name, a full name, a
documentation string, and options which affect how that option is parsed, its
default value, etc. These are passed to
[tools.cli](https://github.com/clojure/tools.cli), the standard Clojure library
for option handling.
Now, let's pass that option specification to the CLI:
```clj
(defn -main
"Handles command line arguments. Can either run a test, or a web server for
browsing results."
[& args]
(cli/run! (merge (cli/single-test-cmd {:test-fn etcd-test
:opt-spec cli-opts})
(cli/serve-cmd))
args)
```
If we re-run our test with `lein run test -q ...`, we'll see a new `:quorum` option in our test map:
```clj
10:02:42.532 [main] INFO jepsen.cli - Test options:
{:concurrency 10,
:test-count 1,
:time-limit 30,
:quorum true,
...
```
Jepsen parsed our `-q` option, found the option specification we provided, and
added a `:quorum true` pair to the options map. That options map was passed to
`etcd-test`, which `merge`d it into the test map. Viola! We have a `:quorum`
key in our test!
Now, let's use that quorum option to control whether the client issues quorum
reads, in the Client `invoke` function:
```clj
(case (:f op)
:read (let [value (-> conn
(v/get k {:quorum? (:quorum test)})
parse-long-nil)]
(assoc op :type :ok, :value (independent/tuple k value)))
```
Let's try `lein run` with and without quorum reads, and see whether it lets us
see the stale reads bug again.
```bash
$ lein run test -q ...
...
$ lein run test ...
...
clojure.lang.ExceptionInfo: throw+: {:errorCode 209, :message "Invalid field", :cause "invalid value for \"quorum\"", :index 0, :status 400}
...
```
Huh. Let's double-check what the value was for `:quorum` in the test map. It's logged at the beginning of every Jepsen run:
```clj
2018-02-04 09:53:24,867{GMT} INFO [jepsen test runner] jepsen.core: Running test:
{:concurrency 10,
:db
#object[jepsen.etcdemo$db$reify__4946 0x15a8bbe5 "jepsen.etcdemo$db$reify__4946@15a8bbe5"],
:name "etcd",
:start-time
#object[org.joda.time.DateTime 0x54a5799f "2018-02-04T09:53:24.000-06:00"],
:net
#object[jepsen.net$reify__3493 0x2a2b3aff "jepsen.net$reify__3493@2a2b3aff"],
:client {:conn nil},
:barrier
#object[java.util.concurrent.CyclicBarrier 0x6987b74e "java.util.concurrent.CyclicBarrier@6987b74e"],
:ssh
{:username "root",
:password "root",
:strict-host-key-checking false,
:private-key-path nil},
:checker
#object[jepsen.checker$compose$reify__3220 0x71098fb3 "jepsen.checker$compose$reify__3220@71098fb3"],
:nemesis
#object[jepsen.nemesis$partitioner$reify__3601 0x47c15468 "jepsen.nemesis$partitioner$reify__3601@47c15468"],
:active-histories #<Atom@18bf1bad: #{}>,
:nodes ["n1" "n2" "n3" "n4" "n5"],
:test-count 1,
:generator
#object[jepsen.generator$time_limit$reify__1996 0x483fe83a "jepsen.generator$time_limit$reify__1996@483fe83a"],
:os
#object[jepsen.os.debian$reify__2908 0x8aa1562 "jepsen.os.debian$reify__2908@8aa1562"],
:time-limit 30,
:model {:value nil}}
```
Oh. That's odd. There... *isn't* a `:quorum` key here. Option flags only appear
in the options map if they're present on the command line; if they're left out
of the command line, they're left out of the option map too. When we ask for
`(:quorum test)`, and `test` *has* no `:quorum` option, we'll get `nil`.
There are a few easy ways to fix this. We could coerce `nil` to `false` by
using `(boolean (:quorum test))`, at the client, or in `etcd-test`. Or we could
force the opt spec to provide a default value when the flag is omitted, by
adding `:default false` to the quorum opt-spec. We'll apply `boolean` in
`etcd-test`, just in case someone calls it directly, instead of through the
CLI.
```clj
(defn etcd-test
"Given an options map from the command line runner (e.g. :nodes, :ssh,
:concurrency ...), constructs a test map. Special options:
:quorum Whether to use quorum reads"
[opts]
(let [quorum (boolean (:quorum opts))]
(merge tests/noop-test
opts
{:pure-generators true
:name (str "etcd q=" quorum)
:quorum quorum
...
```
We're binding `quorum` to a variable here so that we can use its boolean value
in two places. We add it to the test's `name`, which makes it easy to tell
which tests used quorum reads at a glance. We also add it to the `:quorum`
option. Since we merge `opts` *before* that, our boolean version of `:quorum`
will take precedence over whatever in `opts`. Now, without `-q`, our test can
find errors again:
```bash
$ lein run test --time-limit 60 --concurrency 100 -q
...
Everything looks good! ヽ(‘ー`)ノ
$ lein run test --time-limit 60 --concurrency 100
...
Analysis invalid! (ﾉಥ益ಥ）ﾉ ┻━┻
```
## Tunable difficulty
Depending on how powerful your computer is, you may have noticed some tests get
stuck on painfully slow analyses. It's hard to control this up-front--the
difficulty of a test goes like `~n!`, where n is the number of concurrent
processes. A couple crashed processes can make the difference between seconds
and days to check.
To help with this problem, let's add some tuning options to our test which
control the number of operations you can perform on any single key, and how
fast operations are generated.
In the generator, let's change our hardcoded 1/10 delay to a parameter, given
as a rate per second, and change our hardcoded limit on each key's generator to a configurable parameter.
```clj
(defn etcd-test
"Given an options map from the command line runner (e.g. :nodes, :ssh,
:concurrency ...), constructs a test map."
[opts]
(let [quorum (boolean (:quorum opts))]
(merge tests/noop-test
opts
{:pure-generators true
:name (str "etcd q=" quorum)
:quorum quorum
:os debian/os
:db (db "v3.1.5")
:client (Client. nil)
:nemesis (nemesis/partition-random-halves)
:checker (checker/compose
{:perf (checker/perf)
:indep (independent/checker
(checker/compose
{:linear (checker/linearizable
{:model (model/cas-register)
:algorithm :linear})
:timeline (timeline/html)}))})
:generator (->> (independent/concurrent-generator
10
(range)
(fn [k]
(->> (gen/mix [r w cas])
(gen/stagger (/ (:rate opts)))
(gen/limit (:ops-per-key opts)))))
(gen/nemesis
(->> [(gen/sleep 5)
{:type :info, :f :start}
(gen/sleep 5)
{:type :info, :f :stop}]
cycle))
(gen/time-limit (:time-limit opts)))})))
```
And add corresponding command-line options
```clj
(def cli-opts
"Additional command line options."
[["-q" "--quorum" "Use quorum reads, instead of reading from any primary."]
["-r" "--rate HZ" "Approximate number of requests per second, per thread."
:default 10
:parse-fn read-string
:validate [#(and (number? %) (pos? %)) "Must be a positive number"]]
[nil "--ops-per-key NUM" "Maximum number of operations on any given key."
:default 100
:parse-fn parse-long
:validate [pos? "Must be a positive integer."]]])
```
We don't have to provide a short name for every option: we use `nil` to
indicate that `--ops-per-key` has no short form. The capital words after each
flag (e.g. "HZ" & "NUM") are arbitrary placeholders for values that you would
pass. They'll be printed as a part of the usage documentation. We provide a
`:default` for both options, which is used if there's no flag at the command
line. For rates, we want to allow integers, decimals, and fractions, so...
we'll use Clojure's built-in `read-string` function to parse all three. Then
we'll validate that it's both a number and that it's positive, to keep people
from passing strings, negative numbers, zero rates, etc.
Now, if we want to run a less aggressive test, we can try
```bash
$ lein run test --time-limit 10 --concurrency 10 --ops-per-key 10 -r 1
...
Everything looks good! ヽ(‘ー`)ノ
```
Looking through the history for each key, we can see that operations proceeded
very slowly, and there are only 10 per key. This test is much easier to check!
However, it also fails to find the bug! This is an inherent tension in Jepsen:
we have to be aggressive to find errors, but verifying those aggressive
histories can be *much* more difficult--even impossible.
Linearizability checking is NP-hard; there's no way around that. We can design
somewhat more efficient checkers, but eventually, that exponential cliff is
going to bite us. Perhaps, however... we could verify a *weaker* property.
Something in linear or logarithmic time. Let's [add a commutative
test](08-set.md)
================================================
FILE: doc/tutorial/08-set.md
================================================
# Adding a Set Test
We can model an etcd cluster as a set of registers, each identified by a key,
which support reads, writes, and compare-and-sets. But that's not the only
possible system we could build on top of etcd. We could, for instance, treat it
as a set of keys, and ignore the values altogether. Or we could implement a
queue on top of an etcd directory. In theory, we could model *every part* of
the etcd API, but the state space would be quite large, and the implementation
perhaps time-consuming. Typically, we'll focus on important or often-used parts
of the API.
But what makes a test *useful*? Our linearizable test is quite general,
performing different types of randomized operations, and determining whether
any pattern of those operations is linearizable. However, it is also quite
expensive. It'd be nice if we could design a test which is simple to verify,
but still tells us something useful.
Consider a set, supporting `add` and `read` operations. If we only read, our
test is trivially satisfied by seeing the empty set. If we only write, every
test will always pass, since it is always legal to add something to a set.
Clearly, we need a combination of reads and writes. Moreover, a read should be
the *last* thing that happens, since any writes *after* the final read couldn't
affect the outcome of the test.
What elements should we add? If we always add the same element, the test has
some resolving power: if every add returns `ok` but we don't read that element,
we know we've found a bug. However, if *any* add works, then the final read
will include that element, and we won't be able to tell if the other adds
actually worked or not. Perhaps it is most useful, then, to choose *distinct*
elements, so that every add operation has some independent effect on the read. If we choose ordered elements, we can get a rough picture of whether loss is evenly distributed over time, or occurs in chunks, so we'll do that as well.
Our operations, then, will be something like
```clj
{:type :invoke, :f :add, :value 0}
{:type :invoke, :f :add, :value 1}
...
{:type :invoke, :f :read, :value #{0 1}}
```
And we'll know the database performed correctly if every successful add is
present in the final read. We could obtain more information by performing
multiple reads, and tracking which have completed or are in-flight, but for
now, let's keep it simple.
## A New Namespace
It's starting to get a little cluttered in `jepsen.etcdemo`, so we're going to
break things up into a dedicated namespace for our new test. We'll call it `jepsen.etcdemo.set`:
```sh
$ mkdir src/jepsen/etcdemo
$ vim src/jepsen/etcdemo/set.clj
```
We'll be designing a new client and a generator, so we'll need those namespaces
from Jepsen. And of course we'll be using our etcd client library,
Verschlimmbesserung--and we'll want to handle exceptions from it, so that means
Slingshot too.
```clj
(ns jepsen.etcdemo.set
(:require [jepsen
[checker :as checker]
[client :as client]
[generator :as gen]]
[slingshot.slingshot :refer [try+]]
[verschlimmbesserung.core :as v]))
```
We'll need a new client that can add things to sets, and read them back--but we
have to choose how to store that set in the database. One option is to use
separate keys, or a pool of keys. Another is to use a single key, and have the
value be a serialized data type, like a JSON array or Clojure set. We'll do the latter.
```clj
(defrecord SetClient [k conn]
client/Client
(open! [this test node]
(assoc this :conn (v/connect (client-url node)
```
Oh. That's a problem. We don't have `client-url` here. We could pull it in from
`jepsen.etcdemo`, but we'd like to require *this* namespace from
`jepsen.etcdemo` later, and Clojure tries very hard to avoid circular
dependencies between namespaces. Let's create a new, supporting namespace
called `jepsen.etcdemo.support`. Like `jepsen.etcdemo.set`, it'll have its own
file.
```bash
$ vim src/jepsen/etcdemo/support.clj
```
Let's move the url-constructing functions from `jepsen.etcdemo` to
`jepsen.etcdemo.support`.
```clj
(ns jepsen.etcdemo.support
(:require [clojure.string :as str]))
(defn node-url
"An HTTP url for connecting to a node on a particular port."
[node port]
(str "http://" node ":" port))
(defn peer-url
"The HTTP url for other peers to talk to a node."
[node]
(node-url node 2380))
(defn client-url
"The HTTP url clients use to talk to a node."
[node]
(node-url node 2379))
(defn initial-cluster
"Constructs an initial cluster string for a test, like
\"foo=foo:2380,bar=bar:2380,...\""
[test]
(->> (:nodes test)
(map (fn [node]
(str node "=" (peer-url node))))
(str/join ",")))
```
Now we'll require our support namespace from `jepsen.etcdemo`, and replace
calls to those
gitextract_id8dqfc2/
├── .gitignore
├── .travis.yml
├── README.md
├── antithesis/
│ ├── .gitignore
│ ├── CHANGELOG.md
│ ├── LICENSE
│ ├── README.md
│ ├── composer/
│ │ ├── check
│ │ └── op
│ ├── doc/
│ │ └── intro.md
│ ├── project.clj
│ ├── src/
│ │ └── jepsen/
│ │ ├── antithesis/
│ │ │ ├── Random.java
│ │ │ └── composer.clj
│ │ └── antithesis.clj
│ └── test/
│ └── jepsen/
│ ├── antithesis/
│ │ └── composer_test.clj
│ └── antithesis_test.clj
├── charybdefs/
│ ├── .gitignore
│ ├── README.md
│ ├── project.clj
│ ├── src/
│ │ └── jepsen/
│ │ └── charybdefs.clj
│ └── test/
│ ├── .gitignore
│ └── jepsen/
│ └── charybdefs/
│ └── remote_test.clj
├── contributing.md
├── doc/
│ ├── color.md
│ ├── lxc-f36.md
│ ├── lxc.md
│ ├── plan.md
│ ├── tutorial/
│ │ ├── 01-scaffolding.md
│ │ ├── 02-db.md
│ │ ├── 03-client.md
│ │ ├── 04-checker.md
│ │ ├── 05-nemesis.md
│ │ ├── 06-refining.md
│ │ ├── 07-parameters.md
│ │ ├── 08-set.md
│ │ └── index.md
│ └── whats-here.md
├── docker/
│ ├── .gitignore
│ ├── README.md
│ ├── bin/
│ │ ├── build-docker-compose
│ │ ├── console
│ │ ├── up
│ │ └── web
│ ├── control/
│ │ ├── .gitignore
│ │ ├── Dockerfile
│ │ ├── bashrc
│ │ └── init.sh
│ ├── docker-compose.dev.yml
│ ├── node/
│ │ ├── Dockerfile
│ │ └── setup-jepsen.sh
│ ├── secret/
│ │ └── .gitkeep
│ └── template/
│ ├── db.yml
│ ├── depends.yml
│ └── docker-compose.yml
├── generator/
│ ├── .gitignore
│ ├── CHANGELOG.md
│ ├── LICENSE
│ ├── README.md
│ ├── doc/
│ │ └── intro.md
│ ├── project.clj
│ ├── src/
│ │ └── jepsen/
│ │ ├── generator/
│ │ │ ├── context.clj
│ │ │ ├── test.clj
│ │ │ └── translation_table.clj
│ │ ├── generator.clj
│ │ └── random.clj
│ └── test/
│ └── jepsen/
│ ├── generator/
│ │ ├── context_test.clj
│ │ └── translation_table_test.clj
│ ├── generator_test.clj
│ └── random_test.clj
├── jepsen/
│ ├── .eastwood.clj
│ ├── LICENSE.txt
│ ├── project.clj
│ ├── resources/
│ │ ├── bump-time.c
│ │ ├── corrupt-file.c
│ │ ├── log4j.properties
│ │ ├── strobe-time-experiment.c
│ │ └── strobe-time.c
│ ├── src/
│ │ └── jepsen/
│ │ ├── adya.clj
│ │ ├── checker/
│ │ │ ├── clock.clj
│ │ │ ├── perf.clj
│ │ │ ├── plot.clj
│ │ │ └── timeline.clj
│ │ ├── checker.clj
│ │ ├── cli.clj
│ │ ├── client.clj
│ │ ├── codec.clj
│ │ ├── control/
│ │ │ ├── clj_ssh.clj
│ │ │ ├── core.clj
│ │ │ ├── docker.clj
│ │ │ ├── k8s.clj
│ │ │ ├── net.clj
│ │ │ ├── retry.clj
│ │ │ ├── scp.clj
│ │ │ ├── sshj.clj
│ │ │ └── util.clj
│ │ ├── control.clj
│ │ ├── core.clj
│ │ ├── db/
│ │ │ └── watchdog.clj
│ │ ├── db.clj
│ │ ├── faketime.clj
│ │ ├── fs_cache.clj
│ │ ├── generator/
│ │ │ └── interpreter.clj
│ │ ├── independent.clj
│ │ ├── lazyfs.clj
│ │ ├── nemesis/
│ │ │ ├── combined.clj
│ │ │ ├── file.clj
│ │ │ ├── membership/
│ │ │ │ └── state.clj
│ │ │ ├── membership.clj
│ │ │ └── time.clj
│ │ ├── nemesis.clj
│ │ ├── net/
│ │ │ └── proto.clj
│ │ ├── net.clj
│ │ ├── os/
│ │ │ ├── centos.clj
│ │ │ ├── debian.clj
│ │ │ ├── smartos.clj
│ │ │ └── ubuntu.clj
│ │ ├── os.clj
│ │ ├── print.clj
│ │ ├── reconnect.clj
│ │ ├── repl.clj
│ │ ├── report.clj
│ │ ├── role.clj
│ │ ├── store/
│ │ │ ├── FileOffsetOutputStream.java
│ │ │ ├── FressianReader.java
│ │ │ ├── format.clj
│ │ │ └── fressian.clj
│ │ ├── store.clj
│ │ ├── tests/
│ │ │ ├── adya.clj
│ │ │ ├── bank.clj
│ │ │ ├── causal.clj
│ │ │ ├── causal_reverse.clj
│ │ │ ├── cycle/
│ │ │ │ ├── append.clj
│ │ │ │ └── wr.clj
│ │ │ ├── cycle.clj
│ │ │ ├── kafka.clj
│ │ │ ├── linearizable_register.clj
│ │ │ └── long_fork.clj
│ │ ├── tests.clj
│ │ ├── util.clj
│ │ └── web.clj
│ └── test/
│ └── jepsen/
│ ├── checker/
│ │ └── timeline_test.clj
│ ├── checker_test.clj
│ ├── cli_test.clj
│ ├── common_test.clj
│ ├── control/
│ │ ├── net_test.clj
│ │ └── util_test.clj
│ ├── control_test.clj
│ ├── core_test.clj
│ ├── db/
│ │ └── watchdog_test.clj
│ ├── db_test.clj
│ ├── fs_cache_test.clj
│ ├── generator/
│ │ └── interpreter_test.clj
│ ├── generator_test.clj
│ ├── independent_test.clj
│ ├── lazyfs_test.clj
│ ├── nemesis/
│ │ ├── combined_test.clj
│ │ ├── file_test.clj
│ │ └── time_test.clj
│ ├── nemesis_test.clj
│ ├── perf_test.clj
│ ├── print_test.clj
│ ├── role_test.clj
│ ├── store/
│ │ └── format_test.clj
│ ├── store_test.clj
│ ├── tests/
│ │ ├── causal_reverse_test.clj
│ │ ├── kafka_test.clj
│ │ └── long_fork_test.clj
│ └── util_test.clj
└── txn/
├── .gitignore
├── CHANGELOG.md
├── LICENSE
├── README.md
├── doc/
│ └── intro.md
├── project.clj
├── src/
│ └── jepsen/
│ ├── txn/
│ │ └── micro_op.clj
│ └── txn.clj
└── test/
└── jepsen/
└── txn_test.clj
SYMBOL INDEX (106 symbols across 7 files)
FILE: antithesis/src/jepsen/antithesis/Random.java
class Random (line 9) | public class Random implements RandomGenerator {
method Random (line 14) | public Random() {
method nextLong (line 19) | public long nextLong() {
method nextDouble (line 23) | public double nextDouble() {
method nextBoolean (line 31) | public boolean nextBoolean() {
method randomChoice (line 35) | public <T> T randomChoice(List<T> list) {
FILE: jepsen/resources/bump-time.c
function main (line 7) | int main(int argc, char **argv) {
FILE: jepsen/resources/corrupt-file.c
type ExitCode (line 40) | enum ExitCode {
type Options (line 48) | enum Options {
type argp_option (line 55) | struct argp_option
type Modes (line 90) | enum Modes {
type opts_t (line 102) | typedef struct {
function opts_t (line 115) | opts_t default_opts() {
function print_opts (line 130) | void print_opts(opts_t opts) {
function validate_opts (line 146) | int validate_opts(opts_t opts) {
function error_t (line 189) | static error_t parse_opt(int key, char *arg, struct argp_state *state) {
type argp (line 255) | struct argp
function off_t (line 260) | off_t rand_int(off_t max) {
function off_t (line 273) | off_t rand_exp_int(double lambda) {
function mkdir_p (line 279) | int mkdir_p(const char *path) {
function off_t (line 300) | off_t chunk_offset(opts_t opts, off_t chunk) {
function off_t (line 305) | off_t chunk_count(opts_t opts, off_t file_size) {
function corrupt_snapshot (line 337) | int corrupt_snapshot(opts_t opts, int fd, off_t file_size, off_t
function corrupt_restore (line 414) | int corrupt_restore(opts_t opts, int fd, off_t file_size, off_t
function off_t (line 486) | off_t rand_source_offset(opts_t opts, off_t dest_offset, off_t file_size) {
function corrupt_copy (line 516) | int corrupt_copy(opts_t opts, int fd, off_t file_size, off_t chunk_count) {
function corrupt_bitflip (line 564) | int corrupt_bitflip(opts_t opts, int fd, off_t file_size,
function corrupt (line 676) | int corrupt(opts_t opts) {
function clear_snapshots (line 722) | int clear_snapshots() {
function main (line 734) | int main (int argc, char **argv) {
FILE: jepsen/resources/strobe-time-experiment.c
function nanos_to_timespec (line 19) | struct timespec nanos_to_timespec(int64_t nanos) {
function timespec_to_nanos (line 30) | int64_t nanos timespec_to_nanos(struct timespec t) {
function monotonic_now (line 38) | struct timespec monotonic_now() {
function wall_now (line 45) | struct timespec wall_now() {
function wall_tz (line 58) | struct timezone wall_tz() {
function set_wall_clock (line 69) | void set_wall_clock(struct timespec ts, struct timezone tz) {
function balance_timespec_m (line 80) | void balance_timespec_m(struct timespec *t) {
function add_timespec (line 92) | struct timespec add_timespec(struct timespec a, struct timespec b) {
function sub_timespec (line 101) | struct timespec sub_timespec(struct timespec a, struct timespec b) {
function mod_timespec (line 110) | struct timespec mod_timespec(struct timespec a, struct timespec n) {
function cmp_timespec (line 115) | int8_t cmp_timespec(struct timespec a, struct timespec b) {
function next_tick (line 134) | struct timespec next_tick(struct timespec dt, struct timespec anchor, st...
function sleep_until_next_tick (line 141) | void sleep_until_next_tick(struct timespec dt, struct timespec anchor) {
function main (line 151) | int main(int argc, char **argv) {
FILE: jepsen/resources/strobe-time.c
function nanos_to_timespec (line 19) | struct timespec nanos_to_timespec(int64_t nanos) {
function monotonic_now (line 29) | struct timespec monotonic_now() {
function wall_now (line 36) | struct timespec wall_now() {
function set_wall_clock (line 46) | void set_wall_clock(struct timespec ts) {
function balance_timespec_m (line 55) | void balance_timespec_m(struct timespec *t) {
function add_timespec (line 67) | struct timespec add_timespec(struct timespec a, struct timespec b) {
function sub_timespec (line 76) | struct timespec sub_timespec(struct timespec a, struct timespec b) {
function cmp_timespec (line 85) | int8_t cmp_timespec(struct timespec a, struct timespec b) {
function main (line 101) | int main(int argc, char **argv) {
FILE: jepsen/src/jepsen/store/FileOffsetOutputStream.java
class FileOffsetOutputStream (line 12) | public class FileOffsetOutputStream extends OutputStream implements Auto...
method FileOffsetOutputStream (line 19) | public FileOffsetOutputStream(FileChannel file, long offset, CRC32 che...
method bytesWritten (line 29) | public long bytesWritten() {
method checksum (line 34) | public CRC32 checksum() {
method close (line 38) | public void close() {
method flush (line 41) | public void flush() throws IOException {
method write (line 45) | public void write(int b) throws IOException {
method write (line 57) | public void write(byte[] bs) throws IOException {
method write (line 66) | public void write(byte[] bs, int offset, int len) throws IOException {
FILE: jepsen/src/jepsen/store/FressianReader.java
class FressianReader (line 29) | public class FressianReader implements org.fressian.Reader, Closeable {
method FressianReader (line 37) | public FressianReader(InputStream is) {
method FressianReader (line 41) | public FressianReader(InputStream is, ILookup<Object, ReadHandler> han...
method FressianReader (line 45) | public FressianReader(InputStream is, ILookup<Object, ReadHandler> han...
method readBoolean (line 52) | public boolean readBoolean() throws IOException {
method readInt (line 71) | public long readInt() throws IOException {
method internalReadInt (line 75) | private long internalReadInt() throws IOException {
method readDouble (line 258) | public double readDouble() throws IOException {
method readFloat (line 264) | public float readFloat() throws IOException {
method readObject (line 283) | public Object readObject() throws IOException {
method read (line 287) | private Object read(int code) throws IOException {
method handleStruct (line 698) | private Object handleStruct(Object tag, int fields) throws IOException {
method readCount (line 708) | private int readCount() throws IOException {
method internalReadInt32 (line 712) | private int internalReadInt32() throws IOException {
method readInt32 (line 716) | private int readInt32() throws IOException {
method internalReadString (line 720) | private StringBuilder internalReadString(int length) throws IOException {
method internalReadStringBuilder (line 724) | private StringBuilder internalReadStringBuilder(StringBuilder buf, int...
method internalReadChunkedString (line 732) | private String internalReadChunkedString(int length) throws IOException {
method internalReadBytes (line 765) | private byte[] internalReadBytes(int length) throws IOException {
method internalReadChunkedBytes (line 771) | private byte[] internalReadChunkedBytes() throws IOException {
method getHandler (line 795) | private Object getHandler(String tag) {
method internalReadDouble (line 803) | private double internalReadDouble(int code) throws IOException {
method readObjects (line 822) | private Object[] readObjects(int length) throws IOException {
method readClosedList (line 830) | private Object[] readClosedList() throws IOException {
method readOpenList (line 841) | private Object[] readOpenList() throws IOException {
method close (line 857) | public void close() throws IOException {
class MapEntry (line 861) | static class MapEntry implements Map.Entry {
method MapEntry (line 865) | public MapEntry(Object key, Object value) {
method getKey (line 870) | public Object getKey() {
method getValue (line 874) | public Object getValue() {
method setValue (line 878) | public Object setValue(Object o) {
method lookupCache (line 886) | private Object lookupCache(ArrayList cache, int index) {
method internalReadList (line 898) | private List internalReadList(int length) throws IOException {
method validateFooter (line 902) | private void validateFooter(int calculatedLength, int magicFromStream)...
method getPriorityCache (line 915) | private ArrayList<Object> getPriorityCache() {
method getStructCache (line 920) | private ArrayList<Object> getStructCache() {
method resetCaches (line 925) | private void resetCaches() {
method validateFooter (line 930) | public void validateFooter() throws IOException {
method readNextCode (line 936) | private int readNextCode() throws IOException {
method readAndCacheObject (line 940) | private Object readAndCacheObject(ArrayList<Object> cache) throws IOEx...
method convertList (line 953) | public List convertList(Object[] items) {
method convertBytes (line 959) | public Object convertBytes(byte[] bytes) {
method convertDouble (line 965) | public Object convertDouble(double d) {
method convertFloat (line 971) | public Object convertFloat(float f) {
Condensed preview — 177 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (1,475K chars).
[
{
"path": ".gitignore",
"chars": 427,
"preview": ".cake\n.lein-repl-history\n.rebel_readline_history\npom.xml\npom.xml.asc\nrepl-port\n*.jar\n*.tar\n*.tar.bz2\n*.war\n*.deb\n*~\n.*.s"
},
{
"path": ".travis.yml",
"chars": 623,
"preview": "language: clojure\nlein: lein\ndist: trusty\njdk:\n - oraclejdk8\nbranches:\n only:\n - master\naddons:\n apt:\n package"
},
{
"path": "README.md",
"chars": 11670,
"preview": "# Jepsen\n\nBreaking distributed systems so you don't have to.\n\nJepsen is a Clojure library. A test is a Clojure program w"
},
{
"path": "antithesis/.gitignore",
"chars": 125,
"preview": "/target\n/classes\n/checkouts\nprofiles.clj\npom.xml\npom.xml.asc\n*.jar\n*.class\n/.lein-*\n/.nrepl-port\n/.prepl-port\n.hgignore\n"
},
{
"path": "antithesis/CHANGELOG.md",
"chars": 785,
"preview": "# Change Log\nAll notable changes to this project will be documented in this file. This change log follows the convention"
},
{
"path": "antithesis/LICENSE",
"chars": 14372,
"preview": "Eclipse Public License - v 2.0\n\n THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE\n PUBLIC LICE"
},
{
"path": "antithesis/README.md",
"chars": 3909,
"preview": "# jepsen.antithesis\n\nThis library supports running Jepsen tests inside Antithesis environments. It\nprovides entropy, lif"
},
{
"path": "antithesis/composer/check",
"chars": 164,
"preview": "#!/bin/bash\n\nset -e\n\nDIR=\"/tmp/jepsen/antithesis\"\nmkdir -p \"$DIR\"\n\nFIFO=\"${DIR}/check\"\n\n# Create a FIFO\n#echo \"$FIFO\"\nmk"
},
{
"path": "antithesis/composer/op",
"chars": 234,
"preview": "#!/bin/bash\n\nset -e\n\nDIR=\"/tmp/jepsen/antithesis\"\n\nmkdir -p \"$DIR\"\n\n# Pick a hopefully unique ID for this op\nID=\"$(date "
},
{
"path": "antithesis/doc/intro.md",
"chars": 109,
"preview": "# Introduction to antithesis\n\nTODO: write [great documentation](https://jacobian.org/writing/what-to-write/)\n"
},
{
"path": "antithesis/project.clj",
"chars": 966,
"preview": "(defproject io.jepsen/antithesis \"0.1.1-SNAPSHOT\"\n :description \"Support for running Jepsen inside Antithesis\"\n :url \""
},
{
"path": "antithesis/src/jepsen/antithesis/Random.java",
"chars": 978,
"preview": "// A java.util.random which draws from Antithesis' SDK.\n\npackage jepsen.antithesis;\n\nimport java.util.Arrays;\nimport jav"
},
{
"path": "antithesis/src/jepsen/antithesis/composer.clj",
"chars": 22265,
"preview": "(ns jepsen.antithesis.composer\n \"Antithesis' Test Composer drives Jepsen by calling shell scripts, which\n communicate "
},
{
"path": "antithesis/src/jepsen/antithesis.clj",
"chars": 15152,
"preview": "(ns jepsen.antithesis\n \"Provides support for running Jepsen tests in Antithesis. Provides an RNG,\n lifecycle hooks, an"
},
{
"path": "antithesis/test/jepsen/antithesis/composer_test.clj",
"chars": 6442,
"preview": "(ns jepsen.antithesis.composer-test\n (:refer-clojure :exclude [run!])\n (:require [clojure [edn :as edn]\n "
},
{
"path": "antithesis/test/jepsen/antithesis_test.clj",
"chars": 9014,
"preview": "(ns jepsen.antithesis-test\n (:require [clojure [pprint :refer [pprint]]\n [test :refer :all]]\n "
},
{
"path": "charybdefs/.gitignore",
"chars": 99,
"preview": "/target\n/classes\n/checkouts\npom.xml\npom.xml.asc\n*.jar\n*.class\n/.lein-*\n/.nrepl-port\n.hgignore\n.hg/\n"
},
{
"path": "charybdefs/README.md",
"chars": 245,
"preview": "# charybdefs\n\nA wrapper around [CharybdeFS](https://github.com/scylladb/charybdefs) for use\nin a jepsen nemesis.\n\n## Usa"
},
{
"path": "charybdefs/project.clj",
"chars": 374,
"preview": "(defproject jepsen-charybdefs \"0.1.0-SNAPSHOT\"\n :description \"charybdefs wrapper for use in jepsen\"\n :url \"https://git"
},
{
"path": "charybdefs/src/jepsen/charybdefs.clj",
"chars": 2883,
"preview": "(ns jepsen.charybdefs\n (:require [clojure.tools.logging :refer [info]]\n [jepsen.control :as c]\n ["
},
{
"path": "charybdefs/test/.gitignore",
"chars": 11,
"preview": "config.edn\n"
},
{
"path": "charybdefs/test/jepsen/charybdefs/remote_test.clj",
"chars": 1108,
"preview": "(ns jepsen.charybdefs.remote-test\n (:require [clojure.test :refer :all]\n [jepsen.control :as c]\n "
},
{
"path": "contributing.md",
"chars": 725,
"preview": "# Contributing to Jepsen\n\nHi there, and thanks for helping make Jepsen better! I've got just one request:\nstart your com"
},
{
"path": "doc/color.md",
"chars": 115,
"preview": "Color scheme:\n\nLight:\n\nok #6DB6FE\ninfo #FFAA26\nfail #FEB5DA\n\nDark:\n\nok: #81BFFC\ninfo: #FFA400\nfail: #FF1E90\n"
},
{
"path": "doc/lxc-f36.md",
"chars": 7892,
"preview": "# How to set up nodes via LXC\n\n## Fedora 36\n\nAs a user can be sudoer:\n\n```\nsudo dnf install -y openssh-server\nsudo dnf -"
},
{
"path": "doc/lxc.md",
"chars": 10195,
"preview": "# How to set up nodes via LXC\n\n#### [Mint 22.3 Zena](#mint-22.3-zena-1)\n#### [Debian 13/trixie - Incus](#debian-13trixie"
},
{
"path": "doc/plan.md",
"chars": 1810,
"preview": "# Stuff to improve!\n\n## Error handling\n\n- Knossos: Better error messages when users pass models that fail on the\n first"
},
{
"path": "doc/tutorial/01-scaffolding.md",
"chars": 12225,
"preview": "# Test scaffolding\n\nIn this tutorial, we're going to write a test for etcd: a distributed consensus\nsystem. I want to en"
},
{
"path": "doc/tutorial/02-db.md",
"chars": 13391,
"preview": "# Database automation\n\nIn a Jepsen test, a `DB` encapsulates code for setting up and tearing down a\ndatabase, queue, or "
},
{
"path": "doc/tutorial/03-client.md",
"chars": 17109,
"preview": "# Writing a client\n\nA Jepsen *client* takes *invocation operations* and applies them to the system\nbeing tested, returni"
},
{
"path": "doc/tutorial/04-checker.md",
"chars": 7695,
"preview": "# Checking Correctness\n\nWith our generator and clients performing operations, we've got a history to\nanalyze for correct"
},
{
"path": "doc/tutorial/05-nemesis.md",
"chars": 7252,
"preview": "# Introducing Faults\n\nThe nemesis is a special client, not bound to any particular node, which\nintroduces failures acros"
},
{
"path": "doc/tutorial/06-refining.md",
"chars": 9853,
"preview": "# Refining Tests\n\nOur test identified a fault, but it took some luck and clever guessing to\nstumble upon it. It's time t"
},
{
"path": "doc/tutorial/07-parameters.md",
"chars": 10308,
"preview": "# Tuning with Parameters\n\nWe allowed our last test to pass by including a `quorum` flag on reads, but in order to see th"
},
{
"path": "doc/tutorial/08-set.md",
"chars": 19674,
"preview": "# Adding a Set Test\n\nWe can model an etcd cluster as a set of registers, each identified by a key,\nwhich support reads, "
},
{
"path": "doc/tutorial/index.md",
"chars": 754,
"preview": "# Tutorial\n\nThis tutorial will walk you through writing a Jepsen test from scratch. It is\nalso the basis for a [training"
},
{
"path": "doc/whats-here.md",
"chars": 9127,
"preview": "# What's Here\n\nThis document provides an overview of Jepsen's various namespaces and how they\nwork together.\n\n## Core Na"
},
{
"path": "docker/.gitignore",
"chars": 40,
"preview": "secret/*\n!.gitkeep\n./docker-compose.yml\n"
},
{
"path": "docker/README.md",
"chars": 1438,
"preview": "# Dockerized Jepsen\n\nThis docker image attempts to simplify the setup required by Jepsen.\nIt is intended to be used by a"
},
{
"path": "docker/bin/build-docker-compose",
"chars": 838,
"preview": "#!/usr/bin/env bash\n\n# Builds a docker-compose file. You'd THINK we could do this with `replicas`\n# but nooooooo, down t"
},
{
"path": "docker/bin/console",
"chars": 56,
"preview": "#!/usr/bin/env bash\ndocker exec -it jepsen-control bash\n"
},
{
"path": "docker/bin/up",
"chars": 5573,
"preview": "#!/usr/bin/env bash\n\n# \"To provide additional docker compose args, set the COMPOSE var. Ex:\n# COMPOSE=\"-f FILE_PATH_HERE"
},
{
"path": "docker/bin/web",
"chars": 111,
"preview": "#!/usr/bin/env bash\n\nPORT=$(docker port jepsen-control 8080 | cut -d : -f 2)\nxdg-open \"http://localhost:$PORT\"\n"
},
{
"path": "docker/control/.gitignore",
"chars": 6,
"preview": "jepsen"
},
{
"path": "docker/control/Dockerfile",
"chars": 1345,
"preview": "FROM jgoerzen/debian-base-minimal:bookworm as debian-addons\nFROM debian:bookworm-slim\n\nCOPY --from=debian-addons /usr/lo"
},
{
"path": "docker/control/bashrc",
"chars": 431,
"preview": "eval $(ssh-agent) &> /dev/null\nssh-add /root/.ssh/id_rsa &> /dev/null\n\ncat <<EOF\nWelcome to Jepsen on Docker\n==========="
},
{
"path": "docker/control/init.sh",
"chars": 856,
"preview": "#!/bin/sh\n\n: \"${SSH_PRIVATE_KEY?SSH_PRIVATE_KEY is empty, please use up.sh}\"\n: \"${SSH_PUBLIC_KEY?SSH_PUBLIC_KEY is empty"
},
{
"path": "docker/docker-compose.dev.yml",
"chars": 124,
"preview": "services:\n control:\n volumes:\n - ${JEPSEN_ROOT}:/jepsen # Mounts $JEPSEN_ROOT on host to /jepsen control contai"
},
{
"path": "docker/node/Dockerfile",
"chars": 1468,
"preview": "# See https://salsa.debian.org/jgoerzen/docker-debian-base\n# See https://hub.docker.com/r/jgoerzen/debian-base-standard\n"
},
{
"path": "docker/node/setup-jepsen.sh",
"chars": 714,
"preview": "#!/bin/bash\n\n# We add our hostname to the shared volume, so that control can find us\necho \"Adding hostname to shared vol"
},
{
"path": "docker/secret/.gitkeep",
"chars": 0,
"preview": ""
},
{
"path": "docker/template/db.yml",
"chars": 88,
"preview": " n%%N%%:\n << : *default-node\n container_name: jepsen-n%%N%%\n hostname: n%%N%%\n"
},
{
"path": "docker/template/depends.yml",
"chars": 49,
"preview": " n%%N%%:\n condition: service_healthy\n"
},
{
"path": "docker/template/docker-compose.yml",
"chars": 1308,
"preview": "x-node:\n &default-node\n privileged: true\n build: ./node\n env_file: ./secret/node.env\n secrets:\n - authorized_key"
},
{
"path": "generator/.gitignore",
"chars": 125,
"preview": "/target\n/classes\n/checkouts\nprofiles.clj\npom.xml\npom.xml.asc\n*.jar\n*.class\n/.lein-*\n/.nrepl-port\n/.prepl-port\n.hgignore\n"
},
{
"path": "generator/CHANGELOG.md",
"chars": 797,
"preview": "# Change Log\nAll notable changes to this project will be documented in this file. This change log follows the convention"
},
{
"path": "generator/LICENSE",
"chars": 14372,
"preview": "Eclipse Public License - v 2.0\n\n THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE\n PUBLIC LICE"
},
{
"path": "generator/README.md",
"chars": 1371,
"preview": "# jepsen.generator\n\nThis library provides the compositional generator system at the heart of\n[Jepsen](https://jepsen.io)"
},
{
"path": "generator/doc/intro.md",
"chars": 115,
"preview": "# Introduction to jepsen.generator\n\nTODO: write [great documentation](https://jacobian.org/writing/what-to-write/)\n"
},
{
"path": "generator/project.clj",
"chars": 932,
"preview": "(defproject io.jepsen/generator \"0.1.2-SNAPSHOT\"\n :description \"Pure functional generators for Jepsen tests\"\n :url \"ht"
},
{
"path": "generator/src/jepsen/generator/context.clj",
"chars": 13529,
"preview": "(ns jepsen.generator.context\n \"Generators work with an immutable *context* that tells them what time it is,\n what proc"
},
{
"path": "generator/src/jepsen/generator/test.clj",
"chars": 7675,
"preview": "(ns jepsen.generator.test\n \"This namespace contains functions for testing generators. See the\n `jepsen.generator-test`"
},
{
"path": "generator/src/jepsen/generator/translation_table.clj",
"chars": 4009,
"preview": "(ns jepsen.generator.translation-table\n \"We burn a lot of time in hashcode and map manipulation for thread names,\n whi"
},
{
"path": "generator/src/jepsen/generator.clj",
"chars": 71348,
"preview": "(ns jepsen.generator\n \"# In a Nutshell\n\n Generators tell Jepsen what to do during a test. Generators are purely\n func"
},
{
"path": "generator/src/jepsen/random.clj",
"chars": 18629,
"preview": "(ns jepsen.random\n \"Pluggable generation of random values.\n\n ## Pluggable\n\n First, randomness should be pluggable. In"
},
{
"path": "generator/test/jepsen/generator/context_test.clj",
"chars": 4311,
"preview": "(ns jepsen.generator.context-test\n (:require [clojure [datafy :refer [datafy]]\n [pprint :refer [ppr"
},
{
"path": "generator/test/jepsen/generator/translation_table_test.clj",
"chars": 836,
"preview": "(ns jepsen.generator.translation-table-test\n (:require [clojure [test :refer :all]\n [pprint :refer "
},
{
"path": "generator/test/jepsen/generator_test.clj",
"chars": 22545,
"preview": "(ns jepsen.generator-test\n (:require [jepsen.generator [context :as ctx]\n [test :as gen.te"
},
{
"path": "generator/test/jepsen/random_test.clj",
"chars": 9965,
"preview": "(ns jepsen.random-test\n (:require [jepsen.random :as r]\n [clojure [pprint :refer [pprint]]\n "
},
{
"path": "jepsen/.eastwood.clj",
"chars": 822,
"preview": "(disable-warning\n {:linter :constant-test\n :for-macro 'dom-top.core/assert+\n :if-inside-macroexpansion-of #{'clojure.c"
},
{
"path": "jepsen/LICENSE.txt",
"chars": 11408,
"preview": "Eclipse Public License -v 1.0\n=============================\n\nTHE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THI"
},
{
"path": "jepsen/project.clj",
"chars": 3304,
"preview": "(defproject jepsen \"0.3.12-SNAPSHOT\"\n :description \"Distributed systems testing framework.\"\n :url \"https://jep"
},
{
"path": "jepsen/resources/bump-time.c",
"chars": 1435,
"preview": "#define _POSIX_C_SOURCE 200809L\n#include <stdio.h>\n#include <time.h>\n#include <stdlib.h>\n#include <stdint.h>\n\nint main(i"
},
{
"path": "jepsen/resources/corrupt-file.c",
"chars": 22215,
"preview": "#define _GNU_SOURCE\n#define _FILE_OFFSET_BITS 64\n\n#define OFF_MAX (sizeof(off_t) == sizeof(long long) ? LLONG_MAX : size"
},
{
"path": "jepsen/resources/log4j.properties",
"chars": 450,
"preview": "# Based on the example properties given at http://logging.apache.org/log4j/1.2/manual.html\n# Set root logger level to DE"
},
{
"path": "jepsen/resources/strobe-time-experiment.c",
"chars": 6130,
"preview": "#include <stdio.h>\n#include <time.h>\n#include <sys/time.h>\n#include <stdlib.h>\n#include <stdint.h>\n\nconst int64_t NANOS_"
},
{
"path": "jepsen/resources/strobe-time.c",
"chars": 4529,
"preview": "#include <stdio.h>\n#include <time.h>\n#include <sys/time.h>\n#include <stdlib.h>\n#include <stdint.h>\n\nconst int64_t NANOS_"
},
{
"path": "jepsen/src/jepsen/adya.clj",
"chars": 49,
"preview": "(ns jepsen.adya\n \"Moved to jepsen.tests.adya.\")\n"
},
{
"path": "jepsen/src/jepsen/checker/clock.clj",
"chars": 3239,
"preview": "(ns jepsen.checker.clock\n \"Helps analyze clock skew over time.\"\n (:require [clojure.core.reducers :as r]\n ["
},
{
"path": "jepsen/src/jepsen/checker/perf.clj",
"chars": 25199,
"preview": "(ns jepsen.checker.perf\n \"Supporting functions for performance analysis.\"\n (:require [clojure.stacktrace :as trace]\n "
},
{
"path": "jepsen/src/jepsen/checker/plot.clj",
"chars": 7719,
"preview": "(ns jepsen.checker.plot\n \"Draws plots as a part of the checker. This namespace should\n eventually subsume jepsen.check"
},
{
"path": "jepsen/src/jepsen/checker/timeline.clj",
"chars": 7535,
"preview": "(ns jepsen.checker.timeline\n \"Renders an HTML timeline of a history.\"\n (:require [clojure.core.reducers :as r]\n "
},
{
"path": "jepsen/src/jepsen/checker.clj",
"chars": 39570,
"preview": "(ns jepsen.checker\n \"Validates that a history is correct with respect to some model.\"\n (:refer-clojure :exclude [set])"
},
{
"path": "jepsen/src/jepsen/cli.clj",
"chars": 21704,
"preview": "(ns jepsen.cli\n \"Command line interface. Provides a default main method for common Jepsen\n functions (like the web int"
},
{
"path": "jepsen/src/jepsen/client.clj",
"chars": 5501,
"preview": "(ns jepsen.client\n \"Applies operations to a database.\"\n (:require [clojure.tools.logging :refer :all]\n [clo"
},
{
"path": "jepsen/src/jepsen/codec.clj",
"chars": 805,
"preview": "(ns jepsen.codec\n \"Serializes and deserializes objects to/from bytes.\"\n (:require [clojure.edn :as edn]\n [b"
},
{
"path": "jepsen/src/jepsen/control/clj_ssh.clj",
"chars": 3913,
"preview": "(ns jepsen.control.clj-ssh\n \"A CLJ-SSH powered implementation of the Remote protocol.\"\n (:require [clojure.tools.loggi"
},
{
"path": "jepsen/src/jepsen/control/core.clj",
"chars": 5742,
"preview": "(ns jepsen.control.core\n \"Provides the base protocol for running commands on remote nodes, as well as\n common function"
},
{
"path": "jepsen/src/jepsen/control/docker.clj",
"chars": 3123,
"preview": "(ns jepsen.control.docker\n \"The recommended way is to use SSH to setup and teardown databases. It's\n however sometimes"
},
{
"path": "jepsen/src/jepsen/control/k8s.clj",
"chars": 3507,
"preview": "(ns jepsen.control.k8s\n \"The recommended way is to use SSH to setup and teardown databases.\n It's however sometimes co"
},
{
"path": "jepsen/src/jepsen/control/net.clj",
"chars": 2521,
"preview": "(ns jepsen.control.net\n \"Network control functions.\"\n (:refer-clojure :exclude [partition])\n (:require [clojure.strin"
},
{
"path": "jepsen/src/jepsen/control/retry.clj",
"chars": 2588,
"preview": "(ns jepsen.control.retry\n \"SSH client libraries appear to be near universally-flaky. Maybe race\n conditions, maybe und"
},
{
"path": "jepsen/src/jepsen/control/scp.clj",
"chars": 5677,
"preview": "(ns jepsen.control.scp\n \"Built-in JDK SSH libraries can be orders of magnitude slower than plain old\n SCP for copying "
},
{
"path": "jepsen/src/jepsen/control/sshj.clj",
"chars": 8522,
"preview": "(ns jepsen.control.sshj\n \"An sshj-backed control Remote. Experimental; I'm considering replacing\n jepsen.control's use"
},
{
"path": "jepsen/src/jepsen/control/util.clj",
"chars": 18723,
"preview": "(ns jepsen.control.util\n \"Utility functions for scripting installations.\"\n (:require [jepsen [control :refer :all]\n "
},
{
"path": "jepsen/src/jepsen/control.clj",
"chars": 12026,
"preview": "(ns jepsen.control\n \"Provides control over a remote node. There's a lot of dynamically bound\n state in this namespace "
},
{
"path": "jepsen/src/jepsen/core.clj",
"chars": 17848,
"preview": "(ns jepsen.core\n \"Entry point for all Jepsen tests. Coordinates the setup of servers, running\n tests, creating and res"
},
{
"path": "jepsen/src/jepsen/db/watchdog.clj",
"chars": 6947,
"preview": "(ns jepsen.db.watchdog\n \"Databases often like to crash, and they may not restart themselves\n automatically, which mea"
},
{
"path": "jepsen/src/jepsen/db.clj",
"chars": 9315,
"preview": "(ns jepsen.db\n \"Allows Jepsen to set up and tear down databases.\"\n (:require [clojure [string :as str]]\n [c"
},
{
"path": "jepsen/src/jepsen/faketime.clj",
"chars": 2491,
"preview": "(ns jepsen.faketime\n \"Libfaketime is useful for making clocks run at differing rates! This\n namespace provides utiliti"
},
{
"path": "jepsen/src/jepsen/fs_cache.clj",
"chars": 9479,
"preview": "(ns jepsen.fs-cache\n \"Some systems Jepsen tests are expensive or time-consuming to set up. They\n might involve lengthy"
},
{
"path": "jepsen/src/jepsen/generator/interpreter.clj",
"chars": 15709,
"preview": "(ns jepsen.generator.interpreter\n \"This namespace interprets operations from a pure generator, handling worker\n thread"
},
{
"path": "jepsen/src/jepsen/independent.clj",
"chars": 16244,
"preview": "(ns jepsen.independent\n \"Some tests are expensive to check--for instance, linearizability--which\n requires we verify o"
},
{
"path": "jepsen/src/jepsen/lazyfs.clj",
"chars": 10032,
"preview": "(ns jepsen.lazyfs\n \"Lazyfs allows the injection of filesystem-level faults: specifically, losing\n data which was writt"
},
{
"path": "jepsen/src/jepsen/nemesis/combined.clj",
"chars": 22826,
"preview": "(ns jepsen.nemesis.combined\n \"A nemesis which combines common operations on nodes and processes: clock\n skew, crashes,"
},
{
"path": "jepsen/src/jepsen/nemesis/file.clj",
"chars": 10555,
"preview": "(ns jepsen.nemesis.file\n \"Fault injection involving files on disk. This nemesis can copy chunks\n randomly within a fil"
},
{
"path": "jepsen/src/jepsen/nemesis/membership/state.clj",
"chars": 2320,
"preview": "(ns jepsen.nemesis.membership.state\n \"This namespace defines the protocol for nemesis membership state\n machines---how"
},
{
"path": "jepsen/src/jepsen/nemesis/membership.clj",
"chars": 11259,
"preview": "(ns jepsen.nemesis.membership\n \"EXPERIMENTAL: provides standardized support for nemeses which add and remove\n nodes fr"
},
{
"path": "jepsen/src/jepsen/nemesis/time.clj",
"chars": 8036,
"preview": "(ns jepsen.nemesis.time\n \"Functions for messing with time and clocks.\"\n (:require [clojure.tools.logging :refer [info "
},
{
"path": "jepsen/src/jepsen/nemesis.clj",
"chars": 23416,
"preview": "(ns jepsen.nemesis\n (:require [clojure.set :as set]\n [clojure.java.io :as io]\n [clojure.tools.log"
},
{
"path": "jepsen/src/jepsen/net/proto.clj",
"chars": 1252,
"preview": "(ns jepsen.net.proto\n \"Protocols for network manipulation. High-level functions live in\n jepsen.net.\")\n\n(defprotocol N"
},
{
"path": "jepsen/src/jepsen/net.clj",
"chars": 10079,
"preview": "(ns jepsen.net\n \"Controls network manipulation.\n\n TODO: break this up into jepsen.net.proto (polymorphism) and jepsen."
},
{
"path": "jepsen/src/jepsen/os/centos.clj",
"chars": 4944,
"preview": "(ns jepsen.os.centos\n \"Common tasks for CentOS boxes.\"\n (:require [clojure.set :as set]\n [clojure.tools.log"
},
{
"path": "jepsen/src/jepsen/os/debian.clj",
"chars": 6114,
"preview": "(ns jepsen.os.debian\n \"Common tasks for Debian Trixie.\"\n (:require [clojure [set :as set]\n [string"
},
{
"path": "jepsen/src/jepsen/os/smartos.clj",
"chars": 4045,
"preview": "(ns jepsen.os.smartos\n \"Common tasks for SmartOS boxes.\"\n (:require [clojure.set :as set]\n [clojure.tools.l"
},
{
"path": "jepsen/src/jepsen/os/ubuntu.clj",
"chars": 1290,
"preview": "(ns jepsen.os.ubuntu\n \"Common tasks for Ubuntu boxes. Tested against Ubuntu 18.04.\"\n (:require [clojure.set :as set]\n "
},
{
"path": "jepsen/src/jepsen/os.clj",
"chars": 423,
"preview": "(ns jepsen.os\n \"Controls operating system setup and teardown.\")\n\n(defprotocol OS\n (setup! [os test node] \"Set up t"
},
{
"path": "jepsen/src/jepsen/print.clj",
"chars": 7466,
"preview": "(ns jepsen.print\n \"Handles printing and logging things as strings or to the console.\"\n (:require [clojure.string :as s"
},
{
"path": "jepsen/src/jepsen/reconnect.clj",
"chars": 6065,
"preview": "(ns jepsen.reconnect\n \"Stateful wrappers for automatically reconnecting network clients.\n\n A wrapper is a map with a c"
},
{
"path": "jepsen/src/jepsen/repl.clj",
"chars": 266,
"preview": "(ns jepsen.repl\n \"Helper functions for mucking around with tests!\"\n (:require [jepsen [history :as h]\n "
},
{
"path": "jepsen/src/jepsen/report.clj",
"chars": 469,
"preview": "(ns jepsen.report\n \"Prints out stuff.\"\n (:require [jepsen.util :as util]\n [clojure.java.io :as io]\n "
},
{
"path": "jepsen/src/jepsen/role.clj",
"chars": 6337,
"preview": "(ns jepsen.role\n \"Supports tests where each node has a single, distinct role. For instance,\n one node might run ZooKee"
},
{
"path": "jepsen/src/jepsen/store/FileOffsetOutputStream.java",
"chars": 2275,
"preview": "package jepsen.store.format;\n\nimport java.io.IOException;\nimport java.io.OutputStream;\nimport java.nio.ByteBuffer;\nimpor"
},
{
"path": "jepsen/src/jepsen/store/FressianReader.java",
"chars": 30463,
"preview": "// Copyright (c) Metadata Partners, LLC, with modifications by Jepsen, LLC.\n// All rights reserved. The use and dist"
},
{
"path": "jepsen/src/jepsen/store/format.clj",
"chars": 62729,
"preview": "(ns jepsen.store.format\n \"Jepsen tests are logically a map. To save this map to disk, we originally\n wrote it as a sin"
},
{
"path": "jepsen/src/jepsen/store/fressian.clj",
"chars": 10330,
"preview": "(ns jepsen.store.fressian\n \"Supports serialization of various Jepsen datatypes via Fressian.\"\n (:require [clojure.data"
},
{
"path": "jepsen/src/jepsen/store.clj",
"chars": 18584,
"preview": "(ns jepsen.store\n \"Persistent storage for test runs and later analysis.\"\n (:refer-clojure :exclude [load test])\n (:re"
},
{
"path": "jepsen/src/jepsen/tests/adya.clj",
"chars": 3300,
"preview": "(ns jepsen.tests.adya\n \"Generators and checkers for tests of Adya's proscribed behaviors for\n weakly-consistent system"
},
{
"path": "jepsen/src/jepsen/tests/bank.clj",
"chars": 6937,
"preview": "(ns jepsen.tests.bank\n \"Helper functions for doing bank tests, where you simulate transfers between\n accounts, and ver"
},
{
"path": "jepsen/src/jepsen/tests/causal.clj",
"chars": 4553,
"preview": "(ns jepsen.tests.causal\n (:refer-clojure :exclude [test])\n (:require [jepsen [checker :as checker]\n "
},
{
"path": "jepsen/src/jepsen/tests/causal_reverse.clj",
"chars": 4333,
"preview": "(ns jepsen.tests.causal-reverse\n \"Checks for a strict serializability anomaly in which T1 < T2, but T2 is\n visible wit"
},
{
"path": "jepsen/src/jepsen/tests/cycle/append.clj",
"chars": 1495,
"preview": "(ns jepsen.tests.cycle.append\n \"Detects cycles in histories where operations are transactions over named\n lists lists,"
},
{
"path": "jepsen/src/jepsen/tests/cycle/wr.clj",
"chars": 1435,
"preview": "(ns jepsen.tests.cycle.wr\n \"A test which looks for cycles in write/read transactions. Writes are assumed\n to be unique"
},
{
"path": "jepsen/src/jepsen/tests/cycle.clj",
"chars": 630,
"preview": "(ns jepsen.tests.cycle\n \"Tests based on transactional cycle detection via Elle. If you're looking for\n code that used "
},
{
"path": "jepsen/src/jepsen/tests/kafka.clj",
"chars": 113434,
"preview": "(ns jepsen.tests.kafka\n \"This workload is intended for systems which behave like the popular Kafka\n queue. This includ"
},
{
"path": "jepsen/src/jepsen/tests/linearizable_register.clj",
"chars": 2542,
"preview": "(ns jepsen.tests.linearizable-register\n \"Common generators and checkers for linearizability over a set of independent\n "
},
{
"path": "jepsen/src/jepsen/tests/long_fork.clj",
"chars": 11949,
"preview": "(ns jepsen.tests.long-fork\n \"Tests for an anomaly in parallel snapshot isolation (but which is prohibited\n in normal s"
},
{
"path": "jepsen/src/jepsen/tests.clj",
"chars": 2091,
"preview": "(ns jepsen.tests\n \"Provide utilities for writing tests using jepsen.\"\n (:require [jepsen.os :as os]\n [jepse"
},
{
"path": "jepsen/src/jepsen/util.clj",
"chars": 34878,
"preview": "(ns jepsen.util\n \"Kitchen sink\"\n (:refer-clojure :exclude [parse-long]) ; Clojure added this in 1.11.1\n (:require [cl"
},
{
"path": "jepsen/src/jepsen/web.clj",
"chars": 16200,
"preview": "(ns jepsen.web\n \"Web server frontend for browsing test results.\"\n (:require [jepsen.store :as store]\n [cloj"
},
{
"path": "jepsen/test/jepsen/checker/timeline_test.clj",
"chars": 2751,
"preview": "(ns jepsen.checker.timeline-test\n (:refer-clojure :exclude [set])\n (:require [clojure [datafy :refer [datafy]]\n "
},
{
"path": "jepsen/test/jepsen/checker_test.clj",
"chars": 32803,
"preview": "(ns jepsen.checker-test\n (:refer-clojure :exclude [set])\n (:use clojure.test)\n (:require [clojure [datafy :refer [dat"
},
{
"path": "jepsen/test/jepsen/cli_test.clj",
"chars": 701,
"preview": "(ns jepsen.cli-test\n (:require [clojure [test :refer :all]]\n [jepsen [cli :as cli]]))\n\n(deftest without-defa"
},
{
"path": "jepsen/test/jepsen/common_test.clj",
"chars": 1778,
"preview": "(ns jepsen.common-test\n \"Support functions for writing tests.\"\n (:require [clojure.tools.logging :refer :all]\n "
},
{
"path": "jepsen/test/jepsen/control/net_test.clj",
"chars": 482,
"preview": "(ns jepsen.control.net-test\n (:require [clojure [pprint :refer [pprint]]\n [test :refer :all]]\n "
},
{
"path": "jepsen/test/jepsen/control/util_test.clj",
"chars": 6276,
"preview": "(ns jepsen.control.util-test\n (:require [clojure.tools.logging :refer [info warn]]\n [clojure [string :as str"
},
{
"path": "jepsen/test/jepsen/control_test.clj",
"chars": 6475,
"preview": "(ns jepsen.control-test\n (:require [clojure [string :as str]\n [test :refer :all]]\n [cloj"
},
{
"path": "jepsen/test/jepsen/core_test.clj",
"chars": 14807,
"preview": "(ns jepsen.core-test\n (:refer-clojure :exclude [run!])\n (:require [clojure [pprint :refer [pprint]]\n "
},
{
"path": "jepsen/test/jepsen/db/watchdog_test.clj",
"chars": 3941,
"preview": "(ns jepsen.db.watchdog-test\n (:require [clojure [pprint :refer [pprint]]\n [test :refer :all]]\n "
},
{
"path": "jepsen/test/jepsen/db_test.clj",
"chars": 1742,
"preview": "(ns jepsen.db-test\n \"Tests for jepsen.db\"\n (:require [clojure [test :refer :all]]\n [jepsen [db :as db]]))\n\n"
},
{
"path": "jepsen/test/jepsen/fs_cache_test.clj",
"chars": 3354,
"preview": "(ns jepsen.fs-cache-test\n (:require [clojure [test :refer :all]]\n [clojure.java.io :as io]\n [cloj"
},
{
"path": "jepsen/test/jepsen/generator/interpreter_test.clj",
"chars": 13400,
"preview": "(ns jepsen.generator.interpreter-test\n (:refer-clojure :exclude [run!])\n (:require [clojure [data :refer [diff]]\n "
},
{
"path": "jepsen/test/jepsen/generator_test.clj",
"chars": 3116,
"preview": "(ns jepsen.generator-test\n (:require [jepsen.generator [context :as ctx]\n [test :as gen.te"
},
{
"path": "jepsen/test/jepsen/independent_test.clj",
"chars": 2366,
"preview": "(ns jepsen.independent-test\n (:require [clojure.test :refer :all]\n [clojure.pprint :refer [pprint]]\n "
},
{
"path": "jepsen/test/jepsen/lazyfs_test.clj",
"chars": 3809,
"preview": "(ns jepsen.lazyfs-test\n \"Tests for the lazyfs write-losing filesystem\"\n (:require [clojure [data :refer [diff]]\n "
},
{
"path": "jepsen/test/jepsen/nemesis/combined_test.clj",
"chars": 2618,
"preview": "(ns jepsen.nemesis.combined-test\n (:require [clojure [pprint :refer [pprint]]\n [test :refer :all]]\n"
},
{
"path": "jepsen/test/jepsen/nemesis/file_test.clj",
"chars": 25556,
"preview": "(ns jepsen.nemesis.file-test\n (:require [clj-commons.byte-streams :as bs]\n [clojure [pprint :refer [pprint]]"
},
{
"path": "jepsen/test/jepsen/nemesis/time_test.clj",
"chars": 1097,
"preview": "(ns jepsen.nemesis.time-test\n (:require [clojure [pprint :refer [pprint]]\n [test :refer :all]]\n "
},
{
"path": "jepsen/test/jepsen/nemesis_test.clj",
"chars": 7839,
"preview": "(ns jepsen.nemesis-test\n (:require [clojure [pprint :refer [pprint]]\n [set :as set]\n "
},
{
"path": "jepsen/test/jepsen/perf_test.clj",
"chars": 8025,
"preview": "(ns jepsen.perf-test\n (:refer-clojure :exclude [run!])\n (:use clojure.test)\n (:require [jepsen.os :as os]\n "
},
{
"path": "jepsen/test/jepsen/print_test.clj",
"chars": 787,
"preview": "(ns jepsen.print-test\n (:require [clojure [pprint]\n [test :refer :all]]\n [jepsen [histor"
},
{
"path": "jepsen/test/jepsen/role_test.clj",
"chars": 6345,
"preview": "(ns jepsen.role-test\n (:refer-clojure :exclude [test])\n (:require [clojure [pprint :refer [pprint]]\n "
},
{
"path": "jepsen/test/jepsen/store/format_test.clj",
"chars": 14011,
"preview": "(ns jepsen.store.format-test\n (:require [byte-streams :as bs]\n [clojure [pprint :refer [pprint]]\n "
},
{
"path": "jepsen/test/jepsen/store_test.clj",
"chars": 4073,
"preview": "(ns jepsen.store-test\n (:refer-clojure :exclude [load test])\n (:use clojure.test)\n (:require [clojure.data.fressian :"
},
{
"path": "jepsen/test/jepsen/tests/causal_reverse_test.clj",
"chars": 3621,
"preview": "(ns jepsen.tests.causal-reverse-test\n (:require [jepsen.tests.causal-reverse :refer :all]\n [clojure.test :re"
},
{
"path": "jepsen/test/jepsen/tests/kafka_test.clj",
"chars": 41810,
"preview": "(ns jepsen.tests.kafka-test\n (:require [clojure [pprint :refer [pprint]]\n [test :refer :all]\n "
},
{
"path": "jepsen/test/jepsen/tests/long_fork_test.clj",
"chars": 897,
"preview": "(ns jepsen.tests.long-fork-test\n (:require [clojure.test :refer :all]\n [clojure.pprint :refer [pprint]]\n "
},
{
"path": "jepsen/test/jepsen/util_test.clj",
"chars": 7800,
"preview": "(ns jepsen.util-test\n (:refer-clojure :exclude [parse-long])\n (:require [clojure [pprint :refer [pprint]]\n "
},
{
"path": "txn/.gitignore",
"chars": 99,
"preview": "/target\n/classes\n/checkouts\npom.xml\npom.xml.asc\n*.jar\n*.class\n/.lein-*\n/.nrepl-port\n.hgignore\n.hg/\n"
},
{
"path": "txn/CHANGELOG.md",
"chars": 774,
"preview": "# Change Log\nAll notable changes to this project will be documented in this file. This change log follows the convention"
},
{
"path": "txn/LICENSE",
"chars": 11219,
"preview": "THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC\nLICENSE (\"AGREEMENT\"). ANY USE, REPRODUCTION"
},
{
"path": "txn/README.md",
"chars": 2454,
"preview": "# Jepsen.txn\n\nSupport library for generating and analyzing transactional, multi-object\nhistories. This is very much a wo"
},
{
"path": "txn/doc/intro.md",
"chars": 108,
"preview": "# Introduction to jepsen.txn\n\nTODO: write [great documentation](http://jacobian.org/writing/what-to-write/)\n"
},
{
"path": "txn/project.clj",
"chars": 588,
"preview": "(defproject jepsen.txn \"0.1.4-SNAPSHOT\"\n :description \"Library for generating and analyzing multi-object transactional "
},
{
"path": "txn/src/jepsen/txn/micro_op.clj",
"chars": 603,
"preview": "(ns jepsen.txn.micro-op\n \"Transactions are made up of micro-operations. This namespace helps us work\n with those.\"\n ("
},
{
"path": "txn/src/jepsen/txn.clj",
"chars": 3084,
"preview": "(ns jepsen.txn\n \"Manipulates transactions. Transactions are represented as a sequence of\n micro-operations (mops for s"
},
{
"path": "txn/test/jepsen/txn_test.clj",
"chars": 771,
"preview": "(ns jepsen.txn-test\n (:require [clojure.test :refer :all]\n [criterium.core :refer [quick-bench bench with-pr"
}
]
About this extraction
This page contains the full source code of the jepsen-io/jepsen GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 177 files (1.4 MB), approximately 359.5k tokens, and a symbol index with 106 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.