[
{
"path": ".gitignore",
"content": "**/.DS_Store\n**/tla/*.toolbox/model\n**/tla/*.toolbox/*aux\n**/tla/*.toolbox/*.log\n**/tla/*.toolbox/*.pdf\n**/tla/*.toolbox/*.tex\n**/tla/*.toolbox/*___model_SnapShot*.launch\n**/tla/*.toolbox/**/*.tla\n**/tla/*.toolbox/**/*.out\n**/tla/*.toolbox/**/MC.cfg\n**/tla/*.pdf\n**/tla/*.old\n**/*~\ncluster/isabelle/output\n"
},
{
"path": "LICENSE",
"content": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n"
},
{
"path": "README.md",
"content": "# Formal models of core Elasticsearch algorithms\n\nThis repository contains formal models of core [Elasticsearch](https://github.com/elastic/elasticsearch) algorithms and is directly related to implementation efforts around [data replication](https://github.com/elastic/elasticsearch/issues/10708) and [cluster coordination](https://github.com/elastic/elasticsearch/issues/32006). The models in this repository might represent past, current and future designs of Elasticsearch and can differ to their implementations in substantial ways. The formal models mainly serve to illustrate some of the high-level concepts and help to validate resiliency-related aspects.\n\n## Models\n\n### Cluster coordination model\n\nThe cluster coordination TLA+ model ensures the consistency of cluster state updates and represents the core [cluster coordination](https://github.com/elastic/elasticsearch/issues/32006) and metadata replication algorithm implemented in Elasticsearch 7.0. It consists of two files:\n\n- [TLA+ specification](ZenWithTerms/tla/ZenWithTerms.tla) which has a [direct one-to-one implementation in Elasticsearch](https://github.com/elastic/elasticsearch/blob/master/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java)\n- [TLC model checking configuration](ZenWithTerms/tla/ZenWithTerms.toolbox/ZenWithTerms___model.launch)\n\n### Data replication model\n\nThe data replication TLA+ model describes the Elasticsearch [sequence number](https://github.com/elastic/elasticsearch/issues/10708) based data replication approach, implemented since Elasticsearch 6.0, which consists of two files:\n\n- [TLA+ specification](data/tla/replication.tla)\n- [TLC model checking configuration](data/tla/replication.toolbox/replication___model.launch)\n\n### Replica engine\n\nA TLA+ model of how the\n[engine](https://github.com/elastic/elasticsearch/blob/00fd73acc4a2991f96438f8c1948016c5b9eefb2/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java)\nhandles replication requests.\n\n- [TLA+ specification](ReplicaEngine/tla/ReplicaEngine.tla)\n- [TLC model checking configuration](ReplicaEngine/tla/ReplicaEngine.toolbox/ReplicaEngine___model.launch)\n\n### Alternative cluster coordination model\n\nThe alternative cluster coordination TLA+ model consists of two files:\n\n- [TLA+ specification](cluster/tla/consensus.tla)\n- [TLC model checking configuration](cluster/tla/consensus.toolbox/consensus___model.launch)\n\nThe alternative cluster consensus Isabelle model consists of the following theories:\n\n- [Basic definitions](cluster/isabelle/Preliminaries.thy)\n- [An implementation in functional style](cluster/isabelle/Implementation.thy)\n- [An implementation in monadic style, along with a proof it's equivalent to the previous](cluster/isabelle/Monadic.thy)\n- [The proof that each slot is consistent, based on Lamport's Synod algorithm](cluster/isabelle/OneSlot.thy)\n- [The proof that the implementation ensures consistency](cluster/isabelle/Zen.thy)\n\n## How to edit/run TLA+:\n\n- Install the [TLA Toolbox](http://research.microsoft.com/en-us/um/people/lamport/tla/toolbox.html)\n - If on Mac OS, [move the downloaded app to the Applications folder first](https://groups.google.com/forum/#!topic/tlaplus/bL04c6BiYxo)\n- Read some [documentation](http://research.microsoft.com/en-us/um/people/lamport/tla/book.html)\n\nHow to run the model checker in headless mode:\n\n- Download [tla2tools.jar](http://research.microsoft.com/en-us/um/people/lamport/tla/tools.html)\n- Run the model checker once in TLA+ Toolbox on desktop (can be aborted once started). This generates the folder `elasticsearch.toolbox/model/` that contains all model files that are required to run the model checker in headless mode.\n- Copy the above folder and `tla2tools.jar` to the server running in headless mode.\n- `cd` to the folder and run `java -Xmx30G -cp ../tla2tools.jar tlc2.TLC MC -deadlock -workers 12`. The setting `-Xmx30G` denotes the amount of memory to allocate to the model checker and `-workers 12` the number of worker threads (should be equal to the number of cores on machine). The setting `-deadlock` ensures that TLC explores the full reachable state space, not searching for deadlocks.\n"
},
{
"path": "ReplicaEngine/tla/ReplicaEngine.tla",
"content": "-------------------------- MODULE ReplicaEngine --------------------------\n\nEXTENDS Naturals, FiniteSets, Sequences, TLC\n\n(* Actions on the Lucene index *)\nCONSTANTS Lucene_addDocuments, Lucene_updateDocuments, Lucene_deleteDocuments\n\nCONSTANTS ADD, RETRY_ADD, UPDATE, DELETE, NULL\n\nCONSTANTS DocContent\n\nCONSTANTS DocAutoIdTimestamp\n\nCONSTANTS DuplicationLimit\n\n(* We model the activity of a single document, since distinct documents\n (according to their IDs) are independent. Also each indexing operation\n occurs under a lock for that document ID, so there is not much concurrency\n to consider. *)\n\n\n(* The set of individual requests that can occur on the document *)\nRequest(request_count)\n (* ADD: An optimised append-only write can only occur as the first operation\n on the document ID in seqno order. Any subsequent attempts to ADD the\n document have the retry flag set and modelled as a RETRY_ADD. Other operations\n on the document are also possible. *)\n == [type : {ADD}, seqno : {1}, content : DocContent, autoIdTimeStamp : {DocAutoIdTimestamp}]\n (* RETRY_ADD: A retry of a write that does involve an internally-generated\n document ID. *)\n \\cup [type : {RETRY_ADD}, seqno : 1..request_count, content : DocContent, autoIdTimeStamp : {DocAutoIdTimestamp}]\n (* UPDATE: A write that does not involve an internally-generated document ID. *)\n \\cup [type : {UPDATE}, seqno : 1..request_count, content : DocContent]\n (* DELETE *)\n \\cup [type : {DELETE}, seqno : 1..request_count]\n\n(* The set of sets of requests, which have distinct seqnos *)\nRequestSet(request_count)\n == { rs \\in SUBSET Request(request_count):\n /\\ Cardinality(rs) = request_count\n /\\ Cardinality({r.seqno : r \\in rs}) = request_count\n /\\ (* Also ADDs and RETRY_ADDs should have the same content *)\n Cardinality({r.content: r \\in { r \\in rs: r.type \\in {ADD, RETRY_ADD}}}) <= 1\n }\n\n(* Apply a set of operations to a document in seqno order *)\nRECURSIVE ApplyOps(_, _, _)\nApplyOps(requests, nextSeqno, currentContent)\n == IF \\A r \\in requests: r.seqno < nextSeqno\n THEN currentContent\n ELSE LET r == CHOOSE r \\in requests: r.seqno = nextSeqno\n IN IF r \\in requests /\\ r.seqno = nextSeqno\n THEN ApplyOps(requests, nextSeqno + 1,\n CASE r.type = DELETE -> NULL\n [] r.type = ADD -> r.content\n [] r.type = RETRY_ADD -> r.content\n [] r.type = UPDATE -> r.content)\n ELSE Assert(FALSE, \"Bad sequence\")\n\n(* Calculate the final doc by applying all the requests in order *)\nFinalDoc(requests) == ApplyOps(requests, 1, NULL)\n\n(* Apply each the operation in the Lucene buffer, rejecting an\n addDocuments when there is already a document present as this\n would lead to duplication. *)\nRECURSIVE ApplyBufferedOperations(_, _)\nApplyBufferedOperations(buffer, origDoc)\n == IF buffer = <<>>\n THEN origDoc\n ELSE LET nextOp == Head(buffer)\n IN ApplyBufferedOperations(Tail(buffer),\n CASE nextOp.type = Lucene_deleteDocuments -> NULL\n [] \\/ nextOp.type = Lucene_updateDocuments\n \\/ /\\ nextOp.type = Lucene_addDocuments\n /\\ origDoc = NULL -> [content |-> nextOp.content, seqno |-> nextOp.seqno]\n [] OTHER -> Assert(FALSE, \"Error: Lucene_addDocuments when origDoc /= NULL\"))\n\nMax(a,b) == IF a <= b THEN b ELSE a\n\n(* --algorithm basic\n\nvariables\n request_count \\in 1..4,\n replication_requests \\in RequestSet(request_count),\n expected_doc = FinalDoc(replication_requests),\n versionMap_needsSafeAccess = FALSE,\n versionMap_isUnsafe = FALSE,\n versionMap_entry = NULL,\n\n(* Other concurrent activity can flag that the version map needs to be safely accessed *)\nprocess SafeAccessEnablerProcess = \"SafeAccessEnabler\"\nbegin\n SafeAccessEnablerLoop:\n while pc[\"Consumer\"] /= \"Done\" do\n versionMap_needsSafeAccess := (versionMap_needsSafeAccess = FALSE);\n (* Technically the only way this can go back to FALSE is via a refresh, but\n we should not need this fact, so model both kinds of change. *)\n end while;\nend process;\n\n(* Other concurrent activity can make the version map become unsafe, if safe access mode is disabled *)\nprocess UnsafePutterProcess = \"UnsafePutter\"\nbegin\n UnsafePutterLoop:\n while pc[\"Consumer\"] /= \"Done\" do\n await versionMap_needsSafeAccess = FALSE;\n versionMap_isUnsafe := TRUE;\n end while;\nend process;\n\n(* Other concurrent activity can increase the maxUnsafeAutoIdTimestamp *)\nprocess MaxUnsafeAutoIdTimestampIncreaserProcess = \"MaxUnsafeAutoIdTimestampIncreaser\"\nbegin\n MaxUnsafeAutoIdTimestampIncreaserLoop:\n while pc[\"Consumer\"] /= \"Done\" do\n with newTimestamp \\in {DocAutoIdTimestamp - 1, DocAutoIdTimestamp, DocAutoIdTimestamp + 1} do\n await maxUnsafeAutoIdTimestamp < newTimestamp;\n maxUnsafeAutoIdTimestamp := newTimestamp;\n end with;\n end while;\nend process;\n\n(* Lucene refreshes can happen at any time *)\nprocess LuceneProcess = \"ReplicaLucene\"\nvariables\n lucene_document = NULL,\n lucene_buffer = <<>>,\nbegin\n LuceneLoop:\n while pc[\"Consumer\"] /= \"Done\" \\/ lucene_buffer /= <<>> do\n\n lucene_document := ApplyBufferedOperations(lucene_buffer, lucene_document);\n lucene_buffer := <<>>;\n \n (* TODO Model the inner structure of the version map so this refresh can be\n broken into the individual steps that occur concurrently with ongoing indexing. *)\n \n versionMap_isUnsafe := FALSE;\n versionMap_needsSafeAccess := FALSE;\n \n if versionMap_entry /= NULL\n then\n if versionMap_entry.type = UPDATE\n then\n versionMap_entry := NULL;\n else\n assert versionMap_entry.type = DELETE;\n versionMap_entry := [ versionMap_entry EXCEPT !.flushed = TRUE ];\n end if;\n end if; \n end while;\nend process;\n\n(* Flushed deletes expire after a time and are cleaned up *)\nprocess DeleteCollectorProcess = \"DeleteCollector\"\nbegin\n DeleteCollectorLoop:\n while pc[\"Consumer\"] /= \"Done\" do\n await /\\ versionMap_entry /= NULL\n /\\ versionMap_entry.type = DELETE\n /\\ versionMap_entry.seqno <= localCheckPoint \\* PR #28790\n /\\ versionMap_entry.flushed = TRUE;\n versionMap_entry := NULL;\n end while;\nend process;\n\n(* Local checkpoint advances as each operation is marked as completed *)\nprocess LocalCheckpointTrackerProcess = \"LocalCheckpointTracker\"\nvariables\n localCheckPoint = 0,\n completedSeqnos = {}\nbegin\n LocalCheckpointTrackerLoop:\n while pc[\"Consumer\"] /= \"Done\" do\n await localCheckPoint + 1 \\in completedSeqnos;\n localCheckPoint := localCheckPoint + 1;\n end while;\nend process\n\nprocess UnsafeSeqnoIncreaserProcess = \"UnsafeSeqnoIncreaserProcess\"\nvariables\n maxSeqNoOfNonAppendOnlyOperations = 0,\nbegin\n UnsafeSeqnoIncreaserProcessLoop:\n while pc[\"Consumer\"] /= \"Done\" /\\ maxSeqNoOfNonAppendOnlyOperations < request_count + 1 do\n maxSeqNoOfNonAppendOnlyOperations := maxSeqNoOfNonAppendOnlyOperations + 1;\n end while;\nend process\n\n\n(* The process that consumes replication requests for a particular document ID, which\n are processed in series because of the lock in the version map. *)\nprocess ConsumerProcess = \"Consumer\"\nvariables\n duplicationCount = 0,\n maxUnsafeAutoIdTimestamp \\in {0, DocAutoIdTimestamp - 1, DocAutoIdTimestamp, DocAutoIdTimestamp + 1},\n req, plan,\n deleteFromLucene, currentlyDeleted,\n currentNotFoundOrDeleted, useLuceneUpdateDocument, indexIntoLucene,\nbegin\n ConsumerLoop:\n while replication_requests /= {} do\n with replication_request \\in replication_requests do\n if replication_request.type = ADD\n then\n (* Never see two ADDs - if duplicated, one of them is a RETRY_ADD *)\n either\n (* Process ADD without duplication *)\n replication_requests := replication_requests \\ {replication_request};\n req := replication_request;\n or\n await duplicationCount < DuplicationLimit;\n duplicationCount := duplicationCount + 1;\n \n (* Process ADD and leave a duplicate RETRY_ADD for later *)\n replication_requests := (replication_requests \\ {replication_request})\n \\cup {[replication_request EXCEPT !.type = RETRY_ADD]};\n req := replication_request;\n or\n await duplicationCount < DuplicationLimit;\n duplicationCount := duplicationCount + 1;\n\n (* Process duplicate RETRY_ADD and leave the original ADD *)\n req := [replication_request EXCEPT !.type = RETRY_ADD];\n end either;\n else\n req := replication_request;\n either\n await duplicationCount < DuplicationLimit;\n duplicationCount := duplicationCount + 1;\n or\n replication_requests := replication_requests \\ {replication_request};\n end either;\n end if;\n end with;\n \n if req.type = DELETE\n then\n\n versionMap_needsSafeAccess := TRUE; \n \n (* planDeletionAsNonPrimary *)\n \n maxSeqNoOfNonAppendOnlyOperations := Max(maxSeqNoOfNonAppendOnlyOperations, req.seqno);\n\n if req.seqno <= localCheckPoint\n then\n (* OP_STALE_OR_EQUAL *)\n plan := \"processButSkipLucene\";\n deleteFromLucene := FALSE;\n currentlyDeleted := FALSE;\n else\n if versionMap_isUnsafe\n then\n (* Perform a Lucene refresh *)\n AwaitRefreshOnDelete: \\* Label here to allow for other concurrent activity\n await lucene_buffer = <<>>;\n versionMap_needsSafeAccess := TRUE;\n end if;\n \n compareDeleteOpToLuceneDocBasedOnSeqNo: \\* Label needed because of AwaitRefreshOnDelete label\n \n if versionMap_entry /= NULL\n then\n (* Doc is in version map *)\n if req.seqno > versionMap_entry.seqno\n then\n (* OP_NEWER *)\n plan := \"processNormally\";\n deleteFromLucene := TRUE;\n currentlyDeleted := FALSE;\n else\n (* OP_STALE_OR_EQUAL *)\n plan := \"processButSkipLucene\";\n deleteFromLucene := FALSE;\n currentlyDeleted := FALSE;\n end if;\n else\n (* Doc is not in version map - check Lucene *)\n if lucene_document = NULL\n then\n (* LUCENE_DOC_NOT_FOUND *)\n plan := \"processNormallyExceptNotFound\";\n deleteFromLucene := TRUE;\n currentlyDeleted := TRUE;\n else\n if req.seqno > lucene_document.seqno\n then\n (* OP_NEWER *)\n plan := \"processNormally\"; \n deleteFromLucene := TRUE;\n currentlyDeleted := FALSE;\n else\n (* OP_STALE_OR_EQUAL *)\n plan := \"processButSkipLucene\"; \n deleteFromLucene := FALSE;\n currentlyDeleted := FALSE;\n end if;\n end if;\n end if;\n end if;\n \n ExecuteDeletePlan: \\* Label needed because of AwaitRefreshOnDelete label\n if deleteFromLucene\n then\n if currentlyDeleted = FALSE\n then\n lucene_buffer := Append(lucene_buffer, [ type |-> Lucene_deleteDocuments ]);\n end if;\n\n versionMap_entry := [ type |-> DELETE, seqno |-> req.seqno, flushed |-> FALSE ];\n end if;\n \n completedSeqnos := completedSeqnos \\cup {req.seqno};\n else\n\n (* planIndexingAsNonPrimary *)\n \n (* A RETRY_ADD has canOptimiseAddDocument = TRUE and\n mayHaveBeenIndexedBefore = TRUE so is planned normally,\n but also updates maxUnsafeAutoIdTimestamp within\n mayHaveBeenIndexedBefore() *)\n\n if req.type = RETRY_ADD\n then\n maxUnsafeAutoIdTimestamp := Max(maxUnsafeAutoIdTimestamp, req.autoIdTimeStamp);\n end if;\n \n (* An ADD can be optimized if mayHaveBeenIndexedBefore = FALSE\n which is calculated by comparing timestamps. *)\n \n if /\\ req.type = ADD\n /\\ maxUnsafeAutoIdTimestamp < req.autoIdTimeStamp\n /\\ maxSeqNoOfNonAppendOnlyOperations < req.seqno \\* PR #28787\n then\n plan := \"optimisedAppendOnly\";\n currentNotFoundOrDeleted := TRUE;\n useLuceneUpdateDocument := FALSE;\n indexIntoLucene := TRUE;\n else\n if req.type \\notin {ADD, RETRY_ADD}\n then\n maxSeqNoOfNonAppendOnlyOperations := Max(maxSeqNoOfNonAppendOnlyOperations, req.seqno);\n end if;\n \n (* All other operations are planned normally *)\n versionMap_needsSafeAccess := TRUE;\n \n if req.seqno <= localCheckPoint\n then\n (* OP_STALE_OR_EQUAL *) \n plan := \"processButSkipLucene\";\n currentNotFoundOrDeleted := FALSE;\n useLuceneUpdateDocument := FALSE;\n indexIntoLucene := FALSE;\n else\n if versionMap_isUnsafe\n then\n (* Perform a Lucene refresh *)\n AwaitRefreshOnIndex: \\* Label here to allow for other concurrent activity\n await lucene_buffer = <<>>;\n versionMap_needsSafeAccess := TRUE; \n end if;\n \n compareIndexOpToLuceneDocBasedOnSeqNo: \\* Label needed because of AwaitRefreshOnIndex label\n \n if req.seqno <= localCheckPoint \\* PR #29276\n then \\* PR #29276\n (* OP_STALE_OR_EQUAL *) \\* PR #29276\n plan := \"processButSkipLucene\"; \\* PR #29276\n currentNotFoundOrDeleted := FALSE; \\* PR #29276\n useLuceneUpdateDocument := FALSE; \\* PR #29276\n indexIntoLucene := FALSE; \\* PR #29276\n elsif versionMap_entry /= NULL\n then\n (* Doc is in version map *)\n if req.seqno > versionMap_entry.seqno\n then\n (* OP_NEWER *)\n plan := \"processNormally\";\n currentNotFoundOrDeleted := FALSE;\n useLuceneUpdateDocument := TRUE;\n indexIntoLucene := TRUE;\n else\n (* OP_STALE_OR_EQUAL *)\n plan := \"processButSkipLucene\";\n currentNotFoundOrDeleted := FALSE;\n useLuceneUpdateDocument := FALSE;\n indexIntoLucene := FALSE;\n end if;\n else\n (* Doc is not in version map - check Lucene *)\n if lucene_document = NULL\n then\n (* LUCENE_DOC_NOT_FOUND *)\n plan := \"processNormallyExceptNotFound\";\n currentNotFoundOrDeleted := TRUE;\n useLuceneUpdateDocument := FALSE;\n indexIntoLucene := TRUE;\n else\n if req.seqno > lucene_document.seqno\n then\n (* OP_NEWER *)\n plan := \"processNormally\"; \n currentNotFoundOrDeleted := FALSE;\n useLuceneUpdateDocument := TRUE;\n indexIntoLucene := TRUE;\n else\n (* OP_STALE_OR_EQUAL *)\n plan := \"processButSkipLucene\"; \n currentNotFoundOrDeleted := FALSE;\n useLuceneUpdateDocument := FALSE;\n indexIntoLucene := FALSE;\n end if;\n end if;\n end if;\n end if;\n end if;\n \n (* planIndexingAsNonPrimary finished - now time to execute the plan *)\n ExecuteIndexPlan: \\* Label needed because of AwaitRefreshOnIndex label\n \n if indexIntoLucene\n then\n lucene_buffer := Append(lucene_buffer, \n [ type |-> IF useLuceneUpdateDocument THEN Lucene_updateDocuments ELSE Lucene_addDocuments\n , seqno |-> req.seqno\n , content |-> req.content\n ]);\n\n if versionMap_needsSafeAccess\n then\n versionMap_entry := [ type |-> UPDATE, seqno |-> req.seqno ];\n else\n versionMap_isUnsafe := TRUE;\n\n if /\\ versionMap_entry /= NULL\n /\\ versionMap_entry.type = DELETE\n /\\ versionMap_entry.seqno < req.seqno\n then\n versionMap_entry := NULL; \\* Desync bug #3 (no PR number yet)\n end if;\n end if;\n end if;\n \n completedSeqnos := completedSeqnos \\cup {req.seqno}\n end if;\n end while;\nend process\n\nend algorithm\n\n*)\n\\* BEGIN TRANSLATION\nCONSTANT defaultInitValue\nVARIABLES request_count, replication_requests, expected_doc, \n versionMap_needsSafeAccess, versionMap_isUnsafe, versionMap_entry, \n pc, lucene_document, lucene_buffer, localCheckPoint, \n completedSeqnos, maxSeqNoOfNonAppendOnlyOperations, \n duplicationCount, maxUnsafeAutoIdTimestamp, req, plan, \n deleteFromLucene, currentlyDeleted, currentNotFoundOrDeleted, \n useLuceneUpdateDocument, indexIntoLucene\n\nvars == << request_count, replication_requests, expected_doc, \n versionMap_needsSafeAccess, versionMap_isUnsafe, versionMap_entry, \n pc, lucene_document, lucene_buffer, localCheckPoint, \n completedSeqnos, maxSeqNoOfNonAppendOnlyOperations, \n duplicationCount, maxUnsafeAutoIdTimestamp, req, plan, \n deleteFromLucene, currentlyDeleted, currentNotFoundOrDeleted, \n useLuceneUpdateDocument, indexIntoLucene >>\n\nProcSet == {\"SafeAccessEnabler\"} \\cup {\"UnsafePutter\"} \\cup {\"MaxUnsafeAutoIdTimestampIncreaser\"} \\cup {\"ReplicaLucene\"} \\cup {\"DeleteCollector\"} \\cup {\"LocalCheckpointTracker\"} \\cup {\"UnsafeSeqnoIncreaserProcess\"} \\cup {\"Consumer\"}\n\nInit == (* Global variables *)\n /\\ request_count \\in 1..4\n /\\ replication_requests \\in RequestSet(request_count)\n /\\ expected_doc = FinalDoc(replication_requests)\n /\\ versionMap_needsSafeAccess = FALSE\n /\\ versionMap_isUnsafe = FALSE\n /\\ versionMap_entry = NULL\n (* Process LuceneProcess *)\n /\\ lucene_document = NULL\n /\\ lucene_buffer = <<>>\n (* Process LocalCheckpointTrackerProcess *)\n /\\ localCheckPoint = 0\n /\\ completedSeqnos = {}\n (* Process UnsafeSeqnoIncreaserProcess *)\n /\\ maxSeqNoOfNonAppendOnlyOperations = 0\n (* Process ConsumerProcess *)\n /\\ duplicationCount = 0\n /\\ maxUnsafeAutoIdTimestamp \\in {0, DocAutoIdTimestamp - 1, DocAutoIdTimestamp, DocAutoIdTimestamp + 1}\n /\\ req = defaultInitValue\n /\\ plan = defaultInitValue\n /\\ deleteFromLucene = defaultInitValue\n /\\ currentlyDeleted = defaultInitValue\n /\\ currentNotFoundOrDeleted = defaultInitValue\n /\\ useLuceneUpdateDocument = defaultInitValue\n /\\ indexIntoLucene = defaultInitValue\n /\\ pc = [self \\in ProcSet |-> CASE self = \"SafeAccessEnabler\" -> \"SafeAccessEnablerLoop\"\n [] self = \"UnsafePutter\" -> \"UnsafePutterLoop\"\n [] self = \"MaxUnsafeAutoIdTimestampIncreaser\" -> \"MaxUnsafeAutoIdTimestampIncreaserLoop\"\n [] self = \"ReplicaLucene\" -> \"LuceneLoop\"\n [] self = \"DeleteCollector\" -> \"DeleteCollectorLoop\"\n [] self = \"LocalCheckpointTracker\" -> \"LocalCheckpointTrackerLoop\"\n [] self = \"UnsafeSeqnoIncreaserProcess\" -> \"UnsafeSeqnoIncreaserProcessLoop\"\n [] self = \"Consumer\" -> \"ConsumerLoop\"]\n\nSafeAccessEnablerLoop == /\\ pc[\"SafeAccessEnabler\"] = \"SafeAccessEnablerLoop\"\n /\\ IF pc[\"Consumer\"] /= \"Done\"\n THEN /\\ versionMap_needsSafeAccess' = (versionMap_needsSafeAccess = FALSE)\n /\\ pc' = [pc EXCEPT ![\"SafeAccessEnabler\"] = \"SafeAccessEnablerLoop\"]\n ELSE /\\ pc' = [pc EXCEPT ![\"SafeAccessEnabler\"] = \"Done\"]\n /\\ UNCHANGED versionMap_needsSafeAccess\n /\\ UNCHANGED << request_count, replication_requests, \n expected_doc, versionMap_isUnsafe, \n versionMap_entry, lucene_document, \n lucene_buffer, localCheckPoint, \n completedSeqnos, \n maxSeqNoOfNonAppendOnlyOperations, \n duplicationCount, \n maxUnsafeAutoIdTimestamp, req, plan, \n deleteFromLucene, currentlyDeleted, \n currentNotFoundOrDeleted, \n useLuceneUpdateDocument, \n indexIntoLucene >>\n\nSafeAccessEnablerProcess == SafeAccessEnablerLoop\n\nUnsafePutterLoop == /\\ pc[\"UnsafePutter\"] = \"UnsafePutterLoop\"\n /\\ IF pc[\"Consumer\"] /= \"Done\"\n THEN /\\ versionMap_needsSafeAccess = FALSE\n /\\ versionMap_isUnsafe' = TRUE\n /\\ pc' = [pc EXCEPT ![\"UnsafePutter\"] = \"UnsafePutterLoop\"]\n ELSE /\\ pc' = [pc EXCEPT ![\"UnsafePutter\"] = \"Done\"]\n /\\ UNCHANGED versionMap_isUnsafe\n /\\ UNCHANGED << request_count, replication_requests, \n expected_doc, versionMap_needsSafeAccess, \n versionMap_entry, lucene_document, \n lucene_buffer, localCheckPoint, \n completedSeqnos, \n maxSeqNoOfNonAppendOnlyOperations, \n duplicationCount, maxUnsafeAutoIdTimestamp, \n req, plan, deleteFromLucene, \n currentlyDeleted, currentNotFoundOrDeleted, \n useLuceneUpdateDocument, indexIntoLucene >>\n\nUnsafePutterProcess == UnsafePutterLoop\n\nMaxUnsafeAutoIdTimestampIncreaserLoop == /\\ pc[\"MaxUnsafeAutoIdTimestampIncreaser\"] = \"MaxUnsafeAutoIdTimestampIncreaserLoop\"\n /\\ IF pc[\"Consumer\"] /= \"Done\"\n THEN /\\ \\E newTimestamp \\in {DocAutoIdTimestamp - 1, DocAutoIdTimestamp, DocAutoIdTimestamp + 1}:\n /\\ maxUnsafeAutoIdTimestamp < newTimestamp\n /\\ maxUnsafeAutoIdTimestamp' = newTimestamp\n /\\ pc' = [pc EXCEPT ![\"MaxUnsafeAutoIdTimestampIncreaser\"] = \"MaxUnsafeAutoIdTimestampIncreaserLoop\"]\n ELSE /\\ pc' = [pc EXCEPT ![\"MaxUnsafeAutoIdTimestampIncreaser\"] = \"Done\"]\n /\\ UNCHANGED maxUnsafeAutoIdTimestamp\n /\\ UNCHANGED << request_count, \n replication_requests, \n expected_doc, \n versionMap_needsSafeAccess, \n versionMap_isUnsafe, \n versionMap_entry, \n lucene_document, \n lucene_buffer, \n localCheckPoint, \n completedSeqnos, \n maxSeqNoOfNonAppendOnlyOperations, \n duplicationCount, req, \n plan, \n deleteFromLucene, \n currentlyDeleted, \n currentNotFoundOrDeleted, \n useLuceneUpdateDocument, \n indexIntoLucene >>\n\nMaxUnsafeAutoIdTimestampIncreaserProcess == MaxUnsafeAutoIdTimestampIncreaserLoop\n\nLuceneLoop == /\\ pc[\"ReplicaLucene\"] = \"LuceneLoop\"\n /\\ IF pc[\"Consumer\"] /= \"Done\" \\/ lucene_buffer /= <<>>\n THEN /\\ lucene_document' = ApplyBufferedOperations(lucene_buffer, lucene_document)\n /\\ lucene_buffer' = <<>>\n /\\ versionMap_isUnsafe' = FALSE\n /\\ versionMap_needsSafeAccess' = FALSE\n /\\ IF versionMap_entry /= NULL\n THEN /\\ IF versionMap_entry.type = UPDATE\n THEN /\\ versionMap_entry' = NULL\n ELSE /\\ Assert(versionMap_entry.type = DELETE, \n \"Failure of assertion at line 147, column 17.\")\n /\\ versionMap_entry' = [ versionMap_entry EXCEPT !.flushed = TRUE ]\n ELSE /\\ TRUE\n /\\ UNCHANGED versionMap_entry\n /\\ pc' = [pc EXCEPT ![\"ReplicaLucene\"] = \"LuceneLoop\"]\n ELSE /\\ pc' = [pc EXCEPT ![\"ReplicaLucene\"] = \"Done\"]\n /\\ UNCHANGED << versionMap_needsSafeAccess, \n versionMap_isUnsafe, versionMap_entry, \n lucene_document, lucene_buffer >>\n /\\ UNCHANGED << request_count, replication_requests, \n expected_doc, localCheckPoint, completedSeqnos, \n maxSeqNoOfNonAppendOnlyOperations, \n duplicationCount, maxUnsafeAutoIdTimestamp, req, \n plan, deleteFromLucene, currentlyDeleted, \n currentNotFoundOrDeleted, \n useLuceneUpdateDocument, indexIntoLucene >>\n\nLuceneProcess == LuceneLoop\n\nDeleteCollectorLoop == /\\ pc[\"DeleteCollector\"] = \"DeleteCollectorLoop\"\n /\\ IF pc[\"Consumer\"] /= \"Done\"\n THEN /\\ /\\ versionMap_entry /= NULL\n /\\ versionMap_entry.type = DELETE\n /\\ versionMap_entry.seqno <= localCheckPoint\n /\\ versionMap_entry.flushed = TRUE\n /\\ versionMap_entry' = NULL\n /\\ pc' = [pc EXCEPT ![\"DeleteCollector\"] = \"DeleteCollectorLoop\"]\n ELSE /\\ pc' = [pc EXCEPT ![\"DeleteCollector\"] = \"Done\"]\n /\\ UNCHANGED versionMap_entry\n /\\ UNCHANGED << request_count, replication_requests, \n expected_doc, \n versionMap_needsSafeAccess, \n versionMap_isUnsafe, lucene_document, \n lucene_buffer, localCheckPoint, \n completedSeqnos, \n maxSeqNoOfNonAppendOnlyOperations, \n duplicationCount, \n maxUnsafeAutoIdTimestamp, req, plan, \n deleteFromLucene, currentlyDeleted, \n currentNotFoundOrDeleted, \n useLuceneUpdateDocument, \n indexIntoLucene >>\n\nDeleteCollectorProcess == DeleteCollectorLoop\n\nLocalCheckpointTrackerLoop == /\\ pc[\"LocalCheckpointTracker\"] = \"LocalCheckpointTrackerLoop\"\n /\\ IF pc[\"Consumer\"] /= \"Done\"\n THEN /\\ localCheckPoint + 1 \\in completedSeqnos\n /\\ localCheckPoint' = localCheckPoint + 1\n /\\ pc' = [pc EXCEPT ![\"LocalCheckpointTracker\"] = \"LocalCheckpointTrackerLoop\"]\n ELSE /\\ pc' = [pc EXCEPT ![\"LocalCheckpointTracker\"] = \"Done\"]\n /\\ UNCHANGED localCheckPoint\n /\\ UNCHANGED << request_count, \n replication_requests, \n expected_doc, \n versionMap_needsSafeAccess, \n versionMap_isUnsafe, \n versionMap_entry, \n lucene_document, lucene_buffer, \n completedSeqnos, \n maxSeqNoOfNonAppendOnlyOperations, \n duplicationCount, \n maxUnsafeAutoIdTimestamp, req, \n plan, deleteFromLucene, \n currentlyDeleted, \n currentNotFoundOrDeleted, \n useLuceneUpdateDocument, \n indexIntoLucene >>\n\nLocalCheckpointTrackerProcess == LocalCheckpointTrackerLoop\n\nUnsafeSeqnoIncreaserProcessLoop == /\\ pc[\"UnsafeSeqnoIncreaserProcess\"] = \"UnsafeSeqnoIncreaserProcessLoop\"\n /\\ IF pc[\"Consumer\"] /= \"Done\" /\\ maxSeqNoOfNonAppendOnlyOperations < request_count + 1\n THEN /\\ maxSeqNoOfNonAppendOnlyOperations' = maxSeqNoOfNonAppendOnlyOperations + 1\n /\\ pc' = [pc EXCEPT ![\"UnsafeSeqnoIncreaserProcess\"] = \"UnsafeSeqnoIncreaserProcessLoop\"]\n ELSE /\\ pc' = [pc EXCEPT ![\"UnsafeSeqnoIncreaserProcess\"] = \"Done\"]\n /\\ UNCHANGED maxSeqNoOfNonAppendOnlyOperations\n /\\ UNCHANGED << request_count, \n replication_requests, \n expected_doc, \n versionMap_needsSafeAccess, \n versionMap_isUnsafe, \n versionMap_entry, \n lucene_document, \n lucene_buffer, \n localCheckPoint, \n completedSeqnos, \n duplicationCount, \n maxUnsafeAutoIdTimestamp, \n req, plan, deleteFromLucene, \n currentlyDeleted, \n currentNotFoundOrDeleted, \n useLuceneUpdateDocument, \n indexIntoLucene >>\n\nUnsafeSeqnoIncreaserProcess == UnsafeSeqnoIncreaserProcessLoop\n\nConsumerLoop == /\\ pc[\"Consumer\"] = \"ConsumerLoop\"\n /\\ IF replication_requests /= {}\n THEN /\\ \\E replication_request \\in replication_requests:\n IF replication_request.type = ADD\n THEN /\\ \\/ /\\ replication_requests' = replication_requests \\ {replication_request}\n /\\ req' = replication_request\n /\\ UNCHANGED duplicationCount\n \\/ /\\ duplicationCount < DuplicationLimit\n /\\ duplicationCount' = duplicationCount + 1\n /\\ replication_requests' = (replication_requests \\ {replication_request})\n \\cup {[replication_request EXCEPT !.type = RETRY_ADD]}\n /\\ req' = replication_request\n \\/ /\\ duplicationCount < DuplicationLimit\n /\\ duplicationCount' = duplicationCount + 1\n /\\ req' = [replication_request EXCEPT !.type = RETRY_ADD]\n /\\ UNCHANGED replication_requests\n ELSE /\\ req' = replication_request\n /\\ \\/ /\\ duplicationCount < DuplicationLimit\n /\\ duplicationCount' = duplicationCount + 1\n /\\ UNCHANGED replication_requests\n \\/ /\\ replication_requests' = replication_requests \\ {replication_request}\n /\\ UNCHANGED duplicationCount\n /\\ IF req'.type = DELETE\n THEN /\\ versionMap_needsSafeAccess' = TRUE\n /\\ maxSeqNoOfNonAppendOnlyOperations' = Max(maxSeqNoOfNonAppendOnlyOperations, req'.seqno)\n /\\ IF req'.seqno <= localCheckPoint\n THEN /\\ plan' = \"processButSkipLucene\"\n /\\ deleteFromLucene' = FALSE\n /\\ currentlyDeleted' = FALSE\n /\\ pc' = [pc EXCEPT ![\"Consumer\"] = \"ExecuteDeletePlan\"]\n ELSE /\\ IF versionMap_isUnsafe\n THEN /\\ pc' = [pc EXCEPT ![\"Consumer\"] = \"AwaitRefreshOnDelete\"]\n ELSE /\\ pc' = [pc EXCEPT ![\"Consumer\"] = \"compareDeleteOpToLuceneDocBasedOnSeqNo\"]\n /\\ UNCHANGED << plan, \n deleteFromLucene, \n currentlyDeleted >>\n /\\ UNCHANGED << maxUnsafeAutoIdTimestamp, \n currentNotFoundOrDeleted, \n useLuceneUpdateDocument, \n indexIntoLucene >>\n ELSE /\\ IF req'.type = RETRY_ADD\n THEN /\\ maxUnsafeAutoIdTimestamp' = Max(maxUnsafeAutoIdTimestamp, req'.autoIdTimeStamp)\n ELSE /\\ TRUE\n /\\ UNCHANGED maxUnsafeAutoIdTimestamp\n /\\ IF /\\ req'.type = ADD\n /\\ maxUnsafeAutoIdTimestamp' < req'.autoIdTimeStamp\n /\\ maxSeqNoOfNonAppendOnlyOperations < req'.seqno\n THEN /\\ plan' = \"optimisedAppendOnly\"\n /\\ currentNotFoundOrDeleted' = TRUE\n /\\ useLuceneUpdateDocument' = FALSE\n /\\ indexIntoLucene' = TRUE\n /\\ pc' = [pc EXCEPT ![\"Consumer\"] = \"ExecuteIndexPlan\"]\n /\\ UNCHANGED << versionMap_needsSafeAccess, \n maxSeqNoOfNonAppendOnlyOperations >>\n ELSE /\\ IF req'.type \\notin {ADD, RETRY_ADD}\n THEN /\\ maxSeqNoOfNonAppendOnlyOperations' = Max(maxSeqNoOfNonAppendOnlyOperations, req'.seqno)\n ELSE /\\ TRUE\n /\\ UNCHANGED maxSeqNoOfNonAppendOnlyOperations\n /\\ versionMap_needsSafeAccess' = TRUE\n /\\ IF req'.seqno <= localCheckPoint\n THEN /\\ plan' = \"processButSkipLucene\"\n /\\ currentNotFoundOrDeleted' = FALSE\n /\\ useLuceneUpdateDocument' = FALSE\n /\\ indexIntoLucene' = FALSE\n /\\ pc' = [pc EXCEPT ![\"Consumer\"] = \"ExecuteIndexPlan\"]\n ELSE /\\ IF versionMap_isUnsafe\n THEN /\\ pc' = [pc EXCEPT ![\"Consumer\"] = \"AwaitRefreshOnIndex\"]\n ELSE /\\ pc' = [pc EXCEPT ![\"Consumer\"] = \"compareIndexOpToLuceneDocBasedOnSeqNo\"]\n /\\ UNCHANGED << plan, \n currentNotFoundOrDeleted, \n useLuceneUpdateDocument, \n indexIntoLucene >>\n /\\ UNCHANGED << deleteFromLucene, \n currentlyDeleted >>\n ELSE /\\ pc' = [pc EXCEPT ![\"Consumer\"] = \"Done\"]\n /\\ UNCHANGED << replication_requests, \n versionMap_needsSafeAccess, \n maxSeqNoOfNonAppendOnlyOperations, \n duplicationCount, \n maxUnsafeAutoIdTimestamp, req, plan, \n deleteFromLucene, currentlyDeleted, \n currentNotFoundOrDeleted, \n useLuceneUpdateDocument, \n indexIntoLucene >>\n /\\ UNCHANGED << request_count, expected_doc, \n versionMap_isUnsafe, versionMap_entry, \n lucene_document, lucene_buffer, \n localCheckPoint, completedSeqnos >>\n\nExecuteDeletePlan == /\\ pc[\"Consumer\"] = \"ExecuteDeletePlan\"\n /\\ IF deleteFromLucene\n THEN /\\ IF currentlyDeleted = FALSE\n THEN /\\ lucene_buffer' = Append(lucene_buffer, [ type |-> Lucene_deleteDocuments ])\n ELSE /\\ TRUE\n /\\ UNCHANGED lucene_buffer\n /\\ versionMap_entry' = [ type |-> DELETE, seqno |-> req.seqno, flushed |-> FALSE ]\n ELSE /\\ TRUE\n /\\ UNCHANGED << versionMap_entry, \n lucene_buffer >>\n /\\ completedSeqnos' = (completedSeqnos \\cup {req.seqno})\n /\\ pc' = [pc EXCEPT ![\"Consumer\"] = \"ConsumerLoop\"]\n /\\ UNCHANGED << request_count, replication_requests, \n expected_doc, versionMap_needsSafeAccess, \n versionMap_isUnsafe, lucene_document, \n localCheckPoint, \n maxSeqNoOfNonAppendOnlyOperations, \n duplicationCount, \n maxUnsafeAutoIdTimestamp, req, plan, \n deleteFromLucene, currentlyDeleted, \n currentNotFoundOrDeleted, \n useLuceneUpdateDocument, indexIntoLucene >>\n\nExecuteIndexPlan == /\\ pc[\"Consumer\"] = \"ExecuteIndexPlan\"\n /\\ IF indexIntoLucene\n THEN /\\ lucene_buffer' = Append(lucene_buffer,\n [ type |-> IF useLuceneUpdateDocument THEN Lucene_updateDocuments ELSE Lucene_addDocuments\n , seqno |-> req.seqno\n , content |-> req.content\n ])\n /\\ IF versionMap_needsSafeAccess\n THEN /\\ versionMap_entry' = [ type |-> UPDATE, seqno |-> req.seqno ]\n /\\ UNCHANGED versionMap_isUnsafe\n ELSE /\\ versionMap_isUnsafe' = TRUE\n /\\ IF /\\ versionMap_entry /= NULL\n /\\ versionMap_entry.type = DELETE\n /\\ versionMap_entry.seqno < req.seqno\n THEN /\\ versionMap_entry' = NULL\n ELSE /\\ TRUE\n /\\ UNCHANGED versionMap_entry\n ELSE /\\ TRUE\n /\\ UNCHANGED << versionMap_isUnsafe, \n versionMap_entry, lucene_buffer >>\n /\\ completedSeqnos' = (completedSeqnos \\cup {req.seqno})\n /\\ pc' = [pc EXCEPT ![\"Consumer\"] = \"ConsumerLoop\"]\n /\\ UNCHANGED << request_count, replication_requests, \n expected_doc, versionMap_needsSafeAccess, \n lucene_document, localCheckPoint, \n maxSeqNoOfNonAppendOnlyOperations, \n duplicationCount, maxUnsafeAutoIdTimestamp, \n req, plan, deleteFromLucene, \n currentlyDeleted, currentNotFoundOrDeleted, \n useLuceneUpdateDocument, indexIntoLucene >>\n\ncompareDeleteOpToLuceneDocBasedOnSeqNo == /\\ pc[\"Consumer\"] = \"compareDeleteOpToLuceneDocBasedOnSeqNo\"\n /\\ IF versionMap_entry /= NULL\n THEN /\\ IF req.seqno > versionMap_entry.seqno\n THEN /\\ plan' = \"processNormally\"\n /\\ deleteFromLucene' = TRUE\n /\\ currentlyDeleted' = FALSE\n ELSE /\\ plan' = \"processButSkipLucene\"\n /\\ deleteFromLucene' = FALSE\n /\\ currentlyDeleted' = FALSE\n ELSE /\\ IF lucene_document = NULL\n THEN /\\ plan' = \"processNormallyExceptNotFound\"\n /\\ deleteFromLucene' = TRUE\n /\\ currentlyDeleted' = TRUE\n ELSE /\\ IF req.seqno > lucene_document.seqno\n THEN /\\ plan' = \"processNormally\"\n /\\ deleteFromLucene' = TRUE\n /\\ currentlyDeleted' = FALSE\n ELSE /\\ plan' = \"processButSkipLucene\"\n /\\ deleteFromLucene' = FALSE\n /\\ currentlyDeleted' = FALSE\n /\\ pc' = [pc EXCEPT ![\"Consumer\"] = \"ExecuteDeletePlan\"]\n /\\ UNCHANGED << request_count, \n replication_requests, \n expected_doc, \n versionMap_needsSafeAccess, \n versionMap_isUnsafe, \n versionMap_entry, \n lucene_document, \n lucene_buffer, \n localCheckPoint, \n completedSeqnos, \n maxSeqNoOfNonAppendOnlyOperations, \n duplicationCount, \n maxUnsafeAutoIdTimestamp, \n req, \n currentNotFoundOrDeleted, \n useLuceneUpdateDocument, \n indexIntoLucene >>\n\nAwaitRefreshOnDelete == /\\ pc[\"Consumer\"] = \"AwaitRefreshOnDelete\"\n /\\ lucene_buffer = <<>>\n /\\ versionMap_needsSafeAccess' = TRUE\n /\\ pc' = [pc EXCEPT ![\"Consumer\"] = \"compareDeleteOpToLuceneDocBasedOnSeqNo\"]\n /\\ UNCHANGED << request_count, replication_requests, \n expected_doc, versionMap_isUnsafe, \n versionMap_entry, lucene_document, \n lucene_buffer, localCheckPoint, \n completedSeqnos, \n maxSeqNoOfNonAppendOnlyOperations, \n duplicationCount, \n maxUnsafeAutoIdTimestamp, req, plan, \n deleteFromLucene, currentlyDeleted, \n currentNotFoundOrDeleted, \n useLuceneUpdateDocument, \n indexIntoLucene >>\n\ncompareIndexOpToLuceneDocBasedOnSeqNo == /\\ pc[\"Consumer\"] = \"compareIndexOpToLuceneDocBasedOnSeqNo\"\n /\\ IF req.seqno <= localCheckPoint\n THEN /\\ plan' = \"processButSkipLucene\"\n /\\ currentNotFoundOrDeleted' = FALSE\n /\\ useLuceneUpdateDocument' = FALSE\n /\\ indexIntoLucene' = FALSE\n ELSE /\\ IF versionMap_entry /= NULL\n THEN /\\ IF req.seqno > versionMap_entry.seqno\n THEN /\\ plan' = \"processNormally\"\n /\\ currentNotFoundOrDeleted' = FALSE\n /\\ useLuceneUpdateDocument' = TRUE\n /\\ indexIntoLucene' = TRUE\n ELSE /\\ plan' = \"processButSkipLucene\"\n /\\ currentNotFoundOrDeleted' = FALSE\n /\\ useLuceneUpdateDocument' = FALSE\n /\\ indexIntoLucene' = FALSE\n ELSE /\\ IF lucene_document = NULL\n THEN /\\ plan' = \"processNormallyExceptNotFound\"\n /\\ currentNotFoundOrDeleted' = TRUE\n /\\ useLuceneUpdateDocument' = FALSE\n /\\ indexIntoLucene' = TRUE\n ELSE /\\ IF req.seqno > lucene_document.seqno\n THEN /\\ plan' = \"processNormally\"\n /\\ currentNotFoundOrDeleted' = FALSE\n /\\ useLuceneUpdateDocument' = TRUE\n /\\ indexIntoLucene' = TRUE\n ELSE /\\ plan' = \"processButSkipLucene\"\n /\\ currentNotFoundOrDeleted' = FALSE\n /\\ useLuceneUpdateDocument' = FALSE\n /\\ indexIntoLucene' = FALSE\n /\\ pc' = [pc EXCEPT ![\"Consumer\"] = \"ExecuteIndexPlan\"]\n /\\ UNCHANGED << request_count, \n replication_requests, \n expected_doc, \n versionMap_needsSafeAccess, \n versionMap_isUnsafe, \n versionMap_entry, \n lucene_document, \n lucene_buffer, \n localCheckPoint, \n completedSeqnos, \n maxSeqNoOfNonAppendOnlyOperations, \n duplicationCount, \n maxUnsafeAutoIdTimestamp, \n req, deleteFromLucene, \n currentlyDeleted >>\n\nAwaitRefreshOnIndex == /\\ pc[\"Consumer\"] = \"AwaitRefreshOnIndex\"\n /\\ lucene_buffer = <<>>\n /\\ versionMap_needsSafeAccess' = TRUE\n /\\ pc' = [pc EXCEPT ![\"Consumer\"] = \"compareIndexOpToLuceneDocBasedOnSeqNo\"]\n /\\ UNCHANGED << request_count, replication_requests, \n expected_doc, versionMap_isUnsafe, \n versionMap_entry, lucene_document, \n lucene_buffer, localCheckPoint, \n completedSeqnos, \n maxSeqNoOfNonAppendOnlyOperations, \n duplicationCount, \n maxUnsafeAutoIdTimestamp, req, plan, \n deleteFromLucene, currentlyDeleted, \n currentNotFoundOrDeleted, \n useLuceneUpdateDocument, \n indexIntoLucene >>\n\nConsumerProcess == ConsumerLoop \\/ ExecuteDeletePlan \\/ ExecuteIndexPlan\n \\/ compareDeleteOpToLuceneDocBasedOnSeqNo\n \\/ AwaitRefreshOnDelete\n \\/ compareIndexOpToLuceneDocBasedOnSeqNo\n \\/ AwaitRefreshOnIndex\n\nNext == SafeAccessEnablerProcess \\/ UnsafePutterProcess\n \\/ MaxUnsafeAutoIdTimestampIncreaserProcess \\/ LuceneProcess\n \\/ DeleteCollectorProcess \\/ LocalCheckpointTrackerProcess\n \\/ UnsafeSeqnoIncreaserProcess \\/ ConsumerProcess\n \\/ (* Disjunct to prevent deadlock on termination *)\n ((\\A self \\in ProcSet: pc[self] = \"Done\") /\\ UNCHANGED vars)\n\nSpec == Init /\\ [][Next]_vars\n\nTermination == <>(\\A self \\in ProcSet: pc[self] = \"Done\")\n\n\\* END TRANSLATION\n\nTerminated == \\A self \\in ProcSet: pc[self] = \"Done\"\n\nInvariant == Terminated => /\\ expected_doc = NULL => lucene_document = NULL\n /\\ expected_doc /= NULL => lucene_document.content = expected_doc\n\n=============================================================================\n"
},
{
"path": "ReplicaEngine/tla/ReplicaEngine.toolbox/.project",
"content": "\n\n\tReplicaEngine\n\t\n\t\n\t\n\t\n\t\t\n\t\t\ttoolbox.builder.TLAParserBuilder\n\t\t\t\n\t\t\t\n\t\t\n\t\t\n\t\t\ttoolbox.builder.PCalAlgorithmSearchingBuilder\n\t\t\t\n\t\t\t\n\t\t\n\t\n\t\n\t\ttoolbox.natures.TLANature\n\t\n\t\n\t\t\n\t\t\tReplicaEngine.tla\n\t\t\t1\n\t\t\tPARENT-1-PROJECT_LOC/ReplicaEngine.tla\n\t\t\n\t\n\n"
},
{
"path": "ReplicaEngine/tla/ReplicaEngine.toolbox/.settings/org.lamport.tla.toolbox.prefs",
"content": "ProjectRootFile=PARENT-1-PROJECT_LOC/ReplicaEngine.tla\neclipse.preferences.version=1\n"
},
{
"path": "ReplicaEngine/tla/ReplicaEngine.toolbox/ReplicaEngine___model.launch",
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n"
},
{
"path": "Storage/tla/Storage.tla",
"content": "------------------------------ MODULE Storage ------------------------------\nEXTENDS Integers, FiniteSets, TLC\n\nCONSTANTS \n MaxNewMeta, \\* maximum generation of newMeta to limit the state space\n MetaDataContent \\* content that is written to the metadata file\n \nVARIABLES\n metadata, \\* metaData[i] = MetaDataContent if metadata of generation i is present\n manifest, \\* manifest[j] is generation of metadata j-th manifest is referencing\n newMeta, \\* generation of newly created metadata file\n newManifest, \\* generation of newly created manifest file\n state, \\* current state, describes what to do next\n possibleStates \\* set of generations of metadata that limits what can be read from disk\n\n--------------------------------------------------------------------------\n(*************************************************************************)\n(* First we define some helper functions to work with files abstraction. *)\n(* Files is a function from file generation to some content. *)\n(*************************************************************************)\n\n(*************************************************************************)\n(* CurrentGeneration returns the maximum file generation. If there are *)\n(* no files then -1 is returned. *) \n(*************************************************************************)\nCurrentGeneration(files) == \n IF DOMAIN files = {} \n THEN -1 \n ELSE \n CHOOSE gen \\in DOMAIN files : \n \\A otherGen \\in DOMAIN files : gen \\geq otherGen\n\n(*************************************************************************)\n(* DeleteFile removes file with generation delGen. *)\n(*************************************************************************)\nDeleteFile(files, delGen) == [gen \\in DOMAIN files \\ {delGen} |-> files[gen]]\n\n(*************************************************************************)\n(* DeleteFilesExcept removes all files except keepGen. *)\n(*************************************************************************)\nDeleteFilesExcept(files, keepGen) == (keepGen :> files[keepGen])\n\n(*************************************************************************)\n(* WriteFile creates new file with specified generation and content. *)\n(*************************************************************************)\nWriteFile(files, gen, content) == (gen :> content) @@ files\n \n--------------------------------------------------------------------------\n(*************************************************************************)\n(* Now we define functions to emulate write and cleanup of the metadata. *)\n(*************************************************************************)\nWriteMetaOk(gen) == \n /\\ metadata' = WriteFile(metadata, gen, MetaDataContent)\n /\\ state' = \"writeManifest\"\n\nWriteMetaFail(gen) == \n /\\ metadata' = metadata\n /\\ state' = \"writeMeta\"\n \nWriteMetaDirty(gen) == \n /\\ \\/ metadata' = WriteFile(metadata, gen, MetaDataContent)\n \\/ metadata' = metadata\n /\\ state' = \"deleteNewMeta\"\n\nDeleteNewMeta == \n /\\ \\/ metadata' = DeleteFile(metadata, newMeta) \n \\/ metadata' = metadata\n /\\ state' = \"writeMeta\"\n /\\ UNCHANGED <> \n\nDeleteOldMeta ==\n /\\ \\/ metadata' = DeleteFilesExcept(metadata, newMeta) \n \\/ metadata' = metadata\n /\\ state' = \"writeMeta\"\n /\\ UNCHANGED <>\n\nWriteMeta == \n LET gen == CurrentGeneration(metadata) + 1 IN \n /\\ newMeta' = gen\n /\\ \\/ WriteMetaOk(gen) \n \\/ WriteMetaFail(gen) \n \\/ WriteMetaDirty(gen)\n /\\ UNCHANGED <>\n\n--------------------------------------------------------------------------\n(*************************************************************************)\n(* Now we define functions to emulate write and cleanup of the manifest *)\n(* file. *)\n(*************************************************************************) \nWriteManifestOk(gen) == \n /\\ manifest' = WriteFile(manifest, gen, newMeta)\n /\\ state' = \"deleteOldManifest\"\n /\\ possibleStates' = {newMeta}\n\nWriteManifestFail(gen) == \n /\\ manifest' = manifest\n /\\ state' = \"deleteNewMeta\"\n /\\ possibleStates' = possibleStates\n \nWriteManifestDirty(gen) == \n /\\ \\/ manifest' = WriteFile(manifest, gen, newMeta)\n \\/ manifest' = manifest\n /\\ state' = \"deleteNewManifest\"\n /\\ possibleStates' = possibleStates \\union {newMeta}\n \nWriteManifest == \n LET gen == CurrentGeneration(manifest) + 1 IN\n /\\ newManifest' = gen\n /\\ \\/ WriteManifestOk(gen)\n \\/ WriteManifestFail(gen)\n \\/ WriteManifestDirty(gen)\n /\\ UNCHANGED <>\n\nDeleteOldManifest ==\n /\\ \\/ manifest' = DeleteFilesExcept(manifest, newManifest)\n \\/ manifest' = manifest\n /\\ state' = \"deleteOldMeta\"\n /\\ UNCHANGED <>\n\n--------------------------------------------------------------------------\n(*************************************************************************)\n(* Below are 3 versions of the same function, that is called when *)\n(* manifest write was dirty. The buggy one was initially implemented and *)\n(* was caught by https://github.com/elastic/elasticsearch/issues/39077. *)\n(* Pick one and use in Next function. *)\n(* https://github.com/elastic/elasticsearch/pull/40519 implements *)\n(* DeleteNewManifestEasy. *)\n(*************************************************************************) \nDeleteNewManifestBuggy == \n /\\ \\/ manifest' = DeleteFile(manifest, newManifest)\n \\/ manifest' = manifest\n /\\ state' = \"deleteNewMeta\"\n /\\ UNCHANGED <>\n\nDeleteNewManifestEasy == \n /\\ \\/ manifest' = DeleteFile(manifest, newManifest)\n \\/ manifest' = manifest\n /\\ state' = \"writeMeta\"\n /\\ UNCHANGED <>\n \nDeleteNewManifestHard == \n /\\ \\/ /\\ manifest' = DeleteFile(manifest, newManifest)\n /\\ state' = \"deleteNewMeta\"\n \\/ /\\ manifest' = manifest\n /\\ state' = \"writeMeta\"\n /\\ UNCHANGED <>\n--------------------------------------------------------------------------\n(*************************************************************************)\n(* We can define Init and Next functions now. *)\n(*************************************************************************) \nInit == \n /\\ metadata = <<>>\n /\\ manifest = <<>>\n /\\ newMeta = -1 \\* no latest metadata file\n /\\ newManifest = -1 \\* no latest manifest file\n /\\ state = \"writeMeta\" \\* we start with writing metadata file\n /\\ possibleStates = {} \\* no metadata can be read from disk\n \nNext == \n \\/ (state = \"writeMeta\" /\\ WriteMeta)\n \\/ (state = \"writeManifest\" /\\ WriteManifest)\n \\/ (state = \"deleteOldManifest\" /\\ DeleteOldManifest)\n \\/ (state = \"deleteOldMeta\" /\\ DeleteOldMeta)\n \\/ (state = \"deleteNewManifest\" /\\ DeleteNewManifestEasy) \\* try DeleteNewManifestBuggy and DeleteNewManifestHard\n \\/ (state = \"deleteNewMeta\" /\\ DeleteNewMeta)\n--------------------------------------------------------------------------\n(*************************************************************************)\n(* Our model has 2 invariants. *)\n(*************************************************************************)\nMetadataFileReferencedByManifestExists ==\n CurrentGeneration(manifest) /= -1 \n => \n manifest[CurrentGeneration(manifest)] \\in DOMAIN metadata\n \nMetadataReferencedByManifestIsValid ==\n CurrentGeneration(manifest) /= -1 \n =>\n \\E meta \\in possibleStates : meta = manifest[CurrentGeneration(manifest)]\n============"
},
{
"path": "Storage/tla/Storage.toolbox/Storage___model.launch",
"content": "\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n"
},
{
"path": "ZenWithTerms/tla/ZenWithTerms.tla",
"content": "-------------------------------------------------------------------------------------\n\n-------------------------------- MODULE ZenWithTerms --------------------------------\n\\* Imported modules used in this specification\nEXTENDS Naturals, FiniteSets, Sequences, TLC\n\n----\n\nCONSTANTS Values\n\n\\* Set of node ids (all master-eligible nodes)\nCONSTANTS Nodes\n\n\\* RPC message types\nCONSTANTS\n Join,\n PublishRequest,\n PublishResponse,\n Commit\n\n----\n\n\\* Set of requests and responses sent between nodes.\nVARIABLE messages\n\n\\* Transitive closure of value updates as done by leaders \nVARIABLE descendant\n\n\\* Values to bootstrap the cluster\nVARIABLE initialConfiguration\nVARIABLE initialValue\nVARIABLE initialAcceptedVersion\n\n\\* node state (map from node id to state)\nVARIABLE currentTerm\nVARIABLE lastCommittedConfiguration\nVARIABLE lastAcceptedTerm\nVARIABLE lastAcceptedVersion\nVARIABLE lastAcceptedValue\nVARIABLE lastAcceptedConfiguration\nVARIABLE joinVotes\nVARIABLE startedJoinSinceLastReboot\nVARIABLE electionWon\nVARIABLE lastPublishedVersion\nVARIABLE lastPublishedConfiguration\nVARIABLE publishVotes\n\n----\n\nTerms == Nat\n\nVersions == Nat\n\n\\* set of valid configurations (i.e. the set of all non-empty subsets of Nodes)\nValidConfigs == SUBSET(Nodes) \\ {{}}\n\n\\* cluster-state versions that might have come from older systems\nInitialVersions == Nat\n\n\\* quorums correspond to majority of votes in a config\nIsQuorum(votes, config) == Cardinality(votes \\cap config) * 2 > Cardinality(config)\n\nIsElectionQuorum(n, votes) ==\n /\\ IsQuorum(votes, lastCommittedConfiguration[n])\n /\\ IsQuorum(votes, lastAcceptedConfiguration[n])\n\nIsPublishQuorum(n, votes) ==\n /\\ IsQuorum(votes, lastCommittedConfiguration[n])\n /\\ IsQuorum(votes, lastPublishedConfiguration[n])\n\n\\* initial model state\nInit == /\\ messages = {}\n /\\ descendant = {}\n /\\ initialConfiguration \\in ValidConfigs\n /\\ initialValue \\in Values\n /\\ initialAcceptedVersion \\in [Nodes -> InitialVersions]\n /\\ currentTerm = [n \\in Nodes |-> 0]\n /\\ lastCommittedConfiguration = [n \\in Nodes |-> {}] \\* empty config\n /\\ lastAcceptedTerm = [n \\in Nodes |-> 0]\n /\\ lastAcceptedVersion = initialAcceptedVersion\n /\\ lastAcceptedValue \\in {[n \\in Nodes |-> v] : v \\in Values} \\* all agree on initial value\n /\\ lastAcceptedConfiguration = [n \\in Nodes |-> lastCommittedConfiguration[n]]\n /\\ joinVotes = [n \\in Nodes |-> {}]\n /\\ startedJoinSinceLastReboot = [n \\in Nodes |-> FALSE]\n /\\ electionWon = [n \\in Nodes |-> FALSE]\n /\\ lastPublishedVersion = [n \\in Nodes |-> 0]\n /\\ lastPublishedConfiguration = [n \\in Nodes |-> lastCommittedConfiguration[n]]\n /\\ publishVotes = [n \\in Nodes |-> {}]\n\n\\* Bootstrap node n with the initial state and config \nSetInitialState(n) ==\n /\\ lastAcceptedConfiguration[n] = {} \\* not already bootstrapped\n /\\ Assert(lastAcceptedTerm[n] = 0, \"lastAcceptedTerm should be 0\")\n /\\ Assert(lastCommittedConfiguration[n] = {}, \"lastCommittedConfiguration should be empty\")\n /\\ Assert(lastPublishedVersion[n] = 0, \"lastPublishedVersion should be 0\")\n /\\ Assert(lastPublishedConfiguration[n] = {}, \"lastPublishedConfiguration should be empty\")\n /\\ Assert(electionWon[n] = FALSE, \"electionWon should be FALSE\")\n /\\ Assert(joinVotes[n] = {}, \"joinVotes should be empty\")\n /\\ Assert(publishVotes[n] = {}, \"publishVotes should be empty\")\n /\\ lastAcceptedConfiguration' = [lastAcceptedConfiguration EXCEPT ![n] = initialConfiguration]\n /\\ lastAcceptedValue' = [lastAcceptedValue EXCEPT ![n] = initialValue]\n /\\ lastCommittedConfiguration' = [lastCommittedConfiguration EXCEPT ![n] = initialConfiguration]\n /\\ Assert(lastAcceptedTerm[n] = 0, \"lastAcceptedTerm should be 0\")\n /\\ Assert(lastAcceptedConfiguration'[n] /= {}, \"lastAcceptedConfiguration should be non-empty\")\n /\\ Assert(lastCommittedConfiguration'[n] /= {}, \"lastCommittedConfiguration should be non-empty\")\n /\\ UNCHANGED <>\n\n\\* Send join request from node n to node nm for term t\nHandleStartJoin(n, nm, t) ==\n /\\ t > currentTerm[n]\n /\\ LET\n joinRequest == [method |-> Join,\n source |-> n,\n dest |-> nm,\n term |-> t,\n laTerm |-> lastAcceptedTerm[n],\n laVersion |-> lastAcceptedVersion[n]]\n IN\n /\\ currentTerm' = [currentTerm EXCEPT ![n] = t]\n /\\ lastPublishedVersion' = [lastPublishedVersion EXCEPT ![n] = 0]\n /\\ lastPublishedConfiguration' = [lastPublishedConfiguration EXCEPT ![n] = lastAcceptedConfiguration[n]]\n /\\ startedJoinSinceLastReboot' = [startedJoinSinceLastReboot EXCEPT ![n] = TRUE]\n /\\ electionWon' = [electionWon EXCEPT ![n] = FALSE]\n /\\ joinVotes' = [joinVotes EXCEPT ![n] = {}]\n /\\ publishVotes' = [publishVotes EXCEPT ![n] = {}]\n /\\ messages' = messages \\cup { joinRequest }\n /\\ UNCHANGED <>\n\n\\* node n handles a join request and checks if it has received enough joins (= votes)\n\\* for its term to be elected as master\nHandleJoin(n, m) ==\n /\\ m.method = Join\n /\\ m.term = currentTerm[n]\n /\\ startedJoinSinceLastReboot[n]\n /\\ \\/ m.laTerm < lastAcceptedTerm[n]\n \\/ /\\ m.laTerm = lastAcceptedTerm[n]\n /\\ m.laVersion <= lastAcceptedVersion[n]\n /\\ lastAcceptedConfiguration[n] /= {} \\* must be bootstrapped\n /\\ joinVotes' = [joinVotes EXCEPT ![n] = @ \\cup { m.source }]\n /\\ electionWon' = [electionWon EXCEPT ![n] = IsElectionQuorum(n, joinVotes'[n])]\n /\\ IF electionWon[n] = FALSE /\\ electionWon'[n]\n THEN\n \\* initiating publish version with last accepted version to enable client requests\n /\\ lastPublishedVersion' = [lastPublishedVersion EXCEPT ![n] = lastAcceptedVersion[n]]\n ELSE\n UNCHANGED <>\n /\\ UNCHANGED <>\n\n\\* client causes a cluster state change val with configuration cfg\nHandleClientValue(n, t, v, val, cfg) ==\n /\\ electionWon[n]\n /\\ lastPublishedVersion[n] = lastAcceptedVersion[n] \\* means we have the last published value / config (useful for CAS operations, where we need to read the previous value first)\n /\\ t = currentTerm[n]\n /\\ v > lastPublishedVersion[n]\n /\\ cfg /= lastAcceptedConfiguration[n] => lastCommittedConfiguration[n] = lastAcceptedConfiguration[n] \\* only allow reconfiguration if there is not already a reconfiguration in progress\n /\\ IsQuorum(joinVotes[n], cfg) \\* only allow reconfiguration if we have a quorum of (join) votes for the new config\n /\\ LET\n publishRequests == { [method |-> PublishRequest,\n source |-> n,\n dest |-> ns,\n term |-> t,\n version |-> v,\n value |-> val,\n config |-> cfg,\n commConf |-> lastCommittedConfiguration[n]] : ns \\in Nodes }\n newEntry == [prevT |-> lastAcceptedTerm[n],\n prevV |-> lastAcceptedVersion[n],\n nextT |-> t,\n nextV |-> v]\n matchingElems == { e \\in descendant : \n /\\ e.nextT = newEntry.prevT\n /\\ e.nextV = newEntry.prevV }\n newTransitiveElems == { [prevT |-> e.prevT,\n prevV |-> e.prevV,\n nextT |-> newEntry.nextT,\n nextV |-> newEntry.nextV] : e \\in matchingElems }\n IN\n /\\ descendant' = descendant \\cup {newEntry} \\cup newTransitiveElems\n /\\ lastPublishedVersion' = [lastPublishedVersion EXCEPT ![n] = v]\n /\\ lastPublishedConfiguration' = [lastPublishedConfiguration EXCEPT ![n] = cfg]\n /\\ publishVotes' = [publishVotes EXCEPT ![n] = {}] \\* publishVotes are only counted per publish version\n /\\ messages' = messages \\cup publishRequests\n /\\ UNCHANGED <>\n\n\\* handle publish request m on node n\nHandlePublishRequest(n, m) ==\n /\\ m.method = PublishRequest\n /\\ m.term = currentTerm[n]\n /\\ (m.term = lastAcceptedTerm[n]) => (m.version > lastAcceptedVersion[n])\n /\\ lastAcceptedTerm' = [lastAcceptedTerm EXCEPT ![n] = m.term]\n /\\ lastAcceptedVersion' = [lastAcceptedVersion EXCEPT ![n] = m.version]\n /\\ lastAcceptedValue' = [lastAcceptedValue EXCEPT ![n] = m.value]\n /\\ lastAcceptedConfiguration' = [lastAcceptedConfiguration EXCEPT ![n] = m.config]\n /\\ lastCommittedConfiguration' = [lastCommittedConfiguration EXCEPT ![n] = m.commConf] \n /\\ LET\n response == [method |-> PublishResponse,\n source |-> n,\n dest |-> m.source,\n term |-> m.term,\n version |-> m.version]\n IN\n /\\ messages' = messages \\cup {response}\n /\\ UNCHANGED <>\n\n\\* node n commits a change\nHandlePublishResponse(n, m) ==\n /\\ m.method = PublishResponse\n /\\ electionWon[n]\n /\\ m.term = currentTerm[n]\n /\\ m.version = lastPublishedVersion[n]\n /\\ publishVotes' = [publishVotes EXCEPT ![n] = @ \\cup {m.source}]\n /\\ IF\n IsPublishQuorum(n, publishVotes'[n])\n THEN\n LET\n commitRequests == { [method |-> Commit,\n source |-> n,\n dest |-> ns,\n term |-> currentTerm[n],\n version |-> lastPublishedVersion[n]] : ns \\in Nodes }\n IN\n /\\ messages' = messages \\cup commitRequests\n ELSE\n UNCHANGED <>\n /\\ UNCHANGED <>\n\n\\* apply committed configuration to node n\nHandleCommit(n, m) ==\n /\\ m.method = Commit\n /\\ m.term = currentTerm[n]\n /\\ m.term = lastAcceptedTerm[n]\n /\\ m.version = lastAcceptedVersion[n]\n /\\ (electionWon[n] => lastAcceptedVersion[n] = lastPublishedVersion[n])\n /\\ lastCommittedConfiguration' = [lastCommittedConfiguration EXCEPT ![n] = lastAcceptedConfiguration[n]]\n /\\ UNCHANGED <>\n\n\\* crash/restart node n (loses ephemeral state)\nRestartNode(n) ==\n /\\ joinVotes' = [joinVotes EXCEPT ![n] = {}]\n /\\ startedJoinSinceLastReboot' = [startedJoinSinceLastReboot EXCEPT ![n] = FALSE]\n /\\ electionWon' = [electionWon EXCEPT ![n] = FALSE]\n /\\ lastPublishedVersion' = [lastPublishedVersion EXCEPT ![n] = 0]\n /\\ lastPublishedConfiguration' = [lastPublishedConfiguration EXCEPT ![n] = lastAcceptedConfiguration[n]]\n /\\ publishVotes' = [publishVotes EXCEPT ![n] = {}]\n /\\ UNCHANGED <>\n\n\\* next-step relation\nNext ==\n \\/ \\E n \\in Nodes : SetInitialState(n)\n \\/ \\E n, nm \\in Nodes : \\E t \\in Terms : HandleStartJoin(n, nm, t)\n \\/ \\E m \\in messages : HandleJoin(m.dest, m)\n \\/ \\E n \\in Nodes : \\E t \\in Terms : \\E v \\in Versions : \\E val \\in Values : \\E vs \\in ValidConfigs : HandleClientValue(n, t, v, val, vs)\n \\/ \\E m \\in messages : HandlePublishRequest(m.dest, m)\n \\/ \\E m \\in messages : HandlePublishResponse(m.dest, m)\n \\/ \\E m \\in messages : HandleCommit(m.dest, m)\n \\/ \\E n \\in Nodes : RestartNode(n)\n\n----\n\n\\* Invariants\n\nSingleNodeInvariant ==\n \\A n \\in Nodes :\n /\\ lastAcceptedTerm[n] <= currentTerm[n]\n /\\ electionWon[n] = IsElectionQuorum(n, joinVotes[n]) \\* cached value is consistent\n /\\ IF electionWon[n] THEN lastPublishedVersion[n] >= lastAcceptedVersion[n] ELSE lastPublishedVersion[n] = 0\n /\\ electionWon[n] => startedJoinSinceLastReboot[n]\n /\\ publishVotes[n] /= {} => electionWon[n]\n\nOneMasterPerTerm ==\n \\A m1, m2 \\in messages:\n /\\ m1.method = PublishRequest\n /\\ m2.method = PublishRequest\n /\\ m1.term = m2.term\n => m1.source = m2.source\n\nLogMatching ==\n \\A m1, m2 \\in messages:\n /\\ m1.method = PublishRequest\n /\\ m2.method = PublishRequest\n /\\ m1.term = m2.term\n /\\ m1.version = m2.version\n => m1.value = m2.value\n\nCommittedPublishRequest(mp) ==\n /\\ mp.method = PublishRequest\n /\\ \\E mc \\in messages:\n /\\ mc.method = Commit\n /\\ mp.term = mc.term\n /\\ mp.version = mc.version\n\nDescendantRelationIsStrictlyOrdered ==\n \\A d \\in descendant:\n /\\ d.prevT <= d.nextT\n /\\ d.prevV < d.nextV\n\nDescendantRelationIsTransitive ==\n \\A d1, d2 \\in descendant:\n d1.nextT = d2.prevT /\\ d1.nextV = d2.prevV \n => [prevT |-> d1.prevT, prevV |-> d1.prevV, nextT |-> d2.nextT, nextV |-> d2.nextV] \\in descendant\n\nNewerOpsBasedOnOlderCommittedOps ==\n \\A m1, m2 \\in messages :\n /\\ CommittedPublishRequest(m1)\n /\\ m2.method = PublishRequest\n /\\ m2.term >= m1.term\n /\\ m2.version > m1.version\n => [prevT |-> m1.term, prevV |-> m1.version, nextT |-> m2.term, nextV |-> m2.version] \\in descendant\n\n\\* main invariant (follows from NewerOpsBasedOnOlderCommittedOps):\nCommittedValuesDescendantsFromCommittedValues ==\n \\A m1, m2 \\in messages : \n /\\ CommittedPublishRequest(m1)\n /\\ CommittedPublishRequest(m2)\n /\\ \\/ m1.term /= m2.term\n \\/ m1.version /= m2.version\n =>\n \\/ [prevT |-> m1.term, prevV |-> m1.version, nextT |-> m2.term, nextV |-> m2.version] \\in descendant \n \\/ [prevT |-> m2.term, prevV |-> m2.version, nextT |-> m1.term, nextV |-> m1.version] \\in descendant\n\nCommittedValuesDescendantsFromInitialValue ==\n \\E v \\in InitialVersions :\n /\\ \\E n \\in Nodes : v = initialAcceptedVersion[n]\n /\\ \\E votes \\in SUBSET(initialConfiguration) :\n /\\ IsQuorum(votes, initialConfiguration)\n /\\ \\A n \\in votes : initialAcceptedVersion[n] <= v\n /\\ \\A m \\in messages :\n CommittedPublishRequest(m)\n =>\n [prevT |-> 0, prevV |-> v, nextT |-> m.term, nextV |-> m.version] \\in descendant\n\nCommitHasQuorumVsPreviousCommittedConfiguration ==\n \\A mc \\in messages: mc.method = Commit\n => (\\A mprq \\in messages: (/\\ mprq.method = PublishRequest\n /\\ mprq.term = mc.term\n /\\ mprq.version = mc.version) \n\n => IsQuorum({mprs.source: mprs \\in {mprs \\in messages: /\\ mprs.method = PublishResponse\n /\\ mprs.term = mprq.term\n /\\ mprs.version = mprq.version\n }}, mprq.commConf))\n\nP2bInvariant ==\n \\A mc \\in messages: mc.method = Commit\n => (\\A mprq \\in messages: mprq.method = PublishRequest\n => (mprq.term > mc.term => mprq.version > mc.version))\n\n\\* State-exploration limits\nStateConstraint ==\n /\\ \\A n \\in Nodes: IF currentTerm[n] <= 1 THEN lastPublishedVersion[n] <= 2 ELSE lastPublishedVersion[n] <= 3\n /\\ Cardinality(messages) <= 15\n\n====================================================================================================\n"
},
{
"path": "ZenWithTerms/tla/ZenWithTerms.toolbox/.project",
"content": "\n\n\tZenWithTerms\n\t\n\t\n\t\n\t\n\t\t\n\t\t\ttoolbox.builder.TLAParserBuilder\n\t\t\t\n\t\t\t\n\t\t\n\t\t\n\t\t\ttoolbox.builder.PCalAlgorithmSearchingBuilder\n\t\t\t\n\t\t\t\n\t\t\n\t\n\t\n\t\ttoolbox.natures.TLANature\n\t\n\t\n\t\t\n\t\t\tZenWithTerms.tla\n\t\t\t1\n\t\t\tPARENT-1-PROJECT_LOC/ZenWithTerms.tla\n\t\t\n\t\n\n"
},
{
"path": "ZenWithTerms/tla/ZenWithTerms.toolbox/.settings/org.lamport.tla.toolbox.prefs",
"content": "ProjectRootFile=PARENT-1-PROJECT_LOC/ZenWithTerms.tla\neclipse.preferences.version=1\n"
},
{
"path": "ZenWithTerms/tla/ZenWithTerms.toolbox/ZenWithTerms___model.launch",
"content": "\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n"
},
{
"path": "cluster/isabelle/Implementation.thy",
"content": "section \\Implementation\\\n\ntext \\This section presents the implementation of the algorithm.\\\n\ntheory Implementation\n imports Preliminaries\nbegin\n\nsubsection \\Protocol messages\\\n\ntext \\The\nproven-safe core of the protocol works by sending messages as described here. The remainder of the\nprotocol may send other messages too, and may drop, reorder or duplicate any of these messages, but\nmust not send these messages itself to ensure safety. Another way of thinking of these messages is\nto consider them as ``fire-and-forget'' RPC invocations that, on receipt, call some local method, maybe\nupdate the receiving node's state, and maybe yield some further messages. The @{type nat} parameter to each\nmessage refers to a slot number.\\\n\ndatatype TermOption = NO_TERM | SomeTerm Term\n\ninstantiation TermOption :: linorder\nbegin\n\nfun less_TermOption :: \"TermOption \\ TermOption \\ bool\"\n where \"t < NO_TERM = False\"\n | \"NO_TERM < SomeTerm t = True\"\n | \"SomeTerm t\\<^sub>1 < SomeTerm t\\<^sub>2 = (t\\<^sub>1 < t\\<^sub>2)\"\n\ndefinition less_eq_TermOption :: \"TermOption \\ TermOption \\ bool\"\n where \"(t\\<^sub>1 :: TermOption) \\ t\\<^sub>2 \\ t\\<^sub>1 = t\\<^sub>2 \\ t\\<^sub>1 < t\\<^sub>2\"\n\ninstance proof\n fix x y z :: TermOption\n show \"(x < y) = (x \\ y \\ \\ y \\ x)\" unfolding less_eq_TermOption_def apply auto\n using less_TermOption.elims apply fastforce\n by (metis less_TermOption.elims(2) less_TermOption.simps(3) less_not_sym)\n\n show \"x \\ x\" by (simp add: less_eq_TermOption_def)\n\n show \"x \\ y \\ y \\ z \\ x \\ z\" unfolding less_eq_TermOption_def apply auto\n by (metis TermOption.distinct(1) TermOption.inject dual_order.strict_trans less_TermOption.elims(2) less_TermOption.elims(3))\n\n show \"x \\ y \\ y \\ x \\ x = y\" unfolding less_eq_TermOption_def apply auto\n using \\(x < y) = (x \\ y \\ \\ y \\ x)\\ less_eq_TermOption_def by blast\n\n show \"x \\ y \\ y \\ x\" unfolding less_eq_TermOption_def apply auto\n by (metis TermOption.distinct(1) TermOption.inject less_TermOption.elims(3) neqE)\nqed\n\nend\n\nlemma NO_TERM_le [simp]: \"NO_TERM \\ t\" by (cases t, simp_all add: less_eq_TermOption_def)\nlemma le_NO_TERM [simp]: \"(t \\ NO_TERM) = (t = NO_TERM)\" by (cases t, simp_all add: less_eq_TermOption_def)\nlemma le_SomeTerm [simp]: \"(SomeTerm t\\<^sub>1 \\ SomeTerm t\\<^sub>2) = (t\\<^sub>1 \\ t\\<^sub>2)\" by (auto simp add: less_eq_TermOption_def)\n\ndatatype Message\n = StartJoin Term\n | Vote Slot Term TermOption\n | ClientValue Value\n | PublishRequest Slot Term Value\n | PublishResponse Slot Term\n | ApplyCommit Slot Term\n | CatchUpRequest\n | CatchUpResponse Slot \"Node set\" ClusterState\n | DiscardJoinVotes\n | Reboot\n\ntext \\Some prose descriptions of these messages follows, in order to give a bit more of an\nintuitive understanding of their purposes.\\\n\ntext \\The message @{term \"StartJoin t\"} may be sent by any node to attempt to start a master\nelection in the given term @{term t}.\\\n\ntext \\The message @{term \"Vote i t a\"} may be sent by a node in response\nto a @{term StartJoin} message. It indicates that the sender knows all committed values for slots\nstrictly below @{term i}, and that the sender will no longer vote (i.e. send an @{term\nPublishResponse}) in any term prior to @{term t}. The field @{term a} is either @{term\nNone} or @{term \"Some t'\"}. In the former case this indicates that\nthe node has not yet sent any @{term PublishResponse} message in slot @{term i}, and in the latter\ncase it indicates that the largest term in which it has previously sent an @{term PublishResponse}\nmessage is @{term t'}. All\nnodes must avoid sending a @{term Vote} message to two different masters in the same term.\\\n\ntext \\The message @{term \"ClientValue x\"} may be sent by any node and indicates an attempt to\nreach consensus on the value @{term x}.\\\n\ntext \\The message @{term \"PublishRequest i t v\"} may be sent by the elected master of term\n@{term t} to request the other master-eligible nodes to vote for value @{term v} to be committed in\nslot @{term i}.\\\n\ntext \\The message @{term \"PublishResponse i t\"} may be sent by node in response to\nthe corresponding @{term PublishRequest} message, indicating that the sender votes for the value\nproposed by the master of term @{term t} to be committed in slot @{term i}.\\\n\ntext \\The message @{term \"ApplyCommit i t\"} indicates that the value proposed by the master of\nterm @{term t} in slot @{term i} received a quorum of votes and is therefore committed.\\\n\ntext \\The message @{term Reboot} may be sent by any node to represent the restart of a node, which\nloses any ephemeral state.\\\n\ntext \\The abstract model of Zen keeps track of the set of all messages that have ever been\nsent, and asserts that this set obeys certain invariants, listed below. Further below, it will be\nshown that these invariants imply that each slot obeys the @{term oneSlot} invariants above and\nhence that each slot cannot see inconsistent committed values.\\\n\ndatatype Destination = Broadcast | OneNode Node\n\nrecord RoutedMessage =\n sender :: Node\n destination :: Destination\n payload :: Message\n\ntext \\It will be useful to be able to choose the optional term with the greater term,\nso here is a function that does that.\\\n\nsubsection \\Node implementation\\\n\ntext \\Each node holds the following local data.\\\n\nrecord TermValue =\n tvTerm :: Term\n tvValue :: Value\n\nrecord NodeData =\n currentNode :: Node\n currentTerm :: Term\n (* committed state *)\n firstUncommittedSlot :: Slot\n currentVotingNodes :: \"Node set\"\n currentClusterState :: ClusterState\n (* accepted state *)\n lastAcceptedData :: \"TermValue option\"\n (* election state *)\n joinVotes :: \"Node set\"\n electionWon :: bool\n (* publish state *)\n publishPermitted :: bool\n publishVotes :: \"Node set\"\n\ndefinition lastAcceptedValue :: \"NodeData \\ Value\"\n where \"lastAcceptedValue nd \\ tvValue (THE lad. lastAcceptedData nd = Some lad)\"\n\ndefinition lastAcceptedTerm :: \"NodeData \\ TermOption\"\n where \"lastAcceptedTerm nd \\ case lastAcceptedData nd of None \\ NO_TERM | Some lad \\ SomeTerm (tvTerm lad)\"\n\ndefinition isQuorum :: \"NodeData \\ Node set \\ bool\"\n where \"isQuorum nd q \\ q \\ majorities (currentVotingNodes nd)\"\n\nlemma lastAcceptedValue_joinVotes_update[simp]: \"lastAcceptedValue (joinVotes_update f nd) = lastAcceptedValue nd\" by (simp add: lastAcceptedValue_def)\nlemma lastAcceptedTerm_joinVotes_update[simp]: \"lastAcceptedTerm (joinVotes_update f nd) = lastAcceptedTerm nd\" by (simp add: lastAcceptedTerm_def)\n\nlemma lastAcceptedValue_electionWon_update[simp]: \"lastAcceptedValue (electionWon_update f nd) = lastAcceptedValue nd\" by (simp add: lastAcceptedValue_def)\nlemma lastAcceptedTerm_electionWon_update[simp]: \"lastAcceptedTerm (electionWon_update f nd) = lastAcceptedTerm nd\" by (simp add: lastAcceptedTerm_def)\n\ntext \\This method publishes a value via a @{term PublishRequest} message.\\\n\ndefinition publishValue :: \"Value \\ NodeData \\ (NodeData * Message option)\"\n where\n \"publishValue x nd \\\n if electionWon nd \\ publishPermitted nd\n then ( nd \\ publishPermitted := False \\\n , Some (PublishRequest\n (firstUncommittedSlot nd)\n (currentTerm nd) x) )\n else (nd, None)\"\n\ntext \\This method updates the node's current term (if necessary) and discards any data associated\nwith the previous term.\\\n\ndefinition ensureCurrentTerm :: \"Term \\ NodeData \\ NodeData\"\n where\n \"ensureCurrentTerm t nd \\\n if t \\ currentTerm nd\n then nd\n else nd\n \\ joinVotes := {}\n , currentTerm := t\n , electionWon := False\n , publishPermitted := True\n , publishVotes := {} \\\"\n\ntext \\This method updates the node's state on receipt of a vote (a @{term Vote}) in an election.\\\n\ndefinition addElectionVote :: \"Node \\ Slot => TermOption \\ NodeData \\ NodeData\"\n where\n \"addElectionVote s i a nd \\ let newVotes = insert s (joinVotes nd)\n in nd \\ joinVotes := newVotes\n , electionWon := isQuorum nd newVotes \\\"\n\ntext \\Clients request the cluster to achieve consensus on certain values using the @{term ClientValue}\nmessage which is handled as follows.\\\n\ndefinition handleClientValue :: \"Value \\ NodeData \\ (NodeData * Message option)\"\n where\n \"handleClientValue x nd \\ if lastAcceptedTerm nd = NO_TERM then publishValue x nd else (nd, None)\"\n\ntext \\A @{term StartJoin} message is checked for acceptability and then handled by updating the\nnode's term and yielding a @{term Vote} message as follows.\\\n\ndefinition handleStartJoin :: \"Term \\ NodeData \\ (NodeData * Message option)\"\n where\n \"handleStartJoin t nd \\\n if currentTerm nd < t\n then ( ensureCurrentTerm t nd\n , Some (Vote (firstUncommittedSlot nd)\n t\n (lastAcceptedTerm nd)))\n else (nd, None)\"\n\ntext \\A @{term Vote} message is checked for acceptability and then handled as follows, perhaps\nyielding a @{term PublishRequest} message.\\\n\ndefinition handleVote :: \"Node \\ Slot \\ Term \\ TermOption \\ NodeData \\ (NodeData * Message option)\"\n where\n \"handleVote s i t a nd \\\n if t = currentTerm nd\n \\ (i < firstUncommittedSlot nd\n \\ (i = firstUncommittedSlot nd \\ a \\ lastAcceptedTerm nd))\n then let nd1 = addElectionVote s i a nd\n in (if lastAcceptedTerm nd = NO_TERM then (nd1, None) else publishValue (lastAcceptedValue nd1) nd1)\n else (nd, None)\"\n\ntext \\A @{term PublishRequest} message is checked for acceptability and then handled as follows,\nyielding a @{term PublishResponse} message.\\\n\ndefinition handlePublishRequest :: \"Slot \\ Term \\ Value \\ NodeData \\ (NodeData * Message option)\"\n where\n \"handlePublishRequest i t x nd \\\n if i = firstUncommittedSlot nd\n \\ t = currentTerm nd\n then ( nd \\ lastAcceptedData := Some \\ tvTerm = t, tvValue = x \\ \\\n , Some (PublishResponse i t))\n else (nd, None)\"\n\ntext \\This method sends an @{term ApplyCommit} message if a quorum of votes has been received.\\\n\ndefinition commitIfQuorate :: \"NodeData \\ (NodeData * Message option)\"\n where\n \"commitIfQuorate nd = (nd, if isQuorum nd (publishVotes nd)\n then Some (ApplyCommit (firstUncommittedSlot nd) (currentTerm nd)) else None)\"\n\ntext \\A @{term PublishResponse} message is checked for acceptability and handled as follows. If\nthis message, together with the previously-received messages, forms a quorum of votes then the\nvalue is committed, yielding an @{term ApplyCommit} message.\\\n\ndefinition handlePublishResponse :: \"Node \\ Slot \\ Term \\ NodeData \\ (NodeData * Message option)\"\n where\n \"handlePublishResponse s i t nd \\\n if i = firstUncommittedSlot nd \\ t = currentTerm nd\n then commitIfQuorate (nd \\ publishVotes := insert s (publishVotes nd) \\)\n else (nd, None)\"\n\ntext \\This method updates the node's state when a value is committed.\\\n\ndefinition applyAcceptedValue :: \"NodeData \\ NodeData\"\n where\n \"applyAcceptedValue nd \\ case lastAcceptedValue nd of\n NoOp \\ nd\n | Reconfigure votingNodes \\ nd\n \\ currentVotingNodes := set votingNodes\n , electionWon := joinVotes nd \\ majorities (set votingNodes) \\\n | ClusterStateDiff diff \\ nd \\ currentClusterState := diff (currentClusterState nd) \\\"\n\ntext \\An @{term ApplyCommit} message is applied to the current node's state, updating its configuration\nand \\texttt{ClusterState} via the @{term applyValue} method. It yields no messages.\\\n\ndefinition handleApplyCommit :: \"Slot \\ Term \\ NodeData \\ NodeData\"\n where\n \"handleApplyCommit i t nd \\\n if i = firstUncommittedSlot nd \\ lastAcceptedTerm nd = SomeTerm t\n then (applyAcceptedValue nd)\n \\ firstUncommittedSlot := i + 1\n , lastAcceptedData := None\n , publishPermitted := True\n , publishVotes := {} \\\n else nd\"\n\ndefinition handleCatchUpRequest :: \"NodeData \\ (NodeData * Message option)\"\n where\n \"handleCatchUpRequest nd = (nd, Some (CatchUpResponse (firstUncommittedSlot nd)\n (currentVotingNodes nd) (currentClusterState nd)))\"\n\ndefinition handleCatchUpResponse :: \"Slot \\ Node set \\ ClusterState \\ NodeData \\ NodeData\"\n where\n \"handleCatchUpResponse i conf cs nd \\\n if firstUncommittedSlot nd < i\n then nd \\ firstUncommittedSlot := i\n , publishPermitted := True\n , publishVotes := {}\n , currentVotingNodes := conf\n , currentClusterState := cs\n , lastAcceptedData := None\n , joinVotes := {}\n , electionWon := False \\\n else nd\"\n\ntext \\A @{term Reboot} message simulates the effect of a reboot, discarding any ephemeral state but\npreserving the persistent state. It yields no messages.\\\n\ndefinition handleReboot :: \"NodeData \\ NodeData\"\n where\n \"handleReboot nd \\\n \\ currentNode = currentNode nd\n , currentTerm = currentTerm nd\n , firstUncommittedSlot = firstUncommittedSlot nd\n , currentVotingNodes = currentVotingNodes nd\n , currentClusterState = currentClusterState nd\n , lastAcceptedData = lastAcceptedData nd\n , joinVotes = {}\n , electionWon = False\n , publishPermitted = False\n , publishVotes = {} \\\"\n\ntext \\A @{term DiscardJoinVotes} message discards the votes received by a node. It yields\nno messages.\\\n\ndefinition handleDiscardJoinVotes :: \"NodeData \\ NodeData\"\n where\n \"handleDiscardJoinVotes nd \\ nd \\ electionWon := False, joinVotes := {} \\\"\n\ntext \\This function dispatches incoming messages to the appropriate handler method, and\nroutes any responses to the appropriate places. In particular, @{term Vote} messages\n(sent by the @{term handleStartJoin} method) and\n@{term PublishResponse} messages (sent by the @{term handlePublishRequest} method) are\nonly sent to a single node, whereas all other responses are broadcast to all nodes.\\\n\ndefinition ProcessMessage :: \"NodeData \\ RoutedMessage \\ (NodeData * RoutedMessage option)\"\n where\n \"ProcessMessage nd msg \\\n let respondTo =\n (\\ d (nd, mmsg). case mmsg of\n None \\ (nd, None)\n | Some msg \\ (nd,\n Some \\ sender = currentNode nd, destination = d,\n payload = msg \\));\n respondToSender = respondTo (OneNode (sender msg));\n respondToAll = respondTo Broadcast\n in\n if destination msg \\ { Broadcast, OneNode (currentNode nd) }\n then case payload msg of\n StartJoin t\n \\ respondToSender (handleStartJoin t nd)\n | Vote i t a\n \\ respondToAll (handleVote (sender msg) i t a nd)\n | ClientValue x\n \\ respondToAll (handleClientValue x nd)\n | PublishRequest i t x\n \\ respondToSender (handlePublishRequest i t x nd)\n | PublishResponse i t\n \\ respondToAll (handlePublishResponse (sender msg) i t nd)\n | ApplyCommit i t\n \\ (handleApplyCommit i t nd, None)\n | CatchUpRequest\n \\ respondToSender (handleCatchUpRequest nd)\n | CatchUpResponse i conf cs\n \\ (handleCatchUpResponse i conf cs nd, None)\n | DiscardJoinVotes\n \\ (handleDiscardJoinVotes nd, None)\n | Reboot\n \\ (handleReboot nd, None)\n else (nd, None)\"\n\ntext \\Nodes are initialised to this state. The data required is the initial configuration, @{term Q\\<^sub>0}\nand the initial \\texttt{ClusterState}, here shown as @{term \"ClusterState 0\"}.\\\n\ndefinition initialNodeState :: \"Node \\ NodeData\"\n where \"initialNodeState n =\n \\ currentNode = n\n , currentTerm = 0\n , firstUncommittedSlot = 0\n , currentVotingNodes = V\\<^sub>0\n , currentClusterState = CS\\<^sub>0\n , lastAcceptedData = None\n , joinVotes = {}\n , electionWon = False\n , publishPermitted = False\n , publishVotes = {} \\\"\n(* Note: publishPermitted could be True initially, but in the actual implementation we call the\nsame constructor whether we're starting up from afresh or recovering from a reboot, and the value\nis really unimportant as we need to run an election in a new term before becoming master anyway,\nso it's hard to justify putting any effort into calculating different values for these two cases.\nInstead just set it to False initially.*)\n\nend\n"
},
{
"path": "cluster/isabelle/Monadic.thy",
"content": "theory Monadic\n imports Implementation \"~~/src/HOL/Library/Monad_Syntax\"\nbegin\n\ndatatype Exception = IllegalArgumentException\n\ndatatype ('e,'a) Result = Success 'a | Exception 'e\n\ndatatype 'a Action = Action \"NodeData \\ (NodeData * RoutedMessage list * (Exception,'a) Result)\"\n\ndefinition runM :: \"'a Action \\ NodeData \\ (NodeData * RoutedMessage list * (Exception,'a) Result)\"\n where \"runM ma \\ case ma of Action unwrapped_ma \\ unwrapped_ma\"\n\nlemma runM_Action[simp]: \"runM (Action f) = f\" by (simp add: runM_def)\nlemma runM_inject[intro]: \"(\\nd. runM ma nd = runM mb nd) \\ ma = mb\" by (cases ma, cases mb, auto simp add: runM_def)\n\ndefinition return :: \"'a \\ 'a Action\" where \"return a \\ Action (\\ nd. (nd, [], Success a))\"\n\nlemma runM_return[simp]: \"runM (return a) nd = (nd, [], Success a)\" unfolding runM_def return_def by simp\n\ndefinition Action_bind :: \"'a Action \\ ('a \\ 'b Action) \\ 'b Action\"\n where \"Action_bind ma mf \\ Action (\\ nd0. case runM ma nd0 of\n (nd1, msgs1, result1) \\ (case result1 of\n Exception e \\ (nd1, msgs1, Exception e)\n | Success a \\ (case runM (mf a) nd1 of\n (nd2, msgs2, result2) \\ (nd2, msgs1 @ msgs2, result2))))\"\n\nadhoc_overloading bind Action_bind\n\nlemma runM_bind: \"runM (a \\ f) nd0 = (case runM a nd0 of (nd1, msgs1, result1) \\ (case result1 of Exception e \\ (nd1, msgs1, Exception e) | Success b \\