Repository: automerge/automerge-classic Branch: main Commit: 0605308926a0 Files: 51 Total size: 859.2 KB Directory structure: gitextract_5mussk42/ ├── .babelrc ├── .eslintrc.json ├── .github/ │ └── workflows/ │ └── automerge-ci.yml ├── .gitignore ├── .mocharc.yaml ├── @types/ │ └── automerge/ │ └── index.d.ts ├── CHANGELOG.md ├── LICENSE ├── README.md ├── backend/ │ ├── backend.js │ ├── columnar.js │ ├── encoding.js │ ├── index.js │ ├── new.js │ ├── sync.js │ └── util.js ├── frontend/ │ ├── apply_patch.js │ ├── constants.js │ ├── context.js │ ├── counter.js │ ├── index.js │ ├── numbers.js │ ├── observable.js │ ├── proxies.js │ ├── table.js │ └── text.js ├── karma.conf.js ├── karma.sauce.js ├── package.json ├── src/ │ ├── automerge.js │ ├── common.js │ └── uuid.js ├── test/ │ ├── backend_test.js │ ├── columnar_test.js │ ├── context_test.js │ ├── encoding_test.js │ ├── frontend_test.js │ ├── fuzz_test.js │ ├── helpers.js │ ├── new_backend_test.js │ ├── observable_test.js │ ├── proxies_test.js │ ├── sync_test.js │ ├── table_test.js │ ├── test.js │ ├── text_test.js │ ├── typescript_test.ts │ ├── uuid_test.js │ └── wasm.js ├── tsconfig.json └── webpack.config.js ================================================ FILE CONTENTS ================================================ ================================================ FILE: .babelrc ================================================ { "presets": [ [ "@babel/preset-env" ] ] } ================================================ FILE: .eslintrc.json ================================================ { "env": { "browser": true, "commonjs": true, "es2015": true, "node": true, "mocha": true }, "ignorePatterns": "dist/**", "extends": ["eslint:recommended", "plugin:compat/recommended"], "parserOptions": { "ecmaVersion": 2015 }, "rules": { "accessor-pairs": "error", "array-bracket-newline": "off", "array-bracket-spacing": "off", "array-callback-return": "error", "array-element-newline": "off", "arrow-body-style": "error", "arrow-parens": "off", "arrow-spacing": [ "error", { "after": true, "before": true } ], "block-scoped-var": "error", "block-spacing": "error", "brace-style": [ "error", "1tbs", { "allowSingleLine": true } ], "camelcase": "off", "capitalized-comments": "off", "class-methods-use-this": "off", "comma-dangle": "off", "comma-spacing": [ "error", { "after": true, "before": false } ], "comma-style": [ "error", "last" ], "complexity": "off", "computed-property-spacing": [ "off", "never" ], "consistent-return": "off", "consistent-this": "error", "curly": "off", "default-case": "off", "default-case-last": "error", "default-param-last": "error", "dot-location": [ "error", "property" ], "dot-notation": "error", "eol-last": "error", "eqeqeq": "off", "func-call-spacing": "off", "func-name-matching": "error", "func-names": "off", "func-style": [ "error", "declaration" ], "function-paren-newline": "off", "generator-star-spacing": "error", "grouped-accessor-pairs": "error", "guard-for-in": "error", "id-denylist": "error", "id-length": "off", "id-match": "error", "implicit-arrow-linebreak": "off", "indent": "off", "init-declarations": "off", "jsx-quotes": "error", "key-spacing": "off", "keyword-spacing": [ "error", { "after": true, "before": true } ], "line-comment-position": "off", "linebreak-style": "off", "lines-around-comment": "off", "lines-between-class-members": "error", "max-classes-per-file": "off", "max-depth": "off", "max-len": "off", "max-lines": "off", "max-lines-per-function": "off", "max-nested-callbacks": "error", "max-params": "off", "max-statements": "off", "max-statements-per-line": "off", "multiline-comment-style": [ "error", "separate-lines" ], "new-parens": "error", "newline-per-chained-call": "off", "no-alert": "error", "no-array-constructor": "error", "no-await-in-loop": "error", "no-bitwise": "off", "no-caller": "error", "no-confusing-arrow": "error", "no-console": "error", "no-constant-condition": [ "error", { "checkLoops": false } ], "no-constructor-return": "error", "no-continue": "off", "no-div-regex": "error", "no-duplicate-imports": "error", "no-else-return": "off", "no-empty-function": "off", "no-eq-null": "error", "no-eval": "error", "no-extend-native": "error", "no-extra-bind": "error", "no-extra-label": "error", "no-extra-parens": "off", "no-floating-decimal": "error", "no-implicit-coercion": "off", "no-implicit-globals": "error", "no-implied-eval": "error", "no-inline-comments": "off", "no-invalid-this": "error", "no-iterator": "error", "no-label-var": "error", "no-labels": "error", "no-lone-blocks": "error", "no-lonely-if": "off", "no-loop-func": "off", "no-loss-of-precision": "error", "no-magic-numbers": "off", "no-mixed-operators": "off", "no-multi-assign": "error", "no-multi-spaces": "off", "no-multi-str": "error", "no-multiple-empty-lines": "error", "no-negated-condition": "off", "no-nested-ternary": "off", "no-new": "error", "no-new-func": "error", "no-new-object": "error", "no-new-wrappers": "error", "no-nonoctal-decimal-escape": "error", "no-octal-escape": "error", "no-param-reassign": "off", "no-plusplus": "off", "no-promise-executor-return": "error", "no-proto": "error", "no-restricted-exports": "error", "no-restricted-globals": "error", "no-restricted-imports": "error", "no-restricted-properties": "error", "no-restricted-syntax": "error", "no-return-assign": "off", "no-return-await": "error", "no-script-url": "error", "no-self-compare": "error", "no-sequences": "error", "no-shadow": "off", "no-tabs": "error", "no-template-curly-in-string": "error", "no-ternary": "off", "no-throw-literal": "error", "no-trailing-spaces": "error", "no-undef-init": "off", "no-undefined": "off", "no-underscore-dangle": "off", "no-unmodified-loop-condition": "error", "no-unneeded-ternary": "error", "no-unreachable-loop": "error", "no-unsafe-optional-chaining": "error", "no-unused-expressions": "error", "no-unused-vars": ["error", { "args": "after-used" }], "no-use-before-define": "off", "no-useless-backreference": "error", "no-useless-call": "error", "no-useless-computed-key": "error", "no-useless-concat": "error", "no-useless-constructor": "error", "no-useless-rename": "error", "no-useless-return": "error", "no-var": "error", "no-void": "error", "no-warning-comments": "off", "no-whitespace-before-property": "off", "nonblock-statement-body-position": "error", "object-curly-newline": "error", "object-curly-spacing": "off", "object-property-newline": "off", "object-shorthand": "error", "one-var": "off", "one-var-declaration-per-line": "off", "operator-assignment": "off", "operator-linebreak": "error", "padded-blocks": "off", "padding-line-between-statements": "error", "prefer-arrow-callback": "off", "prefer-const": "off", "prefer-destructuring": "off", "prefer-exponentiation-operator": "error", "prefer-named-capture-group": "off", "prefer-numeric-literals": "error", "prefer-object-spread": "off", "prefer-promise-reject-errors": "error", "prefer-regex-literals": "error", "prefer-rest-params": "error", "prefer-spread": "error", "prefer-template": "off", "quote-props": "off", "quotes": "off", "radix": "error", "require-atomic-updates": "error", "require-await": "error", "require-unicode-regexp": "off", "rest-spread-spacing": "error", "semi": "off", "semi-spacing": [ "error", { "after": true, "before": false } ], "semi-style": [ "error", "first" ], "sort-imports": "error", "sort-keys": "off", "sort-vars": "off", "space-before-blocks": "error", "space-before-function-paren": "off", "space-in-parens": [ "error", "never" ], "space-infix-ops": "error", "space-unary-ops": "error", "spaced-comment": [ "error", "always" ], "strict": [ "error", "never" ], "switch-colon-spacing": "error", "symbol-description": "error", "template-curly-spacing": [ "error", "never" ], "template-tag-spacing": "error", "unicode-bom": [ "error", "never" ], "vars-on-top": "error", "wrap-iife": "error", "wrap-regex": "off", "yield-star-spacing": "error", "yoda": [ "error", "never" ] } } ================================================ FILE: .github/workflows/automerge-ci.yml ================================================ name: CI on: [push, pull_request] jobs: node-build: runs-on: ubuntu-latest strategy: matrix: node-version: [12.x, 14.x, 16.x] steps: - name: Check out repo uses: actions/checkout@v2 - name: Use Node.js ${{ matrix.node-version }} uses: actions/setup-node@v2 with: node-version: ${{ matrix.node-version }} cache: 'yarn' - name: Install dependencies run: yarn - name: ESLint run: yarn lint - name: Test suite run: yarn test - name: Bundle run: yarn build - name: Test suite using bundle run: TEST_DIST=1 yarn test - name: Load bundled code run: node -e "const Automerge = require(\"./dist/automerge\")" # browsertest: # runs-on: ubuntu-latest # # Don't run this job when triggered from a forked repository, since the secrets # # (Sauce Labs credentials) are not available in that context # if: ${{ github.repository == 'automerge/automerge' }} # steps: # - uses: actions/checkout@v2 # - name: Use Node.js # uses: actions/setup-node@v2 # with: # node-version: 16.x # cache: 'yarn' # - name: Install dependencies # run: yarn # - name: Bundle # run: yarn build # - name: Sauce Connect # uses: saucelabs/sauce-connect-action@v1 # with: # username: ${{ secrets.SAUCE_USERNAME }} # accessKey: ${{ secrets.SAUCE_ACCESS_KEY }} # tunnelIdentifier: github-action-tunnel # scVersion: 4.7.0 # - name: Run browser tests # run: node_modules/.bin/karma start karma.sauce.js # env: # SAUCE_USERNAME: ${{secrets.SAUCE_USERNAME}} # SAUCE_ACCESS_KEY: ${{secrets.SAUCE_ACCESS_KEY}} npm-publish: name: npm-publish if: ${{ github.repository == 'automerge/automerge' && github.ref == 'refs/heads/main' }} # needs: [ node-build, browsertest ] needs: [ node-build ] runs-on: ubuntu-latest steps: - name: Check out repo uses: actions/checkout@v2 - name: Use Node.js uses: actions/setup-node@v2 with: node-version: 16 - name: Install dependencies run: yarn install - name: npm publish if version has been updated uses: JS-DevTools/npm-publish@v1 with: token: ${{ secrets.NPM_AUTH_TOKEN }} check-version: true ================================================ FILE: .gitignore ================================================ /coverage /dist /node_modules .nyc_output .vscode ================================================ FILE: .mocharc.yaml ================================================ use_strict: true require: - ts-node/register - tsconfig-paths/register watch-files: - 'src/*.js' - 'frontend/*.js' - 'backend/*.js' - 'test/*.js' - 'test/*.ts' spec: - 'test/*test*.js' - 'test/*test*.ts' ================================================ FILE: @types/automerge/index.d.ts ================================================ declare module 'automerge' { /** * The return type of `Automerge.init()`, `Automerge.change()`, etc. where `T` is the * original type. It is a recursively frozen version of the original type. */ type Doc = FreezeObject type ChangeFn = (doc: T) => void // Automerge.* functions function init(options?: InitOptions): Doc function from(initialState: T | Doc, options?: InitOptions): Doc function clone(doc: Doc, options?: InitOptions): Doc function free(doc: Doc): void type InitOptions = | string // = actorId | { actorId?: string deferActorId?: boolean freeze?: boolean patchCallback?: PatchCallback observable?: Observable } type ChangeOptions = | string // = message | { message?: string time?: number patchCallback?: PatchCallback } type PatchCallback = (patch: Patch, before: T, after: T, local: boolean, changes: BinaryChange[]) => void type ObserverCallback = (diff: MapDiff | ListDiff | ValueDiff, before: T, after: T, local: boolean, changes: BinaryChange[]) => void class Observable { observe(object: T, callback: ObserverCallback): void } function merge(localdoc: Doc, remotedoc: Doc): Doc function change(doc: Doc, options: ChangeOptions, callback: ChangeFn): Doc function change(doc: Doc, callback: ChangeFn): Doc function emptyChange>(doc: D, options?: ChangeOptions): D function applyChanges(doc: Doc, changes: BinaryChange[]): [Doc, Patch] function equals(val1: T, val2: T): boolean function encodeChange(change: Change): BinaryChange function decodeChange(binaryChange: BinaryChange): Change function getActorId(doc: Doc): string function getAllChanges(doc: Doc): BinaryChange[] function getChanges(olddoc: Doc, newdoc: Doc): BinaryChange[] function getConflicts(doc: Doc, key: keyof T): any function getHistory(doc: Doc): State[] function getLastLocalChange(doc: Doc): BinaryChange function getObjectById(doc: Doc, objectId: OpId): any function getObjectId(object: any): OpId function load(data: BinaryDocument, options?: InitOptions): Doc function save(doc: Doc): BinaryDocument function generateSyncMessage(doc: Doc, syncState: SyncState): [SyncState, BinarySyncMessage?] function receiveSyncMessage(doc: Doc, syncState: SyncState, message: BinarySyncMessage): [Doc, SyncState, Patch?] function initSyncState(): SyncState // custom CRDT types class TableRow { readonly id: UUID } class Table { constructor() add(item: T): UUID byId(id: UUID): T & TableRow count: number ids: UUID[] remove(id: UUID): void rows: (T & TableRow)[] } class List extends Array { insertAt?(index: number, ...args: T[]): List deleteAt?(index: number, numDelete?: number): List } class Text extends List { constructor(text?: string | string[]) get(index: number): string toSpans(): (string | T)[] } // Note that until https://github.com/Microsoft/TypeScript/issues/2361 is addressed, we // can't treat a Counter like a literal number without force-casting it as a number. // This won't compile: // `assert.strictEqual(c + 10, 13) // Operator '+' cannot be applied to types 'Counter' and '10'.ts(2365)` // But this will: // `assert.strictEqual(c as unknown as number + 10, 13)` class Counter extends Number { constructor(value?: number) increment(delta?: number): void decrement(delta?: number): void toString(): string valueOf(): number value: number } class Int { constructor(value: number) } class Uint { constructor(value: number) } class Float64 { constructor(value: number) } // Readonly variants type ReadonlyTable = ReadonlyArray & Table type ReadonlyList = ReadonlyArray & List type ReadonlyText = ReadonlyList & Text // Front & back namespace Frontend { function applyPatch(doc: Doc, patch: Patch, backendState?: BackendState): Doc function change(doc: Doc, message: string | undefined, callback: ChangeFn): [Doc, Change] function change(doc: Doc, callback: ChangeFn): [Doc, Change] function emptyChange(doc: Doc, message?: string): [Doc, Change] function from(initialState: T | Doc, options?: InitOptions): [Doc, Change] function getActorId(doc: Doc): string function getBackendState(doc: Doc): BackendState function getConflicts(doc: Doc, key: keyof T): any function getElementIds(list: any): string[] function getLastLocalChange(doc: Doc): BinaryChange function getObjectById(doc: Doc, objectId: OpId): Doc function getObjectId(doc: Doc): OpId function init(options?: InitOptions): Doc function setActorId(doc: Doc, actorId: string): Doc } namespace Backend { function applyChanges(state: BackendState, changes: BinaryChange[]): [BackendState, Patch] function applyLocalChange(state: BackendState, change: Change): [BackendState, Patch, BinaryChange] function clone(state: BackendState): BackendState function free(state: BackendState): void function getAllChanges(state: BackendState): BinaryChange[] function getChangeByHash(state: BackendState, hash: Hash): BinaryChange function getChanges(state: BackendState, haveDeps: Hash[]): BinaryChange[] function getChangesAdded(state1: BackendState, state2: BackendState): BinaryChange[] function getHeads(state: BackendState): Hash[] function getMissingDeps(state: BackendState, heads?: Hash[]): Hash[] function getPatch(state: BackendState): Patch function init(): BackendState function load(data: BinaryDocument): BackendState function loadChanges(state: BackendState, changes: BinaryChange[]): BackendState function save(state: BackendState): BinaryDocument function generateSyncMessage(state: BackendState, syncState: SyncState): [SyncState, BinarySyncMessage?] function receiveSyncMessage(state: BackendState, syncState: SyncState, message: BinarySyncMessage): [BackendState, SyncState, Patch?] function encodeSyncMessage(message: SyncMessage): BinarySyncMessage function decodeSyncMessage(bytes: BinarySyncMessage): SyncMessage function initSyncState(): SyncState function encodeSyncState(syncState: SyncState): BinarySyncState function decodeSyncState(bytes: BinarySyncState): SyncState } // Internals type Hash = string // 64-digit hex string type OpId = string // of the form `${counter}@${actorId}` type UUID = string type UUIDGenerator = () => UUID interface UUIDFactory extends UUIDGenerator { setFactory: (generator: UUIDGenerator) => void reset: () => void } const uuid: UUIDFactory interface Clock { [actorId: string]: number } interface State { change: Change snapshot: T } interface BackendState { // no public methods or properties } type BinaryChange = Uint8Array & { __binaryChange: true } type BinaryDocument = Uint8Array & { __binaryDocument: true } type BinarySyncState = Uint8Array & { __binarySyncState: true } type BinarySyncMessage = Uint8Array & { __binarySyncMessage: true } interface SyncState { // no public methods or properties } interface SyncMessage { heads: Hash[] need: Hash[] have: SyncHave[] changes: BinaryChange[] } interface SyncHave { lastSync: Hash[] bloom: Uint8Array } interface Change { message: string actor: string time: number seq: number startOp: number hash?: Hash deps: Hash[] ops: Op[] } interface Op { action: OpAction obj: OpId key: string | number insert: boolean elemId?: OpId child?: OpId value?: number | boolean | string | null datatype?: DataType pred?: OpId[] values?: (number | boolean | string | null)[] multiOp?: number } interface Patch { actor?: string seq?: number pendingChanges: number clock: Clock deps: Hash[] diffs: MapDiff maxOp: number } // Describes changes to a map (in which case propName represents a key in the // map) or a table object (in which case propName is the primary key of a row). interface MapDiff { objectId: OpId // ID of object being updated type: 'map' | 'table' // type of object being updated // For each key/property that is changing, props contains one entry // (properties that are not changing are not listed). The nested object is // empty if the property is being deleted, contains one opId if it is set to // a single value, and contains multiple opIds if there is a conflict. props: {[propName: string]: {[opId: string]: MapDiff | ListDiff | ValueDiff }} } // Describes changes to a list or Automerge.Text object, in which each element // is identified by its index. interface ListDiff { objectId: OpId // ID of object being updated type: 'list' | 'text' // type of objct being updated // This array contains edits in the order they should be applied. edits: (SingleInsertEdit | MultiInsertEdit | UpdateEdit | RemoveEdit)[] } // Describes the insertion of a single element into a list or text object. // The element can be a nested object. interface SingleInsertEdit { action: 'insert' index: number // the list index at which to insert the new element elemId: OpId // the unique element ID of the new list element opId: OpId // ID of the operation that assigned this value value: MapDiff | ListDiff | ValueDiff } // Describes the insertion of a consecutive sequence of primitive values into // a list or text object. In the case of text, the values are strings (each // character as a separate string value). Each inserted value is given a // consecutive element ID: starting with `elemId` for the first value, the // subsequent values are given elemIds with the same actor ID and incrementing // counters. To insert non-primitive values, use SingleInsertEdit. interface MultiInsertEdit { action: 'multi-insert' index: number // the list index at which to insert the first value elemId: OpId // the unique ID of the first inserted element values: number[] | boolean[] | string[] | null[] // list of values to insert datatype?: DataType // all values must be of the same datatype } // Describes the update of the value or nested object at a particular index // of a list or text object. In the case where there are multiple conflicted // values at the same list index, multiple UpdateEdits with the same index // (but different opIds) appear in the edits array of ListDiff. interface UpdateEdit { action: 'update' index: number // the list index to update opId: OpId // ID of the operation that assigned this value value: MapDiff | ListDiff | ValueDiff } // Describes the deletion of one or more consecutive elements from a list or // text object. interface RemoveEdit { action: 'remove' index: number // index of the first list element to remove count: number // number of list elements to remove } // Describes a primitive value, optionally tagged with a datatype that // indicates how the value should be interpreted. interface ValueDiff { type: 'value' value: number | boolean | string | null datatype?: DataType } type OpAction = | 'del' | 'inc' | 'set' | 'link' | 'makeText' | 'makeTable' | 'makeList' | 'makeMap' type CollectionType = | 'list' //.. | 'map' | 'table' | 'text' type DataType = | 'int' | 'uint' | 'float64' | 'counter' | 'timestamp' // TYPE UTILITY FUNCTIONS // Type utility function: Freeze // Generates a readonly version of a given object, array, or map type applied recursively to the nested members of the root type. // It's like TypeScript's `readonly`, but goes all the way down a tree. // prettier-ignore type Freeze = T extends Function ? T : T extends Text ? ReadonlyText : T extends Table ? FreezeTable : T extends List ? FreezeList : T extends Array ? FreezeArray : T extends Map ? FreezeMap : T extends string & infer O ? string & O : FreezeObject interface FreezeTable extends ReadonlyTable> {} interface FreezeList extends ReadonlyList> {} interface FreezeArray extends ReadonlyArray> {} interface FreezeMap extends ReadonlyMap, Freeze> {} type FreezeObject = { readonly [P in keyof T]: Freeze } } ================================================ FILE: CHANGELOG.md ================================================ # Changelog Automerge adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html) for assigning version numbers. However, any feature that is not documented or labelled as "experimental" may change without warning in a minor release. All notable changes to Automerge will be documented in this file, which is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ## Performance branch (working towards Automerge 1.0 release candidate) - **Changed**: The data format for storing/transmitting changes (as returned by `Automerge.getChanges()`) and documents (as returned by `Automerge.save()`) have changed. They now use a binary encoding that is much more compact than the old JSON format. This is a breaking change, but we will provide an upgrade tool to migrate any existing Automerge docs to the new format. - **Changed**: `Automerge.applyChanges()` now returns a two-element array, where the first element is the updated document, and the second element is a patch describing the changes that have been made (including any conflicts that have arisen). This simplifies applications that need to update some other state, such as a user interface, to reflect changes in a document. - **Changed** [#339]: `Automerge.Connection`, `Automerge.DocSet`, and `Automerge.WatchableDoc` have been removed, and replaced with a new Automerge sync protocol that is implemented by the functions `Automerge.generateSyncMessage()` and `Automerge.receiveSyncMessage()`. ([@ept], [@pvh], [@orionz], [@alexjg], [@jeffa5]) - **Changed**: The frontend/backend protocol (change requests generated by `Frontend.change()` and patches returned by `Backend.applyChanges()`) has changed. The new format will be documented separately. - **Removed**: The undo/redo feature is removed for now. The original implementation was a bit of a hack, and we decided not to support the hack for 1.0. Instead, we will bring back a better-designed version of this feature in the future. - **Changed**: Actor IDs are now required to consist only of lowercase hexadecimal digits (UUIDs can still be used, but the hyphens need to be removed). - **Changed**: `Automerge.getConflicts()` now returns *all* conflicting values, including the value chosen as default resolution. - **Changed**: Multiple references to the same object in an Automerge document are no longer allowed. In other words, the document is now required to be a tree, not a DAG. - **Changed**: We no longer assume that the backend state is immutable, giving us greater freedom to implement the backend in a way that maximises performance. (Frontend state and Automerge documents remain immutable as before.) This restricts certain usage patterns: for example, if you update a document, you cannot then take a reference to the document state before that update and update or query it. If you want to be able to continue referencing an old document state, you can copy it using `Automerge.clone()`. - **Removed**: `Automerge.diff` and `Backend.merge`, because they depended on the assumption of an immutable backend. We hope to bring back a better implementation of `Automerge.diff` in the future. - **Changed**: Dependencies between changes are now expressed by referencing the hashes of dependencies, rather than their actorId and sequence number. APIs have changed accordingly: `Backend.getMissingDeps` now returns a list of hashes rather than a vector clock; the second argument of `Backend.getChanges()` is now also a list of hashes. This change also affects network sync protocols, e.g. based on `Automerge.Connection`, which need to now exchange hashes rather than vector clocks. - **Removed**: `Automerge.getMissingDeps()`; use `Backend.getMissingDeps()` instead. - **Removed**: `Backend.getMissingChanges()`; use `Backend.getChanges()` instead. - **Removed**: `Backend.getChangesForActor()` since it does not fit with a hash chaining approach. - **Added**: `Frontend.getLastLocalChange()` returns the binary encoding of the last change made by the local actor, and `Backend.getHeads()` returns the latest change hashes in the current document state. - **Changed**: `Backend.applyLocalChange()` now returns an array of three values: the updated backend state, the patch to apply to the frontend, and the binary encoding of the change. - **Added** [#308]: Experimental `Automerge.Observable` API allows an application to receive a callback whenever a document (or some object within a document) changes ([@ept]) ## [Unreleased] ## [0.14.2] — 2021-01-12 - **Fixed** [#301]: Handling out-of-bounds argument in `Array.slice()` ([@pierreprinetti]) - **Fixed** [#261]: Support calling `Array.indexOf()` with an object ([@philschatz]) ## [0.14.1] — 2020-05-25 - **Fixed** [#249]: Corrected TypeScript declaration for `Automerge.Table.rows` ([@lauritzsh]) - **Fixed** [#252]: Corrected TypeScript declaration for `WatchableDoc` ([@vincentcapicotto]) - **Fixed** [#258]: Changes whose dependencies are missing are now preserved when saving and reloading a document ([@KarenSarmiento], [@ept]) - **Changed** [#260]: If you try to assign an object that is already in an Automerge document, you now get a more descriptive error message ([@johannesjo], [@ept]) ## [0.14.0] — 2020-03-25 - **Removed** [#236]: Undocumented `Automerge.Table` API that allowed rows to be added by providing an array of values. Now rows must be given as an object ([@HerbCaudill]) - **Removed** [#241]: Constructor of `Automerge.Table` no longer takes an array of columns, and the `columns` property of `Automerge.Table` is also removed ([@ept]) - **Changed** [#242]: Rows of `Automerge.Table` now automatically get an `id` property containing the primary key of that row ([@ept]) - **Removed** [#243]: `Automerge.Table` objects no longer have a `set()` method. Use `add()` or `remove()` instead ([@ept]) - **Removed** support for Node 8, which is no longer being maintained - **Added** [#194], [#238]: `Automerge.Text` objects may now contain objects as well as strings; new method `Text.toSpans()` that concatenates characters while leaving objects unchanged ([@pvh], [@ept], [@nornagon]) ## [0.13.0] — 2020-02-24 - **Added** [#232]: New API `Automerge.getAllChanges()` returns all changes ([@ept]) - **Fixed** [#230]: `Text.deleteAt` allows zero characters to be deleted ([@skokenes]) - **Fixed** [#219]: `canUndo` is false immediately after `Automerge.from` ([@ept]) - **Fixed** [#215]: Adjust TypeScript definition of `Freeze` ([@jeffpeterson]) ## [0.12.1] — 2019-08-22 - **Fixed** [#184]: Corrected TypeScript type definition for `Automerge.DocSet` ([@HerbCaudill]) - **Fixed** [#174]: If `.filter()`, `.find()` or similar methods are used inside a change callback, the objects they return can now be mutated ([@ept], [@airhorns]) - **Fixed** [#199]: `Automerge.Text.toString()` now returns the unadulterated text ([@Gozala]) - **Added** [#210]: New method `DocSet.removeDoc()` ([@brentkeller]) ## [0.12.0] — 2019-08-07 - **Changed** [#183]: `Frontend.from()` now accepts initialization options ([@HerbCaudill], [@ept]) - **Changed** [#180]: Mutation methods on `Automerge.Text` are now available without having to assign the object to a document ([@ept]) - **Added** [#181]: Can now specify an initial value when creating `Automerge.Text` objects ([@Gozala], [@ept]) - **Fixed** [#202]: Stack overflow error when making large changes ([@HerbCaudill], [@ept]) ## [0.11.0] — 2019-07-13 - **Added** [#127]: New `Automerge.from` function creates a new document and initializes it with an initial state given as an argument ([@HerbCaudill], [@ept]) - **Added** [#155]: Type definitions now allow TypeScript applications to use Automerge with static type-checking ([@HerbCaudill], [@airhorns], [@aslakhellesoy], [@ept]) - **Changed** [#177]: Automerge documents are no longer made immutable with `Object.freeze` by default, due to the performance cost. Use the `{freeze: true}` option to continue using immutable objects. ([@izuchukwu], [@ept]) - **Fixed** [#165]: Undo/redo now work when using separate frontend and backend ([@ept]) ## [0.10.1] — 2019-05-17 - **Fixed** [#151]: Exception "Duplicate list element ID" after a list element was added and removed again in the same change callback ([@ept], [@minhhien1996]) - **Changed** [#163]: Calling `JSON.stringify` on an Automerge document containing `Automerge.Text`, `Automerge.Table` or `Automerge.Counter` now serializes those objects in a clean way, rather than dumping the object's internal properties ([@ept]) ## [0.10.0] — 2019-02-04 - **Added** [#29]: New `Automerge.Table` datatype provides an unordered collection of records, like a relational database ([@ept]) - **Added** [#139]: JavaScript Date objects are now supported in Automerge documents ([@ept]) - **Added** [#147]: New `Automerge.Counter` datatype provides a CRDT counter ([@ept]) - **Removed** [#148]: `Automerge.inspect` has been removed ([@ept]) - **Fixed** [#145]: Exception "Duplicate list element ID" after reloading document from disk ([@ept]) - **Changed** [#150]: Underscore-prefixed property names are now allowed in map objects; `doc.object._objectId` is now `Automerge.getObjectId(doc.object)`, `doc.object._conflicts.property` is now `Automerge.getConflicts(doc.object, 'property')`, and `doc._actorId` is now `Automerge.getActorId(doc)`. ([@ept]) ## [0.9.2] — 2018-11-05 - **Fixed** [#128]: Fixed crash when Text object was modified in the same change as another object ([@CGNonofr]) - **Fixed** [#129]: Prevent application of duplicate requests in `applyLocalChange()` ([@ept]) - **Changed** [#130]: Frontend API no longer uses `Frontend.getRequests()`; instead, frontend change functions now return request objects directly ([@ept]) ## [0.9.1] — 2018-09-27 - **Changed** [#126]: Backend no longer needs to know the actorId of the local node ([@ept]) - **Changed** [#126]: Frontend can now be initialized without actorId, as long as you call `setActorId` before you make the first change ([@ept]) - **Changed** [#120]: Undo and redo must now be initiated by the frontend, not the backend ([@ept]) - **Fixed** [#120]: Fixed bug that would cause sequence numbers to be reused in some concurrent executions ([@ept]) - **Fixed** [#125]: Exceptions now throw Error objects rather than plain strings ([@wincent]) ## [0.9.0] — 2018-09-18 - **Added** [#112]: Added `Automerge.undo()` and `Automerge.redo()` ([@ept]) - **Added** [#118]: Introduced new Frontend and Backend APIs, and refactored existing APIs to use them; this allows some of the work to be moved to a background thread, and provides better modularisation ([@ept]) - **Removed** Removed the experimental Immutable.js-compatible API (`Automerge.initImmutable()`), a casualty of the refactoring in [#118] ([@ept]) ## [0.8.0] — 2018-08-02 - **Added** [#106]: New `doc._get(UUID)` method allows looking up an object by its `_objectId` inside an `Automerge.change()` callback ([@mattkrick]) - **Added** [#109]: Export `OpSet.getMissingChanges` on the Automerge object ([@mattkrick]) - **Added** [#111]: New `Automerge.emptyChange()` allows a "change" record to be created without actually changing the document ([@ept]) - **Changed** [#110]: Require that the change message in `Automerge.change()` must be a string ([@ept]) - **Changed** [#111]: If `Automerge.change()` does not modify the document, the function now returns the original document object untouched ([@ept]) ## [0.7.11] — 2018-06-26 - **Fixed** [#97]: `delete` operator no longer throws an exception if the property doesn't exist ([@salzhrani], [@EthanRBrown]) - **Fixed** [#104]: Fix an error when loading the webpack-packaged version of Automerge in Node.js ([@ept]) ## [0.7.10] — 2018-06-12 - **Added** [#93]: Allow the UUID implementation to be replaced for testing purposes ([@kpruden]) - **Added** [#74]: Automerge.diff() now includes the path from the root to the modified object ([@ept]) ## [0.7.9] — 2018-05-25 - **Fixed** [#90]: Compatibility with Node 10 ([@aslakhellesoy]) ## [0.7.8] — 2018-05-15 - **Fixed** [#91]: Improve performance of changes that modify many list or map elements ([@ept]) ## [0.7.7] — 2018-04-24 - **Changed** [#87]: Remove babel-polyfill from transpiled library ([@EthanRBrown]) ## 0.7.4, 0.7.5, [0.7.6] — 2018-04-19 - Version bump to fix a build tooling issue ## [0.7.3] — 2018-04-19 - **Changed** [#85]: Publish Babel-transpiled code to npm to improve compatibility ([@EthanRBrown]) ## [0.7.2] — 2018-04-17 - **Changed** [#83]: Changed `_objectId` property on Automerge map objects to be non-enumerable ([@EthanRBrown], [@ept]) - **Changed** [#84]: Changed `_conflicts`, `_state`, and `_actorId` to be non-enumerable properties ([@ept]) - **Fixed** [#77]: Fixed exception when a list element is inserted and updated in the same change callback ([@mmcgrana], [@ept]) - **Fixed** [#78]: Better error message when trying to use an unsupported datatype ([@ept]) ## [0.7.1] — 2018-02-26 - **Fixed** [#69]: `Automerge.load` generates random actorId if none specified ([@saranrapjs]) - **Fixed** [#64]: `Automerge.applyChanges()` allows changes to be applied out-of-order ([@jimpick], [@ept]) ## [0.7.0] — 2018-01-15 - **Added** [#62]: Initial support for Immutable.js API compatibility (read-only for now) ([@ept], [@jeffpeterson]) - **Added** [#45]: Added experimental APIs `Automerge.getMissingDeps`, `Automerge.getChangesForActor`, and `Automerge.WatchableDoc` to support integration with dat hypercore ([@pvh], [@ept]) - **Added** [#46]: Automerge list objects now also have a `_conflicts` property that records concurrent assignments to the same list index, just like map objects have had all along ([@ept]) - **Changed** [#60]: `splice` in an `Automerge.change()` callback returns an array of deleted elements (to match behaviour of [`Array#splice`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/splice)). ([@aslakhellesoy]) - **Fixed** [#57]: Tests now work on operating systems with case-sensitive filesystems ([@mmmm1998]) ## [0.6.0] — 2017-12-13 - **Added** [#44]: New APIs `Automerge.getChanges` and `Automerge.applyChanges` to provide more flexibility for network protocol layer ([@ept]) - **Added** [#41]: New `Automerge.Text` datatype, which is more efficient than a list for character-by-character editing of text ([@ept]) - **Added** [#40]: Lists are now backed by a new indexed skip list data structure, which is faster ([@ept]) - **Changed** [#38]: To save memory, `Automerge.getHistory` now builds snapshots of past states only when requested, rather than remembering them by default ([@ept]) ## [0.5.0] — 2017-09-19 - **Added** [#37]: Added `Automerge.diff` to find the differences between to Automerge documents ([@ept]) - **Added** [#37]: Added support for incremental cache maintenance, bringing a 20x speedup for a 1,000-element list ([@ept]) - **Added** [#36]: Added `Automerge.Connection` and `Automerge.DocSet` classes to support peer-to-peer network protocols ([@ept], [@pvh]) - **Changed**: Renamed `Automerge.changeset` to `Automerge.change` ([@ept]) ## [0.4.3] — 2017-08-16 - **Fixed** [#34]: Fixed a bug that caused list elements to sometimes disappear ([@aslakhellesoy], [@ept]) - **Fixed** [#32]: Fixed a test failure in recent Node.js versions ([@aslakhellesoy]) ## [0.4.2] — 2017-06-29 - **Added**: Set up Karma to run tests in web browsers ([@ept]) - **Added**: Set up Webpack to produce bundled JavaScript file for web browsers ([@ept]) ## [0.4.1] — 2017-06-26 - **Changed**: `Automerge.getHistory` API now uses the object cache, which should be faster ([@ept]) ## [0.4.0] — 2017-06-23 - **Changed**: Automerge documents are now just regular JavaScript objects, and Proxy is used only within `Automerge.changeset` callbacks. Previously everything used Proxy. ([@ept]) - **Changed**: [#30]: Made `_objectId` an enumerable property, so that it is visible by default ([@ept]) - **Changed**: Support all standard JavaScript array methods and iterators on list proxy object ([@ept]) ## [0.3.0] — 2017-06-13 - First public release. [Unreleased]: https://github.com/automerge/automerge/compare/v0.14.2...HEAD [0.14.2]: https://github.com/automerge/automerge/compare/v0.14.1...v0.14.2 [0.14.1]: https://github.com/automerge/automerge/compare/v0.14.0...v0.14.1 [0.14.0]: https://github.com/automerge/automerge/compare/v0.13.1...v0.14.0 [0.13.0]: https://github.com/automerge/automerge/compare/v0.12.1...v0.13.0 [0.12.1]: https://github.com/automerge/automerge/compare/v0.12.0...v0.12.1 [0.12.0]: https://github.com/automerge/automerge/compare/v0.11.0...v0.12.0 [0.11.0]: https://github.com/automerge/automerge/compare/v0.10.1...v0.11.0 [0.10.1]: https://github.com/automerge/automerge/compare/v0.10.0...v0.10.1 [0.10.0]: https://github.com/automerge/automerge/compare/v0.9.2...v0.10.0 [0.9.2]: https://github.com/automerge/automerge/compare/v0.9.1...v0.9.2 [0.9.1]: https://github.com/automerge/automerge/compare/v0.9.0...v0.9.1 [0.9.0]: https://github.com/automerge/automerge/compare/v0.8.0...v0.9.0 [0.8.0]: https://github.com/automerge/automerge/compare/v0.7.11...v0.8.0 [0.7.11]: https://github.com/automerge/automerge/compare/v0.7.10...v0.7.11 [0.7.10]: https://github.com/automerge/automerge/compare/v0.7.9...v0.7.10 [0.7.9]: https://github.com/automerge/automerge/compare/v0.7.8...v0.7.9 [0.7.8]: https://github.com/automerge/automerge/compare/v0.7.7...v0.7.8 [0.7.7]: https://github.com/automerge/automerge/compare/v0.7.6...v0.7.7 [0.7.6]: https://github.com/automerge/automerge/compare/v0.7.3...v0.7.6 [0.7.3]: https://github.com/automerge/automerge/compare/v0.7.2...v0.7.3 [0.7.2]: https://github.com/automerge/automerge/compare/v0.7.1...v0.7.2 [0.7.1]: https://github.com/automerge/automerge/compare/v0.7.0...v0.7.1 [0.7.0]: https://github.com/automerge/automerge/compare/v0.6.0...v0.7.0 [0.6.0]: https://github.com/automerge/automerge/compare/v0.5.0...v0.6.0 [0.5.0]: https://github.com/automerge/automerge/compare/v0.4.3...v0.5.0 [0.4.3]: https://github.com/automerge/automerge/compare/v0.4.2...v0.4.3 [0.4.2]: https://github.com/automerge/automerge/compare/v0.4.1...v0.4.2 [0.4.1]: https://github.com/automerge/automerge/compare/v0.4.0...v0.4.2 [0.4.0]: https://github.com/automerge/automerge/compare/v0.3.0...v0.4.0 [0.3.0]: https://github.com/automerge/automerge/compare/v0.2.0...v0.3.0 [#339]: https://github.com/automerge/automerge/pull/339 [#308]: https://github.com/automerge/automerge/pull/308 [#301]: https://github.com/automerge/automerge/pull/301 [#261]: https://github.com/automerge/automerge/pull/261 [#260]: https://github.com/automerge/automerge/issues/260 [#258]: https://github.com/automerge/automerge/issues/258 [#252]: https://github.com/automerge/automerge/pull/252 [#249]: https://github.com/automerge/automerge/pull/249 [#243]: https://github.com/automerge/automerge/pull/243 [#242]: https://github.com/automerge/automerge/pull/242 [#241]: https://github.com/automerge/automerge/pull/241 [#238]: https://github.com/automerge/automerge/pull/238 [#236]: https://github.com/automerge/automerge/pull/236 [#232]: https://github.com/automerge/automerge/pull/232 [#230]: https://github.com/automerge/automerge/issues/230 [#219]: https://github.com/automerge/automerge/issues/219 [#210]: https://github.com/automerge/automerge/pull/210 [#202]: https://github.com/automerge/automerge/issues/202 [#199]: https://github.com/automerge/automerge/pull/199 [#194]: https://github.com/automerge/automerge/issues/194 [#184]: https://github.com/automerge/automerge/pull/184 [#183]: https://github.com/automerge/automerge/pull/183 [#181]: https://github.com/automerge/automerge/pull/181 [#180]: https://github.com/automerge/automerge/issues/180 [#177]: https://github.com/automerge/automerge/issues/177 [#174]: https://github.com/automerge/automerge/issues/174 [#165]: https://github.com/automerge/automerge/pull/165 [#163]: https://github.com/automerge/automerge/pull/163 [#155]: https://github.com/automerge/automerge/pull/155 [#151]: https://github.com/automerge/automerge/issues/151 [#150]: https://github.com/automerge/automerge/pull/150 [#148]: https://github.com/automerge/automerge/pull/148 [#147]: https://github.com/automerge/automerge/pull/147 [#145]: https://github.com/automerge/automerge/issues/145 [#139]: https://github.com/automerge/automerge/pull/139 [#130]: https://github.com/automerge/automerge/pull/130 [#129]: https://github.com/automerge/automerge/pull/129 [#128]: https://github.com/automerge/automerge/pull/128 [#127]: https://github.com/automerge/automerge/issues/127 [#126]: https://github.com/automerge/automerge/pull/126 [#125]: https://github.com/automerge/automerge/pull/125 [#120]: https://github.com/automerge/automerge/pull/120 [#118]: https://github.com/automerge/automerge/pull/118 [#112]: https://github.com/automerge/automerge/pull/112 [#111]: https://github.com/automerge/automerge/pull/111 [#110]: https://github.com/automerge/automerge/pull/110 [#109]: https://github.com/automerge/automerge/pull/109 [#106]: https://github.com/automerge/automerge/issues/106 [#104]: https://github.com/automerge/automerge/issues/104 [#97]: https://github.com/automerge/automerge/issues/97 [#93]: https://github.com/automerge/automerge/pull/93 [#91]: https://github.com/automerge/automerge/pull/91 [#90]: https://github.com/automerge/automerge/pull/90 [#87]: https://github.com/automerge/automerge/pull/87 [#85]: https://github.com/automerge/automerge/pull/85 [#84]: https://github.com/automerge/automerge/pull/84 [#83]: https://github.com/automerge/automerge/pull/83 [#78]: https://github.com/automerge/automerge/issues/78 [#77]: https://github.com/automerge/automerge/pull/77 [#74]: https://github.com/automerge/automerge/pull/74 [#69]: https://github.com/automerge/automerge/pull/69 [#64]: https://github.com/automerge/automerge/pull/64 [#62]: https://github.com/automerge/automerge/pull/62 [#60]: https://github.com/automerge/automerge/pull/60 [#57]: https://github.com/automerge/automerge/pull/57 [#46]: https://github.com/automerge/automerge/issues/46 [#45]: https://github.com/automerge/automerge/pull/45 [#44]: https://github.com/automerge/automerge/pull/44 [#41]: https://github.com/automerge/automerge/pull/41 [#40]: https://github.com/automerge/automerge/pull/40 [#38]: https://github.com/automerge/automerge/issues/38 [#37]: https://github.com/automerge/automerge/pull/37 [#36]: https://github.com/automerge/automerge/pull/36 [#34]: https://github.com/automerge/automerge/pull/34 [#32]: https://github.com/automerge/automerge/pull/32 [#30]: https://github.com/automerge/automerge/pull/30 [#29]: https://github.com/automerge/automerge/issues/29 [@airhorns]: https://github.com/airhorns [@alexjg]: https://github.com/alexjg [@aslakhellesoy]: https://github.com/aslakhellesoy [@brentkeller]: https://github.com/brentkeller [@CGNonofr]: https://github.com/CGNonofr [@EthanRBrown]: https://github.com/EthanRBrown [@Gozala]: https://github.com/Gozala [@HerbCaudill]: https://github.com/HerbCaudill [@izuchukwu]: https://github.com/izuchukwu [@jeffa5]: https://github.com/jeffa5 [@jeffpeterson]: https://github.com/jeffpeterson [@jimpick]: https://github.com/jimpick [@johannesjo]: https://github.com/johannesjo [@ept]: https://github.com/ept [@KarenSarmiento]: https://github.com/KarenSarmiento [@kpruden]: https://github.com/kpruden [@lauritzsh]: https://github.com/lauritzsh [@mattkrick]: https://github.com/mattkrick [@minhhien1996]: https://github.com/minhhien1996 [@mmcgrana]: https://github.com/mmcgrana [@mmmm1998]: https://github.com/mmmm1998 [@nornagon]: https://github.com/nornagon [@orionz]: https://github.com/orionz [@pierreprinetti]: https://github.com/pierreprinetti [@philschatz]: https://github.com/philschatz [@pvh]: https://github.com/pvh [@salzhrani]: https://github.com/salzhrani [@saranrapjs]: https://github.com/saranrapjs [@skokenes]: https://github.com/skokenes [@vincentcapicotto]: https://github.com/vincentcapicotto [@wincent]: https://github.com/wincent ================================================ FILE: LICENSE ================================================ Copyright (c) 2017-present Martin Kleppmann, Ink & Switch LLC, and the Automerge contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: README.md ================================================ Automerge logo ## Deprecation Notice Automerge now has a shiny new implementation at https://github.com/automerge/automerge. This repository is the original pure javascript implementation. All development effort has shifted to the new implementation which is written in Rust and so can easily be ported to other platforms. ## Original Readme 💬 [Join the Automerge Slack community](https://join.slack.com/t/automerge/shared_invite/zt-e4p3760n-kKh7r3KRH1YwwNfiZM8ktw) [![Build Status](https://github.com/automerge/automerge/actions/workflows/automerge-ci.yml/badge.svg)](https://github.com/automerge/automerge/actions/workflows/automerge-ci.yml) [![Browser Test Status](https://app.saucelabs.com/buildstatus/automerge)](https://app.saucelabs.com/open_sauce/user/automerge/builds) Automerge is a library of data structures for building collaborative applications in JavaScript. Please see [automerge.org](http://automerge.org/) for documentation. For a set of extensible examples in TypeScript, see [automerge-repo](https://github.com/automerge/automerge-repo) ## Setup If you're using npm, `npm install automerge`. If you're using yarn, `yarn add automerge`. Then you can import it with `require('automerge')` as in [the example below](#usage) (or `import * as Automerge from 'automerge'` if using ES2015 or TypeScript). Otherwise, clone this repository, and then you can use the following commands: - `yarn install` — installs dependencies. - `yarn test` — runs the test suite in Node. - `yarn run browsertest` — runs the test suite in web browsers. - `yarn build` — creates a bundled JS file `dist/automerge.js` for web browsers. It includes the dependencies and is set up so that you can load through a script tag. ## Meta Copyright 2017–2021, the Automerge contributors. Released under the terms of the MIT license (see `LICENSE`). ================================================ FILE: backend/backend.js ================================================ const { encodeChange } = require('./columnar') const { BackendDoc } = require('./new') const { backendState } = require('./util') /** * Returns an empty node state. */ function init() { return {state: new BackendDoc(), heads: []} } function clone(backend) { return {state: backendState(backend).clone(), heads: backend.heads} } function free(backend) { backend.state = null backend.frozen = true } /** * Applies a list of `changes` from remote nodes to the node state `backend`. * Returns a two-element array `[state, patch]` where `state` is the updated * node state, and `patch` describes the modifications that need to be made * to the document objects to reflect these changes. */ function applyChanges(backend, changes) { const state = backendState(backend) const patch = state.applyChanges(changes) backend.frozen = true return [{state, heads: state.heads}, patch] } function hashByActor(state, actorId, index) { if (state.hashesByActor[actorId] && state.hashesByActor[actorId][index]) { return state.hashesByActor[actorId][index] } if (!state.haveHashGraph) { state.computeHashGraph() if (state.hashesByActor[actorId] && state.hashesByActor[actorId][index]) { return state.hashesByActor[actorId][index] } } throw new RangeError(`Unknown change: actorId = ${actorId}, seq = ${index + 1}`) } /** * Takes a single change request `request` made by the local user, and applies * it to the node state `backend`. Returns a three-element array `[backend, patch, binaryChange]` * where `backend` is the updated node state,`patch` confirms the * modifications to the document objects, and `binaryChange` is a binary-encoded form of * the change submitted. */ function applyLocalChange(backend, change) { const state = backendState(backend) if (change.seq <= state.clock[change.actor] || 0) { throw new RangeError('Change request has already been applied') } // Add the local actor's last change hash to deps. We do this because when frontend // and backend are on separate threads, the frontend may fire off several local // changes in sequence before getting a response from the backend; since the binary // encoding and hashing is done by the backend, the frontend does not know the hash // of its own last change in this case. Rather than handle this situation as a // special case, we say that the frontend includes only specifies other actors' // deps in changes it generates, and the dependency from the local actor's last // change is always added here in the backend. // // Strictly speaking, we should check whether the local actor's last change is // indirectly reachable through a different actor's change; in that case, it is not // necessary to add this dependency. However, it doesn't do any harm either (only // using a few extra bytes of storage). if (change.seq > 1) { const lastHash = hashByActor(state, change.actor, change.seq - 2) if (!lastHash) { throw new RangeError(`Cannot find hash of localChange before seq=${change.seq}`) } let deps = {[lastHash]: true} for (let hash of change.deps) deps[hash] = true change.deps = Object.keys(deps).sort() } const binaryChange = encodeChange(change) const patch = state.applyChanges([binaryChange], true) backend.frozen = true // On the patch we send out, omit the last local change hash const lastHash = hashByActor(state, change.actor, change.seq - 1) patch.deps = patch.deps.filter(head => head !== lastHash) return [{state, heads: state.heads}, patch, binaryChange] } /** * Returns the state of the document serialised to an Uint8Array. */ function save(backend) { return backendState(backend).save() } /** * Loads the document and/or changes contained in an Uint8Array, and returns a * backend initialised with this state. */ function load(data) { const state = new BackendDoc(data) return {state, heads: state.heads} } /** * Applies a list of `changes` to the node state `backend`, and returns the updated * state with those changes incorporated. Unlike `applyChanges()`, this function * does not produce a patch describing the incremental modifications, making it * a little faster when loading a document from disk. When all the changes have * been loaded, you can use `getPatch()` to construct the latest document state. */ function loadChanges(backend, changes) { const state = backendState(backend) state.applyChanges(changes) backend.frozen = true return {state, heads: state.heads} } /** * Returns a patch that, when applied to an empty document, constructs the * document tree in the state described by the node state `backend`. */ function getPatch(backend) { return backendState(backend).getPatch() } /** * Returns an array of hashes of the current "head" changes (i.e. those changes * that no other change depends on). */ function getHeads(backend) { return backend.heads } /** * Returns the full history of changes that have been applied to a document. */ function getAllChanges(backend) { return getChanges(backend, []) } /** * Returns all changes that are newer than or concurrent to the changes * identified by the hashes in `haveDeps`. If `haveDeps` is an empty array, all * changes are returned. Throws an exception if any of the given hashes is unknown. */ function getChanges(backend, haveDeps) { if (!Array.isArray(haveDeps)) { throw new TypeError('Pass an array of hashes to Backend.getChanges()') } return backendState(backend).getChanges(haveDeps) } /** * Returns all changes that are present in `backend2` but not in `backend1`. * Intended for use in situations where the two backends are for different actors. * To get the changes added between an older and a newer document state of the same * actor, use `getChanges()` instead. `getChangesAdded()` throws an exception if * one of the backend states is frozen (i.e. if it is not the latest state of that * backend instance; this distinction matters when the backend is mutable). */ function getChangesAdded(backend1, backend2) { return backendState(backend2).getChangesAdded(backendState(backend1)) } /** * If the backend has applied a change with the given `hash` (given as a * hexadecimal string), returns that change (as a byte array). Returns undefined * if no change with that hash has been applied. A change with missing * dependencies does not count as having been applied. */ function getChangeByHash(backend, hash) { return backendState(backend).getChangeByHash(hash) } /** * Returns the hashes of any missing dependencies, i.e. where we have applied a * change that has a dependency on a change we have not seen. * * If the argument `heads` is given (an array of hexadecimal strings representing * hashes as returned by `getHeads()`), this function also ensures that all of * those hashes resolve to either a change that has been applied to the document, * or that has been enqueued for later application once missing dependencies have * arrived. Any missing heads hashes are included in the returned array. */ function getMissingDeps(backend, heads = []) { return backendState(backend).getMissingDeps(heads) } module.exports = { init, clone, free, applyChanges, applyLocalChange, save, load, loadChanges, getPatch, getHeads, getAllChanges, getChanges, getChangesAdded, getChangeByHash, getMissingDeps } ================================================ FILE: backend/columnar.js ================================================ const pako = require('pako') const { copyObject, parseOpId, equalBytes } = require('../src/common') const { utf8ToString, hexStringToBytes, bytesToHexString, Encoder, Decoder, RLEEncoder, RLEDecoder, DeltaEncoder, DeltaDecoder, BooleanEncoder, BooleanDecoder } = require('./encoding') // Maybe we should be using the platform's built-in hash implementation? // Node has the crypto module: https://nodejs.org/api/crypto.html and browsers have // https://developer.mozilla.org/en-US/docs/Web/API/SubtleCrypto/digest // However, the WebCrypto API is asynchronous (returns promises), which would // force all our APIs to become asynchronous as well, which would be annoying. // // I think on balance, it's safe enough to use a random library off npm: // - We only need one hash function (not a full suite of crypto algorithms); // - SHA256 is quite simple and has fairly few opportunities for subtle bugs // (compared to asymmetric cryptography anyway); // - It does not need a secure source of random bits and does not need to be // constant-time; // - I have reviewed the source code and it seems pretty reasonable. const { Hash } = require('fast-sha256') // These bytes don't mean anything, they were generated randomly const MAGIC_BYTES = new Uint8Array([0x85, 0x6f, 0x4a, 0x83]) const CHUNK_TYPE_DOCUMENT = 0 const CHUNK_TYPE_CHANGE = 1 const CHUNK_TYPE_DEFLATE = 2 // like CHUNK_TYPE_CHANGE but with DEFLATE compression // Minimum number of bytes in a value before we enable DEFLATE compression (there is no point // compressing very short values since compression may actually make them bigger) const DEFLATE_MIN_SIZE = 256 // The least-significant 3 bits of a columnId indicate its datatype const COLUMN_TYPE = { GROUP_CARD: 0, ACTOR_ID: 1, INT_RLE: 2, INT_DELTA: 3, BOOLEAN: 4, STRING_RLE: 5, VALUE_LEN: 6, VALUE_RAW: 7 } // The 4th-least-significant bit of a columnId is set if the column is DEFLATE-compressed const COLUMN_TYPE_DEFLATE = 8 // In the values in a column of type VALUE_LEN, the bottom four bits indicate the type of the value, // one of the following types in VALUE_TYPE. The higher bits indicate the length of the value in the // associated VALUE_RAW column (in bytes). const VALUE_TYPE = { NULL: 0, FALSE: 1, TRUE: 2, LEB128_UINT: 3, LEB128_INT: 4, IEEE754: 5, UTF8: 6, BYTES: 7, COUNTER: 8, TIMESTAMP: 9, MIN_UNKNOWN: 10, MAX_UNKNOWN: 15 } // make* actions must be at even-numbered indexes in this list const ACTIONS = ['makeMap', 'set', 'makeList', 'del', 'makeText', 'inc', 'makeTable', 'link'] const OBJECT_TYPE = {makeMap: 'map', makeList: 'list', makeText: 'text', makeTable: 'table'} const COMMON_COLUMNS = [ {columnName: 'objActor', columnId: 0 << 4 | COLUMN_TYPE.ACTOR_ID}, {columnName: 'objCtr', columnId: 0 << 4 | COLUMN_TYPE.INT_RLE}, {columnName: 'keyActor', columnId: 1 << 4 | COLUMN_TYPE.ACTOR_ID}, {columnName: 'keyCtr', columnId: 1 << 4 | COLUMN_TYPE.INT_DELTA}, {columnName: 'keyStr', columnId: 1 << 4 | COLUMN_TYPE.STRING_RLE}, {columnName: 'idActor', columnId: 2 << 4 | COLUMN_TYPE.ACTOR_ID}, {columnName: 'idCtr', columnId: 2 << 4 | COLUMN_TYPE.INT_DELTA}, {columnName: 'insert', columnId: 3 << 4 | COLUMN_TYPE.BOOLEAN}, {columnName: 'action', columnId: 4 << 4 | COLUMN_TYPE.INT_RLE}, {columnName: 'valLen', columnId: 5 << 4 | COLUMN_TYPE.VALUE_LEN}, {columnName: 'valRaw', columnId: 5 << 4 | COLUMN_TYPE.VALUE_RAW}, {columnName: 'chldActor', columnId: 6 << 4 | COLUMN_TYPE.ACTOR_ID}, {columnName: 'chldCtr', columnId: 6 << 4 | COLUMN_TYPE.INT_DELTA} ] const CHANGE_COLUMNS = COMMON_COLUMNS.concat([ {columnName: 'predNum', columnId: 7 << 4 | COLUMN_TYPE.GROUP_CARD}, {columnName: 'predActor', columnId: 7 << 4 | COLUMN_TYPE.ACTOR_ID}, {columnName: 'predCtr', columnId: 7 << 4 | COLUMN_TYPE.INT_DELTA} ]) const DOC_OPS_COLUMNS = COMMON_COLUMNS.concat([ {columnName: 'succNum', columnId: 8 << 4 | COLUMN_TYPE.GROUP_CARD}, {columnName: 'succActor', columnId: 8 << 4 | COLUMN_TYPE.ACTOR_ID}, {columnName: 'succCtr', columnId: 8 << 4 | COLUMN_TYPE.INT_DELTA} ]) const DOCUMENT_COLUMNS = [ {columnName: 'actor', columnId: 0 << 4 | COLUMN_TYPE.ACTOR_ID}, {columnName: 'seq', columnId: 0 << 4 | COLUMN_TYPE.INT_DELTA}, {columnName: 'maxOp', columnId: 1 << 4 | COLUMN_TYPE.INT_DELTA}, {columnName: 'time', columnId: 2 << 4 | COLUMN_TYPE.INT_DELTA}, {columnName: 'message', columnId: 3 << 4 | COLUMN_TYPE.STRING_RLE}, {columnName: 'depsNum', columnId: 4 << 4 | COLUMN_TYPE.GROUP_CARD}, {columnName: 'depsIndex', columnId: 4 << 4 | COLUMN_TYPE.INT_DELTA}, {columnName: 'extraLen', columnId: 5 << 4 | COLUMN_TYPE.VALUE_LEN}, {columnName: 'extraRaw', columnId: 5 << 4 | COLUMN_TYPE.VALUE_RAW} ] /** * Maps an opId of the form {counter: 12345, actorId: 'someActorId'} to the form * {counter: 12345, actorNum: 123, actorId: 'someActorId'}, where the actorNum * is the index into the `actorIds` array. */ function actorIdToActorNum(opId, actorIds) { if (!opId || !opId.actorId) return opId const counter = opId.counter const actorNum = actorIds.indexOf(opId.actorId) if (actorNum < 0) throw new RangeError('missing actorId') // should not happen return {counter, actorNum, actorId: opId.actorId} } /** * Comparison function to pass to Array.sort(), which compares two opIds in the * form produced by `actorIdToActorNum` so that they are sorted in increasing * Lamport timestamp order (sorted first by counter, then by actorId). */ function compareParsedOpIds(id1, id2) { if (id1.counter < id2.counter) return -1 if (id1.counter > id2.counter) return +1 if (id1.actorId < id2.actorId) return -1 if (id1.actorId > id2.actorId) return +1 return 0 } /** * Takes `changes`, an array of changes (represented as JS objects). Returns an * object `{changes, actorIds}`, where `changes` is a copy of the argument in * which all string opIds have been replaced with `{counter, actorNum}` objects, * and where `actorIds` is a lexicographically sorted array of actor IDs occurring * in any of the operations. `actorNum` is an index into that array of actorIds. * If `single` is true, the actorId of the author of the change is moved to the * beginning of the array of actorIds, so that `actorNum` is zero when referencing * the author of the change itself. This special-casing is omitted if `single` is * false. */ function parseAllOpIds(changes, single) { const actors = {}, newChanges = [] for (let change of changes) { change = copyObject(change) actors[change.actor] = true change.ops = expandMultiOps(change.ops, change.startOp, change.actor) change.ops = change.ops.map(op => { op = copyObject(op) if (op.obj !== '_root') op.obj = parseOpId(op.obj) if (op.elemId && op.elemId !== '_head') op.elemId = parseOpId(op.elemId) if (op.child) op.child = parseOpId(op.child) if (op.pred) op.pred = op.pred.map(parseOpId) if (op.obj.actorId) actors[op.obj.actorId] = true if (op.elemId && op.elemId.actorId) actors[op.elemId.actorId] = true if (op.child && op.child.actorId) actors[op.child.actorId] = true for (let pred of op.pred) actors[pred.actorId] = true return op }) newChanges.push(change) } let actorIds = Object.keys(actors).sort() if (single) { actorIds = [changes[0].actor].concat(actorIds.filter(actor => actor !== changes[0].actor)) } for (let change of newChanges) { change.actorNum = actorIds.indexOf(change.actor) for (let i = 0; i < change.ops.length; i++) { let op = change.ops[i] op.id = {counter: change.startOp + i, actorNum: change.actorNum, actorId: change.actor} op.obj = actorIdToActorNum(op.obj, actorIds) op.elemId = actorIdToActorNum(op.elemId, actorIds) op.child = actorIdToActorNum(op.child, actorIds) op.pred = op.pred.map(pred => actorIdToActorNum(pred, actorIds)) } } return {changes: newChanges, actorIds} } /** * Encodes the `obj` property of operation `op` into the two columns * `objActor` and `objCtr`. */ function encodeObjectId(op, columns) { if (op.obj === '_root') { columns.objActor.appendValue(null) columns.objCtr.appendValue(null) } else if (op.obj.actorNum >= 0 && op.obj.counter > 0) { columns.objActor.appendValue(op.obj.actorNum) columns.objCtr.appendValue(op.obj.counter) } else { throw new RangeError(`Unexpected objectId reference: ${JSON.stringify(op.obj)}`) } } /** * Encodes the `key` and `elemId` properties of operation `op` into the three * columns `keyActor`, `keyCtr`, and `keyStr`. */ function encodeOperationKey(op, columns) { if (op.key) { columns.keyActor.appendValue(null) columns.keyCtr.appendValue(null) columns.keyStr.appendValue(op.key) } else if (op.elemId === '_head' && op.insert) { columns.keyActor.appendValue(null) columns.keyCtr.appendValue(0) columns.keyStr.appendValue(null) } else if (op.elemId && op.elemId.actorNum >= 0 && op.elemId.counter > 0) { columns.keyActor.appendValue(op.elemId.actorNum) columns.keyCtr.appendValue(op.elemId.counter) columns.keyStr.appendValue(null) } else { throw new RangeError(`Unexpected operation key: ${JSON.stringify(op)}`) } } /** * Encodes the `action` property of operation `op` into the `action` column. */ function encodeOperationAction(op, columns) { const actionCode = ACTIONS.indexOf(op.action) if (actionCode >= 0) { columns.action.appendValue(actionCode) } else if (typeof op.action === 'number') { columns.action.appendValue(op.action) } else { throw new RangeError(`Unexpected operation action: ${op.action}`) } } /** * Given the datatype for a number, determine the typeTag and the value to encode * otherwise guess */ function getNumberTypeAndValue(op) { switch (op.datatype) { case "counter": return [ VALUE_TYPE.COUNTER, op.value ] case "timestamp": return [ VALUE_TYPE.TIMESTAMP, op.value ] case "uint": return [ VALUE_TYPE.LEB128_UINT, op.value ] case "int": return [ VALUE_TYPE.LEB128_INT, op.value ] case "float64": { const buf64 = new ArrayBuffer(8), view64 = new DataView(buf64) view64.setFloat64(0, op.value, true) return [ VALUE_TYPE.IEEE754, new Uint8Array(buf64) ] } default: // increment operators get resolved here ... if (Number.isInteger(op.value) && op.value <= Number.MAX_SAFE_INTEGER && op.value >= Number.MIN_SAFE_INTEGER) { return [ VALUE_TYPE.LEB128_INT, op.value ] } else { const buf64 = new ArrayBuffer(8), view64 = new DataView(buf64) view64.setFloat64(0, op.value, true) return [ VALUE_TYPE.IEEE754, new Uint8Array(buf64) ] } } } /** * Encodes the `value` property of operation `op` into the two columns * `valLen` and `valRaw`. */ function encodeValue(op, columns) { if ((op.action !== 'set' && op.action !== 'inc') || op.value === null) { columns.valLen.appendValue(VALUE_TYPE.NULL) } else if (op.value === false) { columns.valLen.appendValue(VALUE_TYPE.FALSE) } else if (op.value === true) { columns.valLen.appendValue(VALUE_TYPE.TRUE) } else if (typeof op.value === 'string') { const numBytes = columns.valRaw.appendRawString(op.value) columns.valLen.appendValue(numBytes << 4 | VALUE_TYPE.UTF8) } else if (ArrayBuffer.isView(op.value)) { const numBytes = columns.valRaw.appendRawBytes(new Uint8Array(op.value.buffer)) columns.valLen.appendValue(numBytes << 4 | VALUE_TYPE.BYTES) } else if (typeof op.value === 'number') { let [typeTag, value] = getNumberTypeAndValue(op) let numBytes if (typeTag === VALUE_TYPE.LEB128_UINT) { numBytes = columns.valRaw.appendUint53(value) } else if (typeTag === VALUE_TYPE.IEEE754) { numBytes = columns.valRaw.appendRawBytes(value) } else { numBytes = columns.valRaw.appendInt53(value) } columns.valLen.appendValue(numBytes << 4 | typeTag) } else if (typeof op.datatype === 'number' && op.datatype >= VALUE_TYPE.MIN_UNKNOWN && op.datatype <= VALUE_TYPE.MAX_UNKNOWN && op.value instanceof Uint8Array) { const numBytes = columns.valRaw.appendRawBytes(op.value) columns.valLen.appendValue(numBytes << 4 | op.datatype) } else if (op.datatype) { throw new RangeError(`Unknown datatype ${op.datatype} for value ${op.value}`) } else { throw new RangeError(`Unsupported value in operation: ${op.value}`) } } /** * Given `sizeTag` (an unsigned integer read from a VALUE_LEN column) and `bytes` (a Uint8Array * read from a VALUE_RAW column, with length `sizeTag >> 4`), this function returns an object of the * form `{value: value, datatype: datatypeTag}` where `value` is a JavaScript primitive datatype * corresponding to the value, and `datatypeTag` is a datatype annotation such as 'counter'. */ function decodeValue(sizeTag, bytes) { if (sizeTag === VALUE_TYPE.NULL) { return {value: null} } else if (sizeTag === VALUE_TYPE.FALSE) { return {value: false} } else if (sizeTag === VALUE_TYPE.TRUE) { return {value: true} } else if (sizeTag % 16 === VALUE_TYPE.UTF8) { return {value: utf8ToString(bytes)} } else { if (sizeTag % 16 === VALUE_TYPE.LEB128_UINT) { return {value: new Decoder(bytes).readUint53(), datatype: "uint"} } else if (sizeTag % 16 === VALUE_TYPE.LEB128_INT) { return {value: new Decoder(bytes).readInt53(), datatype: "int"} } else if (sizeTag % 16 === VALUE_TYPE.IEEE754) { const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength) if (bytes.byteLength === 8) { return {value: view.getFloat64(0, true), datatype: "float64"} } else { throw new RangeError(`Invalid length for floating point number: ${bytes.byteLength}`) } } else if (sizeTag % 16 === VALUE_TYPE.COUNTER) { return {value: new Decoder(bytes).readInt53(), datatype: 'counter'} } else if (sizeTag % 16 === VALUE_TYPE.TIMESTAMP) { return {value: new Decoder(bytes).readInt53(), datatype: 'timestamp'} } else { return {value: bytes, datatype: sizeTag % 16} } } } /** * Reads one value from the column `columns[colIndex]` and interprets it based * on the column type. `actorIds` is a list of actors that appear in the change; * `actorIds[0]` is the actorId of the change's author. Mutates the `result` * object with the value, and returns the number of columns processed (this is 2 * in the case of a pair of VALUE_LEN and VALUE_RAW columns, which are processed * in one go). */ function decodeValueColumns(columns, colIndex, actorIds, result) { const { columnId, columnName, decoder } = columns[colIndex] if (columnId % 8 === COLUMN_TYPE.VALUE_LEN && colIndex + 1 < columns.length && columns[colIndex + 1].columnId === columnId + 1) { const sizeTag = decoder.readValue() const rawValue = columns[colIndex + 1].decoder.readRawBytes(sizeTag >> 4) const { value, datatype } = decodeValue(sizeTag, rawValue) result[columnName] = value if (datatype) result[columnName + '_datatype'] = datatype return 2 } else if (columnId % 8 === COLUMN_TYPE.ACTOR_ID) { const actorNum = decoder.readValue() if (actorNum === null) { result[columnName] = null } else { if (!actorIds[actorNum]) throw new RangeError(`No actor index ${actorNum}`) result[columnName] = actorIds[actorNum] } } else { result[columnName] = decoder.readValue() } return 1 } /** * Encodes an array of operations in a set of columns. The operations need to * be parsed with `parseAllOpIds()` beforehand. If `forDocument` is true, we use * the column structure of a whole document, otherwise we use the column * structure for an individual change. Returns an array of * `{columnId, columnName, encoder}` objects. */ function encodeOps(ops, forDocument) { const columns = { objActor : new RLEEncoder('uint'), objCtr : new RLEEncoder('uint'), keyActor : new RLEEncoder('uint'), keyCtr : new DeltaEncoder(), keyStr : new RLEEncoder('utf8'), insert : new BooleanEncoder(), action : new RLEEncoder('uint'), valLen : new RLEEncoder('uint'), valRaw : new Encoder(), chldActor : new RLEEncoder('uint'), chldCtr : new DeltaEncoder() } if (forDocument) { columns.idActor = new RLEEncoder('uint') columns.idCtr = new DeltaEncoder() columns.succNum = new RLEEncoder('uint') columns.succActor = new RLEEncoder('uint') columns.succCtr = new DeltaEncoder() } else { columns.predNum = new RLEEncoder('uint') columns.predCtr = new DeltaEncoder() columns.predActor = new RLEEncoder('uint') } for (let op of ops) { encodeObjectId(op, columns) encodeOperationKey(op, columns) columns.insert.appendValue(!!op.insert) encodeOperationAction(op, columns) encodeValue(op, columns) if (op.child && op.child.counter) { columns.chldActor.appendValue(op.child.actorNum) columns.chldCtr.appendValue(op.child.counter) } else { columns.chldActor.appendValue(null) columns.chldCtr.appendValue(null) } if (forDocument) { columns.idActor.appendValue(op.id.actorNum) columns.idCtr.appendValue(op.id.counter) columns.succNum.appendValue(op.succ.length) op.succ.sort(compareParsedOpIds) for (let i = 0; i < op.succ.length; i++) { columns.succActor.appendValue(op.succ[i].actorNum) columns.succCtr.appendValue(op.succ[i].counter) } } else { columns.predNum.appendValue(op.pred.length) op.pred.sort(compareParsedOpIds) for (let i = 0; i < op.pred.length; i++) { columns.predActor.appendValue(op.pred[i].actorNum) columns.predCtr.appendValue(op.pred[i].counter) } } } let columnList = [] for (let {columnName, columnId} of forDocument ? DOC_OPS_COLUMNS : CHANGE_COLUMNS) { if (columns[columnName]) columnList.push({columnId, columnName, encoder: columns[columnName]}) } return columnList.sort((a, b) => a.columnId - b.columnId) } function validDatatype(value, datatype) { if (datatype === undefined) { return (typeof value === 'string' || typeof value === 'boolean' || value === null) } else { return typeof value === 'number' } } function expandMultiOps(ops, startOp, actor) { let opNum = startOp let expandedOps = [] for (const op of ops) { if (op.action === 'set' && op.values && op.insert) { if (op.pred.length !== 0) throw new RangeError('multi-insert pred must be empty') let lastElemId = op.elemId const datatype = op.datatype for (const value of op.values) { if (!validDatatype(value, datatype)) throw new RangeError(`Decode failed: bad value/datatype association (${value},${datatype})`) expandedOps.push({action: 'set', obj: op.obj, elemId: lastElemId, datatype, value, pred: [], insert: true}) lastElemId = `${opNum}@${actor}` opNum += 1 } } else if (op.action === 'del' && op.multiOp > 1) { if (op.pred.length !== 1) throw new RangeError('multiOp deletion must have exactly one pred') const startElemId = parseOpId(op.elemId), startPred = parseOpId(op.pred[0]) for (let i = 0; i < op.multiOp; i++) { const elemId = `${startElemId.counter + i}@${startElemId.actorId}` const pred = [`${startPred.counter + i}@${startPred.actorId}`] expandedOps.push({action: 'del', obj: op.obj, elemId, pred}) opNum += 1 } } else { expandedOps.push(op) opNum += 1 } } return expandedOps } /** * Takes a change as decoded by `decodeColumns`, and changes it into the form * expected by the rest of the backend. If `forDocument` is true, we use the op * structure of a whole document, otherwise we use the op structure for an * individual change. */ function decodeOps(ops, forDocument) { const newOps = [] for (let op of ops) { const obj = (op.objCtr === null) ? '_root' : `${op.objCtr}@${op.objActor}` const elemId = op.keyStr ? undefined : (op.keyCtr === 0 ? '_head' : `${op.keyCtr}@${op.keyActor}`) const action = ACTIONS[op.action] || op.action const newOp = elemId ? {obj, elemId, action} : {obj, key: op.keyStr, action} newOp.insert = !!op.insert if (ACTIONS[op.action] === 'set' || ACTIONS[op.action] === 'inc') { newOp.value = op.valLen if (op.valLen_datatype) newOp.datatype = op.valLen_datatype } if (!!op.chldCtr !== !!op.chldActor) { throw new RangeError(`Mismatched child columns: ${op.chldCtr} and ${op.chldActor}`) } if (op.chldCtr !== null) newOp.child = `${op.chldCtr}@${op.chldActor}` if (forDocument) { newOp.id = `${op.idCtr}@${op.idActor}` newOp.succ = op.succNum.map(succ => `${succ.succCtr}@${succ.succActor}`) checkSortedOpIds(op.succNum.map(succ => ({counter: succ.succCtr, actorId: succ.succActor}))) } else { newOp.pred = op.predNum.map(pred => `${pred.predCtr}@${pred.predActor}`) checkSortedOpIds(op.predNum.map(pred => ({counter: pred.predCtr, actorId: pred.predActor}))) } newOps.push(newOp) } return newOps } /** * Throws an exception if the opIds in the given array are not in sorted order. */ function checkSortedOpIds(opIds) { let last = null for (let opId of opIds) { if (last && compareParsedOpIds(last, opId) !== -1) { throw new RangeError('operation IDs are not in ascending order') } last = opId } } function encoderByColumnId(columnId) { if ((columnId & 7) === COLUMN_TYPE.INT_DELTA) { return new DeltaEncoder() } else if ((columnId & 7) === COLUMN_TYPE.BOOLEAN) { return new BooleanEncoder() } else if ((columnId & 7) === COLUMN_TYPE.STRING_RLE) { return new RLEEncoder('utf8') } else if ((columnId & 7) === COLUMN_TYPE.VALUE_RAW) { return new Encoder() } else { return new RLEEncoder('uint') } } function decoderByColumnId(columnId, buffer) { if ((columnId & 7) === COLUMN_TYPE.INT_DELTA) { return new DeltaDecoder(buffer) } else if ((columnId & 7) === COLUMN_TYPE.BOOLEAN) { return new BooleanDecoder(buffer) } else if ((columnId & 7) === COLUMN_TYPE.STRING_RLE) { return new RLEDecoder('utf8', buffer) } else if ((columnId & 7) === COLUMN_TYPE.VALUE_RAW) { return new Decoder(buffer) } else { return new RLEDecoder('uint', buffer) } } function makeDecoders(columns, columnSpec) { const emptyBuf = new Uint8Array(0) let decoders = [], columnIndex = 0, specIndex = 0 while (columnIndex < columns.length || specIndex < columnSpec.length) { if (columnIndex === columns.length || (specIndex < columnSpec.length && columnSpec[specIndex].columnId < columns[columnIndex].columnId)) { const {columnId, columnName} = columnSpec[specIndex] decoders.push({columnId, columnName, decoder: decoderByColumnId(columnId, emptyBuf)}) specIndex++ } else if (specIndex === columnSpec.length || columns[columnIndex].columnId < columnSpec[specIndex].columnId) { const {columnId, buffer} = columns[columnIndex] decoders.push({columnId, decoder: decoderByColumnId(columnId, buffer)}) columnIndex++ } else { // columns[columnIndex].columnId === columnSpec[specIndex].columnId const {columnId, buffer} = columns[columnIndex], {columnName} = columnSpec[specIndex] decoders.push({columnId, columnName, decoder: decoderByColumnId(columnId, buffer)}) columnIndex++ specIndex++ } } return decoders } function decodeColumns(columns, actorIds, columnSpec) { columns = makeDecoders(columns, columnSpec) let parsedRows = [] while (columns.some(col => !col.decoder.done)) { let row = {}, col = 0 while (col < columns.length) { const columnId = columns[col].columnId let groupId = columnId >> 4, groupCols = 1 while (col + groupCols < columns.length && columns[col + groupCols].columnId >> 4 === groupId) { groupCols++ } if (columnId % 8 === COLUMN_TYPE.GROUP_CARD) { const values = [], count = columns[col].decoder.readValue() for (let i = 0; i < count; i++) { let value = {} for (let colOffset = 1; colOffset < groupCols; colOffset++) { decodeValueColumns(columns, col + colOffset, actorIds, value) } values.push(value) } row[columns[col].columnName] = values col += groupCols } else { col += decodeValueColumns(columns, col, actorIds, row) } } parsedRows.push(row) } return parsedRows } function decodeColumnInfo(decoder) { // A number that is all 1 bits except for the bit that indicates whether a column is // deflate-compressed. We ignore this bit when checking whether columns are sorted by ID. const COLUMN_ID_MASK = (-1 ^ COLUMN_TYPE_DEFLATE) >>> 0 let lastColumnId = -1, columns = [], numColumns = decoder.readUint53() for (let i = 0; i < numColumns; i++) { const columnId = decoder.readUint53(), bufferLen = decoder.readUint53() if ((columnId & COLUMN_ID_MASK) <= (lastColumnId & COLUMN_ID_MASK)) { throw new RangeError('Columns must be in ascending order') } lastColumnId = columnId columns.push({columnId, bufferLen}) } return columns } function encodeColumnInfo(encoder, columns) { const nonEmptyColumns = columns.filter(column => column.encoder.buffer.byteLength > 0) encoder.appendUint53(nonEmptyColumns.length) for (let column of nonEmptyColumns) { encoder.appendUint53(column.columnId) encoder.appendUint53(column.encoder.buffer.byteLength) } } function decodeChangeHeader(decoder) { const numDeps = decoder.readUint53(), deps = [] for (let i = 0; i < numDeps; i++) { deps.push(bytesToHexString(decoder.readRawBytes(32))) } let change = { actor: decoder.readHexString(), seq: decoder.readUint53(), startOp: decoder.readUint53(), time: decoder.readInt53(), message: decoder.readPrefixedString(), deps } const actorIds = [change.actor], numActorIds = decoder.readUint53() for (let i = 0; i < numActorIds; i++) actorIds.push(decoder.readHexString()) change.actorIds = actorIds return change } /** * Assembles a chunk of encoded data containing a checksum, headers, and a * series of encoded columns. Calls `encodeHeaderCallback` with an encoder that * should be used to add the headers. The columns should be given as `columns`. */ function encodeContainer(chunkType, encodeContentsCallback) { const CHECKSUM_SIZE = 4 // checksum is first 4 bytes of SHA-256 hash of the rest of the data const HEADER_SPACE = MAGIC_BYTES.byteLength + CHECKSUM_SIZE + 1 + 5 // 1 byte type + 5 bytes length const body = new Encoder() // Make space for the header at the beginning of the body buffer. We will // copy the header in here later. This is cheaper than copying the body since // the body is likely to be much larger than the header. body.appendRawBytes(new Uint8Array(HEADER_SPACE)) encodeContentsCallback(body) const bodyBuf = body.buffer const header = new Encoder() header.appendByte(chunkType) header.appendUint53(bodyBuf.byteLength - HEADER_SPACE) // Compute the hash over chunkType, length, and body const headerBuf = header.buffer const sha256 = new Hash() sha256.update(headerBuf) sha256.update(bodyBuf.subarray(HEADER_SPACE)) const hash = sha256.digest(), checksum = hash.subarray(0, CHECKSUM_SIZE) // Copy header into the body buffer so that they are contiguous bodyBuf.set(MAGIC_BYTES, HEADER_SPACE - headerBuf.byteLength - CHECKSUM_SIZE - MAGIC_BYTES.byteLength) bodyBuf.set(checksum, HEADER_SPACE - headerBuf.byteLength - CHECKSUM_SIZE) bodyBuf.set(headerBuf, HEADER_SPACE - headerBuf.byteLength) return {hash, bytes: bodyBuf.subarray(HEADER_SPACE - headerBuf.byteLength - CHECKSUM_SIZE - MAGIC_BYTES.byteLength)} } function decodeContainerHeader(decoder, computeHash) { if (!equalBytes(decoder.readRawBytes(MAGIC_BYTES.byteLength), MAGIC_BYTES)) { throw new RangeError('Data does not begin with magic bytes 85 6f 4a 83') } const expectedHash = decoder.readRawBytes(4) const hashStartOffset = decoder.offset const chunkType = decoder.readByte() const chunkLength = decoder.readUint53() const header = {chunkType, chunkLength, chunkData: decoder.readRawBytes(chunkLength)} if (computeHash) { const sha256 = new Hash() sha256.update(decoder.buf.subarray(hashStartOffset, decoder.offset)) const binaryHash = sha256.digest() if (!equalBytes(binaryHash.subarray(0, 4), expectedHash)) { throw new RangeError('checksum does not match data') } header.hash = bytesToHexString(binaryHash) } return header } function encodeChange(changeObj) { const { changes, actorIds } = parseAllOpIds([changeObj], true) const change = changes[0] const { hash, bytes } = encodeContainer(CHUNK_TYPE_CHANGE, encoder => { if (!Array.isArray(change.deps)) throw new TypeError('deps is not an array') encoder.appendUint53(change.deps.length) for (let hash of change.deps.slice().sort()) { encoder.appendRawBytes(hexStringToBytes(hash)) } encoder.appendHexString(change.actor) encoder.appendUint53(change.seq) encoder.appendUint53(change.startOp) encoder.appendInt53(change.time) encoder.appendPrefixedString(change.message || '') encoder.appendUint53(actorIds.length - 1) for (let actor of actorIds.slice(1)) encoder.appendHexString(actor) const columns = encodeOps(change.ops, false) encodeColumnInfo(encoder, columns) for (let column of columns) encoder.appendRawBytes(column.encoder.buffer) if (change.extraBytes) encoder.appendRawBytes(change.extraBytes) }) const hexHash = bytesToHexString(hash) if (changeObj.hash && changeObj.hash !== hexHash) { throw new RangeError(`Change hash does not match encoding: ${changeObj.hash} != ${hexHash}`) } return (bytes.byteLength >= DEFLATE_MIN_SIZE) ? deflateChange(bytes) : bytes } function decodeChangeColumns(buffer) { if (buffer[8] === CHUNK_TYPE_DEFLATE) buffer = inflateChange(buffer) const decoder = new Decoder(buffer) const header = decodeContainerHeader(decoder, true) const chunkDecoder = new Decoder(header.chunkData) if (!decoder.done) throw new RangeError('Encoded change has trailing data') if (header.chunkType !== CHUNK_TYPE_CHANGE) throw new RangeError(`Unexpected chunk type: ${header.chunkType}`) const change = decodeChangeHeader(chunkDecoder) const columns = decodeColumnInfo(chunkDecoder) for (let i = 0; i < columns.length; i++) { if ((columns[i].columnId & COLUMN_TYPE_DEFLATE) !== 0) { throw new RangeError('change must not contain deflated columns') } columns[i].buffer = chunkDecoder.readRawBytes(columns[i].bufferLen) } if (!chunkDecoder.done) { const restLen = chunkDecoder.buf.byteLength - chunkDecoder.offset change.extraBytes = chunkDecoder.readRawBytes(restLen) } change.columns = columns change.hash = header.hash return change } /** * Decodes one change in binary format into its JS object representation. */ function decodeChange(buffer) { const change = decodeChangeColumns(buffer) change.ops = decodeOps(decodeColumns(change.columns, change.actorIds, CHANGE_COLUMNS), false) delete change.actorIds delete change.columns return change } /** * Decodes the header fields of a change in binary format, but does not decode * the operations. Saves work when we only need to inspect the headers. Only * computes the hash of the change if `computeHash` is true. */ function decodeChangeMeta(buffer, computeHash) { if (buffer[8] === CHUNK_TYPE_DEFLATE) buffer = inflateChange(buffer) const header = decodeContainerHeader(new Decoder(buffer), computeHash) if (header.chunkType !== CHUNK_TYPE_CHANGE) { throw new RangeError('Buffer chunk type is not a change') } const meta = decodeChangeHeader(new Decoder(header.chunkData)) meta.change = buffer if (computeHash) meta.hash = header.hash return meta } /** * Compresses a binary change using DEFLATE. */ function deflateChange(buffer) { const header = decodeContainerHeader(new Decoder(buffer), false) if (header.chunkType !== CHUNK_TYPE_CHANGE) throw new RangeError(`Unexpected chunk type: ${header.chunkType}`) const compressed = pako.deflateRaw(header.chunkData) const encoder = new Encoder() encoder.appendRawBytes(buffer.subarray(0, 8)) // copy MAGIC_BYTES and checksum encoder.appendByte(CHUNK_TYPE_DEFLATE) encoder.appendUint53(compressed.byteLength) encoder.appendRawBytes(compressed) return encoder.buffer } /** * Decompresses a binary change that has been compressed with DEFLATE. */ function inflateChange(buffer) { const header = decodeContainerHeader(new Decoder(buffer), false) if (header.chunkType !== CHUNK_TYPE_DEFLATE) throw new RangeError(`Unexpected chunk type: ${header.chunkType}`) const decompressed = pako.inflateRaw(header.chunkData) const encoder = new Encoder() encoder.appendRawBytes(buffer.subarray(0, 8)) // copy MAGIC_BYTES and checksum encoder.appendByte(CHUNK_TYPE_CHANGE) encoder.appendUint53(decompressed.byteLength) encoder.appendRawBytes(decompressed) return encoder.buffer } /** * Takes an Uint8Array that may contain multiple concatenated changes, and * returns an array of subarrays, each subarray containing one change. */ function splitContainers(buffer) { let decoder = new Decoder(buffer), chunks = [], startOffset = 0 while (!decoder.done) { decodeContainerHeader(decoder, false) chunks.push(buffer.subarray(startOffset, decoder.offset)) startOffset = decoder.offset } return chunks } /** * Decodes a list of changes from the binary format into JS objects. * `binaryChanges` is an array of `Uint8Array` objects. */ function decodeChanges(binaryChanges) { let decoded = [] for (let binaryChange of binaryChanges) { for (let chunk of splitContainers(binaryChange)) { if (chunk[8] === CHUNK_TYPE_DOCUMENT) { decoded = decoded.concat(decodeDocument(chunk)) } else if (chunk[8] === CHUNK_TYPE_CHANGE || chunk[8] === CHUNK_TYPE_DEFLATE) { decoded.push(decodeChange(chunk)) } else { // ignoring chunk of unknown type } } } return decoded } function sortOpIds(a, b) { if (a === b) return 0 if (a === '_root') return -1 if (b === '_root') return +1 const a_ = parseOpId(a), b_ = parseOpId(b) if (a_.counter < b_.counter) return -1 if (a_.counter > b_.counter) return +1 if (a_.actorId < b_.actorId) return -1 if (a_.actorId > b_.actorId) return +1 return 0 } /** * Takes a set of operations `ops` loaded from an encoded document, and * reconstructs the changes that they originally came from. * Does not return anything, only mutates `changes`. */ function groupChangeOps(changes, ops) { let changesByActor = {} // map from actorId to array of changes by that actor for (let change of changes) { change.ops = [] if (!changesByActor[change.actor]) changesByActor[change.actor] = [] if (change.seq !== changesByActor[change.actor].length + 1) { throw new RangeError(`Expected seq = ${changesByActor[change.actor].length + 1}, got ${change.seq}`) } if (change.seq > 1 && changesByActor[change.actor][change.seq - 2].maxOp > change.maxOp) { throw new RangeError('maxOp must increase monotonically per actor') } changesByActor[change.actor].push(change) } let opsById = {} for (let op of ops) { if (op.action === 'del') throw new RangeError('document should not contain del operations') op.pred = opsById[op.id] ? opsById[op.id].pred : [] opsById[op.id] = op for (let succ of op.succ) { if (!opsById[succ]) { if (op.elemId) { const elemId = op.insert ? op.id : op.elemId opsById[succ] = {id: succ, action: 'del', obj: op.obj, elemId, pred: []} } else { opsById[succ] = {id: succ, action: 'del', obj: op.obj, key: op.key, pred: []} } } opsById[succ].pred.push(op.id) } delete op.succ } for (let op of Object.values(opsById)) { if (op.action === 'del') ops.push(op) } for (let op of ops) { const { counter, actorId } = parseOpId(op.id) const actorChanges = changesByActor[actorId] // Binary search to find the change that should contain this operation let left = 0, right = actorChanges.length while (left < right) { const index = Math.floor((left + right) / 2) if (actorChanges[index].maxOp < counter) { left = index + 1 } else { right = index } } if (left >= actorChanges.length) { throw new RangeError(`Operation ID ${op.id} outside of allowed range`) } actorChanges[left].ops.push(op) } for (let change of changes) { change.ops.sort((op1, op2) => sortOpIds(op1.id, op2.id)) change.startOp = change.maxOp - change.ops.length + 1 delete change.maxOp for (let i = 0; i < change.ops.length; i++) { const op = change.ops[i], expectedId = `${change.startOp + i}@${change.actor}` if (op.id !== expectedId) { throw new RangeError(`Expected opId ${expectedId}, got ${op.id}`) } delete op.id } } } function decodeDocumentChanges(changes, expectedHeads) { let heads = {} // change hashes that are not a dependency of any other change for (let i = 0; i < changes.length; i++) { let change = changes[i] change.deps = [] for (let index of change.depsNum.map(d => d.depsIndex)) { if (!changes[index] || !changes[index].hash) { throw new RangeError(`No hash for index ${index} while processing index ${i}`) } const hash = changes[index].hash change.deps.push(hash) if (heads[hash]) delete heads[hash] } change.deps.sort() delete change.depsNum if (change.extraLen_datatype !== VALUE_TYPE.BYTES) { throw new RangeError(`Bad datatype for extra bytes: ${VALUE_TYPE.BYTES}`) } change.extraBytes = change.extraLen delete change.extraLen_datatype // Encoding and decoding again to compute the hash of the change changes[i] = decodeChange(encodeChange(change)) heads[changes[i].hash] = true } const actualHeads = Object.keys(heads).sort() let headsEqual = (actualHeads.length === expectedHeads.length), i = 0 while (headsEqual && i < actualHeads.length) { headsEqual = (actualHeads[i] === expectedHeads[i]) i++ } if (!headsEqual) { throw new RangeError(`Mismatched heads hashes: expected ${expectedHeads.join(', ')}, got ${actualHeads.join(', ')}`) } } function encodeDocumentHeader(doc) { const { changesColumns, opsColumns, actorIds, heads, headsIndexes, extraBytes } = doc for (let column of changesColumns) deflateColumn(column) for (let column of opsColumns) deflateColumn(column) return encodeContainer(CHUNK_TYPE_DOCUMENT, encoder => { encoder.appendUint53(actorIds.length) for (let actor of actorIds) { encoder.appendHexString(actor) } encoder.appendUint53(heads.length) for (let head of heads.sort()) { encoder.appendRawBytes(hexStringToBytes(head)) } encodeColumnInfo(encoder, changesColumns) encodeColumnInfo(encoder, opsColumns) for (let column of changesColumns) encoder.appendRawBytes(column.encoder.buffer) for (let column of opsColumns) encoder.appendRawBytes(column.encoder.buffer) for (let index of headsIndexes) encoder.appendUint53(index) if (extraBytes) encoder.appendRawBytes(extraBytes) }).bytes } function decodeDocumentHeader(buffer) { const documentDecoder = new Decoder(buffer) const header = decodeContainerHeader(documentDecoder, true) const decoder = new Decoder(header.chunkData) if (!documentDecoder.done) throw new RangeError('Encoded document has trailing data') if (header.chunkType !== CHUNK_TYPE_DOCUMENT) throw new RangeError(`Unexpected chunk type: ${header.chunkType}`) const actorIds = [], numActors = decoder.readUint53() for (let i = 0; i < numActors; i++) { actorIds.push(decoder.readHexString()) } const heads = [], headsIndexes = [], numHeads = decoder.readUint53() for (let i = 0; i < numHeads; i++) { heads.push(bytesToHexString(decoder.readRawBytes(32))) } const changesColumns = decodeColumnInfo(decoder) const opsColumns = decodeColumnInfo(decoder) for (let i = 0; i < changesColumns.length; i++) { changesColumns[i].buffer = decoder.readRawBytes(changesColumns[i].bufferLen) inflateColumn(changesColumns[i]) } for (let i = 0; i < opsColumns.length; i++) { opsColumns[i].buffer = decoder.readRawBytes(opsColumns[i].bufferLen) inflateColumn(opsColumns[i]) } if (!decoder.done) { for (let i = 0; i < numHeads; i++) headsIndexes.push(decoder.readUint53()) } const extraBytes = decoder.readRawBytes(decoder.buf.byteLength - decoder.offset) return { changesColumns, opsColumns, actorIds, heads, headsIndexes, extraBytes } } function decodeDocument(buffer) { const { changesColumns, opsColumns, actorIds, heads } = decodeDocumentHeader(buffer) const changes = decodeColumns(changesColumns, actorIds, DOCUMENT_COLUMNS) const ops = decodeOps(decodeColumns(opsColumns, actorIds, DOC_OPS_COLUMNS), true) groupChangeOps(changes, ops) decodeDocumentChanges(changes, heads) return changes } /** * DEFLATE-compresses the given column if it is large enough to make the compression worthwhile. */ function deflateColumn(column) { if (column.encoder.buffer.byteLength >= DEFLATE_MIN_SIZE) { column.encoder = {buffer: pako.deflateRaw(column.encoder.buffer)} column.columnId |= COLUMN_TYPE_DEFLATE } } /** * Decompresses the given column if it is DEFLATE-compressed. */ function inflateColumn(column) { if ((column.columnId & COLUMN_TYPE_DEFLATE) !== 0) { column.buffer = pako.inflateRaw(column.buffer) column.columnId ^= COLUMN_TYPE_DEFLATE } } module.exports = { COLUMN_TYPE, VALUE_TYPE, ACTIONS, OBJECT_TYPE, DOC_OPS_COLUMNS, CHANGE_COLUMNS, DOCUMENT_COLUMNS, encoderByColumnId, decoderByColumnId, makeDecoders, decodeValue, splitContainers, encodeChange, decodeChangeColumns, decodeChange, decodeChangeMeta, decodeChanges, encodeDocumentHeader, decodeDocumentHeader, decodeDocument } ================================================ FILE: backend/encoding.js ================================================ /** * UTF-8 decoding and encoding using API that is supported in Node >= 12 and modern browsers: * https://developer.mozilla.org/en-US/docs/Web/API/TextEncoder/encode * https://developer.mozilla.org/en-US/docs/Web/API/TextDecoder/decode * If you're running in an environment where it's not available, please use a polyfill, such as: * https://github.com/anonyco/FastestSmallestTextEncoderDecoder */ const utf8encoder = new TextEncoder() const utf8decoder = new TextDecoder('utf-8') function stringToUtf8(string) { return utf8encoder.encode(string) } function utf8ToString(buffer) { return utf8decoder.decode(buffer) } /** * Converts a string consisting of hexadecimal digits into an Uint8Array. */ function hexStringToBytes(value) { if (typeof value !== 'string') { throw new TypeError('value is not a string') } if (!/^([0-9a-f][0-9a-f])*$/.test(value)) { throw new RangeError('value is not hexadecimal') } if (value === '') { return new Uint8Array(0) } else { return new Uint8Array(value.match(/../g).map(b => parseInt(b, 16))) } } const NIBBLE_TO_HEX = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'] const BYTE_TO_HEX = new Array(256) for (let i = 0; i < 256; i++) { BYTE_TO_HEX[i] = `${NIBBLE_TO_HEX[(i >>> 4) & 0xf]}${NIBBLE_TO_HEX[i & 0xf]}`; } /** * Converts a Uint8Array into the equivalent hexadecimal string. */ function bytesToHexString(bytes) { let hex = '', len = bytes.byteLength for (let i = 0; i < len; i++) { hex += BYTE_TO_HEX[bytes[i]] } return hex } /** * Wrapper around an Uint8Array that allows values to be appended to the buffer, * and that automatically grows the buffer when space runs out. */ class Encoder { constructor() { this.buf = new Uint8Array(16) this.offset = 0 } /** * Returns the byte array containing the encoded data. */ get buffer() { this.finish() return this.buf.subarray(0, this.offset) } /** * Reallocates the encoder's buffer to be bigger. */ grow(minSize = 0) { let newSize = this.buf.byteLength * 4 while (newSize < minSize) newSize *= 2 const newBuf = new Uint8Array(newSize) newBuf.set(this.buf, 0) this.buf = newBuf return this } /** * Appends one byte (0 to 255) to the buffer. */ appendByte(value) { if (this.offset >= this.buf.byteLength) this.grow() this.buf[this.offset] = value this.offset += 1 } /** * Encodes a 32-bit nonnegative integer in a variable number of bytes using * the LEB128 encoding scheme (https://en.wikipedia.org/wiki/LEB128) and * appends it to the buffer. Returns the number of bytes written. */ appendUint32(value) { if (!Number.isInteger(value)) throw new RangeError('value is not an integer') if (value < 0 || value > 0xffffffff) throw new RangeError('number out of range') const numBytes = Math.max(1, Math.ceil((32 - Math.clz32(value)) / 7)) if (this.offset + numBytes > this.buf.byteLength) this.grow() for (let i = 0; i < numBytes; i++) { this.buf[this.offset + i] = (value & 0x7f) | (i === numBytes - 1 ? 0x00 : 0x80) value >>>= 7 // zero-filling right shift } this.offset += numBytes return numBytes } /** * Encodes a 32-bit signed integer in a variable number of bytes using the * LEB128 encoding scheme (https://en.wikipedia.org/wiki/LEB128) and appends * it to the buffer. Returns the number of bytes written. */ appendInt32(value) { if (!Number.isInteger(value)) throw new RangeError('value is not an integer') if (value < -0x80000000 || value > 0x7fffffff) throw new RangeError('number out of range') const numBytes = Math.ceil((33 - Math.clz32(value >= 0 ? value : -value - 1)) / 7) if (this.offset + numBytes > this.buf.byteLength) this.grow() for (let i = 0; i < numBytes; i++) { this.buf[this.offset + i] = (value & 0x7f) | (i === numBytes - 1 ? 0x00 : 0x80) value >>= 7 // sign-propagating right shift } this.offset += numBytes return numBytes } /** * Encodes a nonnegative integer in a variable number of bytes using the LEB128 * encoding scheme, up to the maximum size of integers supported by JavaScript * (53 bits). */ appendUint53(value) { if (!Number.isInteger(value)) throw new RangeError('value is not an integer') if (value < 0 || value > Number.MAX_SAFE_INTEGER) { throw new RangeError('number out of range') } const high32 = Math.floor(value / 0x100000000) const low32 = (value & 0xffffffff) >>> 0 // right shift to interpret as unsigned return this.appendUint64(high32, low32) } /** * Encodes a signed integer in a variable number of bytes using the LEB128 * encoding scheme, up to the maximum size of integers supported by JavaScript * (53 bits). */ appendInt53(value) { if (!Number.isInteger(value)) throw new RangeError('value is not an integer') if (value < Number.MIN_SAFE_INTEGER || value > Number.MAX_SAFE_INTEGER) { throw new RangeError('number out of range') } const high32 = Math.floor(value / 0x100000000) const low32 = (value & 0xffffffff) >>> 0 // right shift to interpret as unsigned return this.appendInt64(high32, low32) } /** * Encodes a 64-bit nonnegative integer in a variable number of bytes using * the LEB128 encoding scheme, and appends it to the buffer. The number is * given as two 32-bit halves since JavaScript cannot accurately represent * integers with more than 53 bits in a single variable. */ appendUint64(high32, low32) { if (!Number.isInteger(high32) || !Number.isInteger(low32)) { throw new RangeError('value is not an integer') } if (high32 < 0 || high32 > 0xffffffff || low32 < 0 || low32 > 0xffffffff) { throw new RangeError('number out of range') } if (high32 === 0) return this.appendUint32(low32) const numBytes = Math.ceil((64 - Math.clz32(high32)) / 7) if (this.offset + numBytes > this.buf.byteLength) this.grow() for (let i = 0; i < 4; i++) { this.buf[this.offset + i] = (low32 & 0x7f) | 0x80 low32 >>>= 7 // zero-filling right shift } this.buf[this.offset + 4] = (low32 & 0x0f) | ((high32 & 0x07) << 4) | (numBytes === 5 ? 0x00 : 0x80) high32 >>>= 3 for (let i = 5; i < numBytes; i++) { this.buf[this.offset + i] = (high32 & 0x7f) | (i === numBytes - 1 ? 0x00 : 0x80) high32 >>>= 7 } this.offset += numBytes return numBytes } /** * Encodes a 64-bit signed integer in a variable number of bytes using the * LEB128 encoding scheme, and appends it to the buffer. The number is given * as two 32-bit halves since JavaScript cannot accurately represent integers * with more than 53 bits in a single variable. The sign of the 64-bit * number is determined by the sign of the `high32` half; the sign of the * `low32` half is ignored. */ appendInt64(high32, low32) { if (!Number.isInteger(high32) || !Number.isInteger(low32)) { throw new RangeError('value is not an integer') } if (high32 < -0x80000000 || high32 > 0x7fffffff || low32 < -0x80000000 || low32 > 0xffffffff) { throw new RangeError('number out of range') } low32 >>>= 0 // interpret as unsigned if (high32 === 0 && low32 <= 0x7fffffff) return this.appendInt32(low32) if (high32 === -1 && low32 >= 0x80000000) return this.appendInt32(low32 - 0x100000000) const numBytes = Math.ceil((65 - Math.clz32(high32 >= 0 ? high32 : -high32 - 1)) / 7) if (this.offset + numBytes > this.buf.byteLength) this.grow() for (let i = 0; i < 4; i++) { this.buf[this.offset + i] = (low32 & 0x7f) | 0x80 low32 >>>= 7 // zero-filling right shift } this.buf[this.offset + 4] = (low32 & 0x0f) | ((high32 & 0x07) << 4) | (numBytes === 5 ? 0x00 : 0x80) high32 >>= 3 // sign-propagating right shift for (let i = 5; i < numBytes; i++) { this.buf[this.offset + i] = (high32 & 0x7f) | (i === numBytes - 1 ? 0x00 : 0x80) high32 >>= 7 } this.offset += numBytes return numBytes } /** * Appends the contents of byte buffer `data` to the buffer. Returns the * number of bytes appended. */ appendRawBytes(data) { if (this.offset + data.byteLength > this.buf.byteLength) { this.grow(this.offset + data.byteLength) } this.buf.set(data, this.offset) this.offset += data.byteLength return data.byteLength } /** * Appends a UTF-8 string to the buffer, without any metadata. Returns the * number of bytes appended. */ appendRawString(value) { if (typeof value !== 'string') throw new TypeError('value is not a string') return this.appendRawBytes(stringToUtf8(value)) } /** * Appends the contents of byte buffer `data` to the buffer, prefixed with the * number of bytes in the buffer (as a LEB128-encoded unsigned integer). */ appendPrefixedBytes(data) { this.appendUint53(data.byteLength) this.appendRawBytes(data) return this } /** * Appends a UTF-8 string to the buffer, prefixed with its length in bytes * (where the length is encoded as an unsigned LEB128 integer). */ appendPrefixedString(value) { if (typeof value !== 'string') throw new TypeError('value is not a string') this.appendPrefixedBytes(stringToUtf8(value)) return this } /** * Takes a value, which must be a string consisting only of hexadecimal * digits, maps it to a byte array, and appends it to the buffer, prefixed * with its length in bytes. */ appendHexString(value) { this.appendPrefixedBytes(hexStringToBytes(value)) return this } /** * Flushes any unwritten data to the buffer. Call this before reading from * the buffer constructed by this Encoder. */ finish() { } } /** * Counterpart to Encoder. Wraps a Uint8Array buffer with a cursor indicating * the current decoding position, and allows values to be incrementally read by * decoding the bytes at the current position. */ class Decoder { constructor(buffer) { if (!(buffer instanceof Uint8Array)) { throw new TypeError(`Not a byte array: ${buffer}`) } this.buf = buffer this.offset = 0 } /** * Returns false if there is still data to be read at the current decoding * position, and true if we are at the end of the buffer. */ get done() { return this.offset === this.buf.byteLength } /** * Resets the cursor position, so that the next read goes back to the * beginning of the buffer. */ reset() { this.offset = 0 } /** * Moves the current decoding position forward by the specified number of * bytes, without decoding anything. */ skip(bytes) { if (this.offset + bytes > this.buf.byteLength) { throw new RangeError('cannot skip beyond end of buffer') } this.offset += bytes } /** * Reads one byte (0 to 255) from the buffer. */ readByte() { this.offset += 1 return this.buf[this.offset - 1] } /** * Reads a LEB128-encoded unsigned integer from the current position in the buffer. * Throws an exception if the value doesn't fit in a 32-bit unsigned int. */ readUint32() { let result = 0, shift = 0 while (this.offset < this.buf.byteLength) { const nextByte = this.buf[this.offset] if (shift === 28 && (nextByte & 0xf0) !== 0) { // more than 5 bytes, or value > 0xffffffff throw new RangeError('number out of range') } result = (result | (nextByte & 0x7f) << shift) >>> 0 // right shift to interpret value as unsigned shift += 7 this.offset++ if ((nextByte & 0x80) === 0) return result } throw new RangeError('buffer ended with incomplete number') } /** * Reads a LEB128-encoded signed integer from the current position in the buffer. * Throws an exception if the value doesn't fit in a 32-bit signed int. */ readInt32() { let result = 0, shift = 0 while (this.offset < this.buf.byteLength) { const nextByte = this.buf[this.offset] if ((shift === 28 && (nextByte & 0x80) !== 0) || // more than 5 bytes (shift === 28 && (nextByte & 0x40) === 0 && (nextByte & 0x38) !== 0) || // positive int > 0x7fffffff (shift === 28 && (nextByte & 0x40) !== 0 && (nextByte & 0x38) !== 0x38)) { // negative int < -0x80000000 throw new RangeError('number out of range') } result |= (nextByte & 0x7f) << shift shift += 7 this.offset++ if ((nextByte & 0x80) === 0) { if ((nextByte & 0x40) === 0 || shift > 28) { return result // positive, or negative value that doesn't need sign-extending } else { return result | (-1 << shift) // sign-extend negative integer } } } throw new RangeError('buffer ended with incomplete number') } /** * Reads a LEB128-encoded unsigned integer from the current position in the * buffer. Allows any integer that can be safely represented by JavaScript * (up to 2^53 - 1), and throws an exception outside of that range. */ readUint53() { const { low32, high32 } = this.readUint64() if (high32 < 0 || high32 > 0x1fffff) { throw new RangeError('number out of range') } return high32 * 0x100000000 + low32 } /** * Reads a LEB128-encoded signed integer from the current position in the * buffer. Allows any integer that can be safely represented by JavaScript * (between -(2^53 - 1) and 2^53 - 1), throws an exception outside of that range. */ readInt53() { const { low32, high32 } = this.readInt64() if (high32 < -0x200000 || (high32 === -0x200000 && low32 === 0) || high32 > 0x1fffff) { throw new RangeError('number out of range') } return high32 * 0x100000000 + low32 } /** * Reads a LEB128-encoded unsigned integer from the current position in the * buffer. Throws an exception if the value doesn't fit in a 64-bit unsigned * int. Returns the number in two 32-bit halves, as an object of the form * `{high32, low32}`. */ readUint64() { let low32 = 0, high32 = 0, shift = 0 while (this.offset < this.buf.byteLength && shift <= 28) { const nextByte = this.buf[this.offset] low32 = (low32 | (nextByte & 0x7f) << shift) >>> 0 // right shift to interpret value as unsigned if (shift === 28) { high32 = (nextByte & 0x70) >>> 4 } shift += 7 this.offset++ if ((nextByte & 0x80) === 0) return { high32, low32 } } shift = 3 while (this.offset < this.buf.byteLength) { const nextByte = this.buf[this.offset] if (shift === 31 && (nextByte & 0xfe) !== 0) { // more than 10 bytes, or value > 2^64 - 1 throw new RangeError('number out of range') } high32 = (high32 | (nextByte & 0x7f) << shift) >>> 0 shift += 7 this.offset++ if ((nextByte & 0x80) === 0) return { high32, low32 } } throw new RangeError('buffer ended with incomplete number') } /** * Reads a LEB128-encoded signed integer from the current position in the * buffer. Throws an exception if the value doesn't fit in a 64-bit signed * int. Returns the number in two 32-bit halves, as an object of the form * `{high32, low32}`. The `low32` half is always non-negative, and the * sign of the `high32` half indicates the sign of the 64-bit number. */ readInt64() { let low32 = 0, high32 = 0, shift = 0 while (this.offset < this.buf.byteLength && shift <= 28) { const nextByte = this.buf[this.offset] low32 = (low32 | (nextByte & 0x7f) << shift) >>> 0 // right shift to interpret value as unsigned if (shift === 28) { high32 = (nextByte & 0x70) >>> 4 } shift += 7 this.offset++ if ((nextByte & 0x80) === 0) { if ((nextByte & 0x40) !== 0) { // sign-extend negative integer if (shift < 32) low32 = (low32 | (-1 << shift)) >>> 0 high32 |= -1 << Math.max(shift - 32, 0) } return { high32, low32 } } } shift = 3 while (this.offset < this.buf.byteLength) { const nextByte = this.buf[this.offset] // On the 10th byte there are only two valid values: all 7 value bits zero // (if the value is positive) or all 7 bits one (if the value is negative) if (shift === 31 && nextByte !== 0 && nextByte !== 0x7f) { throw new RangeError('number out of range') } high32 |= (nextByte & 0x7f) << shift shift += 7 this.offset++ if ((nextByte & 0x80) === 0) { if ((nextByte & 0x40) !== 0 && shift < 32) { // sign-extend negative integer high32 |= -1 << shift } return { high32, low32 } } } throw new RangeError('buffer ended with incomplete number') } /** * Extracts a subarray `length` bytes in size, starting from the current * position in the buffer, and moves the position forward. */ readRawBytes(length) { const start = this.offset if (start + length > this.buf.byteLength) { throw new RangeError('subarray exceeds buffer size') } this.offset += length return this.buf.subarray(start, this.offset) } /** * Extracts `length` bytes from the buffer, starting from the current position, * and returns the UTF-8 string decoding of those bytes. */ readRawString(length) { return utf8ToString(this.readRawBytes(length)) } /** * Extracts a subarray from the current position in the buffer, prefixed with * its length in bytes (encoded as an unsigned LEB128 integer). */ readPrefixedBytes() { return this.readRawBytes(this.readUint53()) } /** * Reads a UTF-8 string from the current position in the buffer, prefixed with its * length in bytes (where the length is encoded as an unsigned LEB128 integer). */ readPrefixedString() { return utf8ToString(this.readPrefixedBytes()) } /** * Reads a byte array from the current position in the buffer, prefixed with its * length in bytes. Returns that byte array converted to a hexadecimal string. */ readHexString() { return bytesToHexString(this.readPrefixedBytes()) } } /** * An encoder that uses run-length encoding to compress sequences of repeated * values. The constructor argument specifies the type of values, which may be * either 'int', 'uint', or 'utf8'. Besides valid values of the selected * datatype, values may also be null. * * The encoded buffer starts with a LEB128-encoded signed integer, the * repetition count. The interpretation of the following values depends on this * repetition count: * - If this number is a positive value n, the next value in the buffer * (encoded as the specified datatype) is repeated n times in the sequence. * - If the repetition count is a negative value -n, then the next n values * (encoded as the specified datatype) in the buffer are treated as a * literal, i.e. they appear in the sequence without any further * interpretation or repetition. * - If the repetition count is zero, then the next value in the buffer is a * LEB128-encoded unsigned integer indicating the number of null values * that appear at the current position in the sequence. * * After one of these three has completed, the process repeats, starting again * with a repetition count, until we reach the end of the buffer. */ class RLEEncoder extends Encoder { constructor(type) { super() this.type = type this.state = 'empty' this.lastValue = undefined this.count = 0 this.literal = [] } /** * Appends a new value to the sequence. If `repetitions` is given, the value is repeated * `repetitions` times. */ appendValue(value, repetitions = 1) { this._appendValue(value, repetitions) } /** * Like `appendValue()`, but this method is not overridden by `DeltaEncoder`. */ _appendValue(value, repetitions = 1) { if (repetitions <= 0) return if (this.state === 'empty') { this.state = (value === null ? 'nulls' : (repetitions === 1 ? 'loneValue' : 'repetition')) this.lastValue = value this.count = repetitions } else if (this.state === 'loneValue') { if (value === null) { this.flush() this.state = 'nulls' this.count = repetitions } else if (value === this.lastValue) { this.state = 'repetition' this.count = 1 + repetitions } else if (repetitions > 1) { this.flush() this.state = 'repetition' this.count = repetitions this.lastValue = value } else { this.state = 'literal' this.literal = [this.lastValue] this.lastValue = value } } else if (this.state === 'repetition') { if (value === null) { this.flush() this.state = 'nulls' this.count = repetitions } else if (value === this.lastValue) { this.count += repetitions } else if (repetitions > 1) { this.flush() this.state = 'repetition' this.count = repetitions this.lastValue = value } else { this.flush() this.state = 'loneValue' this.lastValue = value } } else if (this.state === 'literal') { if (value === null) { this.literal.push(this.lastValue) this.flush() this.state = 'nulls' this.count = repetitions } else if (value === this.lastValue) { this.flush() this.state = 'repetition' this.count = 1 + repetitions } else if (repetitions > 1) { this.literal.push(this.lastValue) this.flush() this.state = 'repetition' this.count = repetitions this.lastValue = value } else { this.literal.push(this.lastValue) this.lastValue = value } } else if (this.state === 'nulls') { if (value === null) { this.count += repetitions } else if (repetitions > 1) { this.flush() this.state = 'repetition' this.count = repetitions this.lastValue = value } else { this.flush() this.state = 'loneValue' this.lastValue = value } } } /** * Copies values from the RLEDecoder `decoder` into this encoder. The `options` object may * contain the following keys: * - `count`: The number of values to copy. If not specified, copies all remaining values. * - `sumValues`: If true, the function computes the sum of all numeric values as they are * copied (null values are counted as zero), and returns that number. * - `sumShift`: If set, values are shifted right by `sumShift` bits before adding to the sum. * * Returns an object of the form `{nonNullValues, sum}` where `nonNullValues` is the number of * non-null values copied, and `sum` is the sum (only if the `sumValues` option is set). */ copyFrom(decoder, options = {}) { const { count, sumValues, sumShift } = options if (!(decoder instanceof RLEDecoder) || (decoder.type !== this.type)) { throw new TypeError('incompatible type of decoder') } let remaining = (typeof count === 'number' ? count : Number.MAX_SAFE_INTEGER) let nonNullValues = 0, sum = 0 if (count && remaining > 0 && decoder.done) throw new RangeError(`cannot copy ${count} values`) if (remaining === 0 || decoder.done) return sumValues ? {nonNullValues, sum} : {nonNullValues} // Copy a value so that we have a well-defined starting state. NB: when super.copyFrom() is // called by the DeltaEncoder subclass, the following calls to readValue() and appendValue() // refer to the overridden methods, while later readRecord(), readRawValue() and _appendValue() // calls refer to the non-overridden RLEDecoder/RLEEncoder methods. let firstValue = decoder.readValue() if (firstValue === null) { const numNulls = Math.min(decoder.count + 1, remaining) remaining -= numNulls decoder.count -= numNulls - 1 this.appendValue(null, numNulls) if (count && remaining > 0 && decoder.done) throw new RangeError(`cannot copy ${count} values`) if (remaining === 0 || decoder.done) return sumValues ? {nonNullValues, sum} : {nonNullValues} firstValue = decoder.readValue() if (firstValue === null) throw new RangeError('null run must be followed by non-null value') } this.appendValue(firstValue) remaining-- nonNullValues++ if (sumValues) sum += (sumShift ? (firstValue >>> sumShift) : firstValue) if (count && remaining > 0 && decoder.done) throw new RangeError(`cannot copy ${count} values`) if (remaining === 0 || decoder.done) return sumValues ? {nonNullValues, sum} : {nonNullValues} // Copy data at the record level without expanding repetitions let firstRun = (decoder.count > 0) while (remaining > 0 && !decoder.done) { if (!firstRun) decoder.readRecord() const numValues = Math.min(decoder.count, remaining) decoder.count -= numValues if (decoder.state === 'literal') { nonNullValues += numValues for (let i = 0; i < numValues; i++) { if (decoder.done) throw new RangeError('incomplete literal') const value = decoder.readRawValue() if (value === decoder.lastValue) throw new RangeError('Repetition of values is not allowed in literal') decoder.lastValue = value this._appendValue(value) if (sumValues) sum += (sumShift ? (value >>> sumShift) : value) } } else if (decoder.state === 'repetition') { nonNullValues += numValues if (sumValues) sum += numValues * (sumShift ? (decoder.lastValue >>> sumShift) : decoder.lastValue) const value = decoder.lastValue this._appendValue(value) if (numValues > 1) { this._appendValue(value) if (this.state !== 'repetition') throw new RangeError(`Unexpected state ${this.state}`) this.count += numValues - 2 } } else if (decoder.state === 'nulls') { this._appendValue(null) if (this.state !== 'nulls') throw new RangeError(`Unexpected state ${this.state}`) this.count += numValues - 1 } firstRun = false remaining -= numValues } if (count && remaining > 0 && decoder.done) throw new RangeError(`cannot copy ${count} values`) return sumValues ? {nonNullValues, sum} : {nonNullValues} } /** * Private method, do not call from outside the class. */ flush() { if (this.state === 'loneValue') { this.appendInt32(-1) this.appendRawValue(this.lastValue) } else if (this.state === 'repetition') { this.appendInt53(this.count) this.appendRawValue(this.lastValue) } else if (this.state === 'literal') { this.appendInt53(-this.literal.length) for (let v of this.literal) this.appendRawValue(v) } else if (this.state === 'nulls') { this.appendInt32(0) this.appendUint53(this.count) } this.state = 'empty' } /** * Private method, do not call from outside the class. */ appendRawValue(value) { if (this.type === 'int') { this.appendInt53(value) } else if (this.type === 'uint') { this.appendUint53(value) } else if (this.type === 'utf8') { this.appendPrefixedString(value) } else { throw new RangeError(`Unknown RLEEncoder datatype: ${this.type}`) } } /** * Flushes any unwritten data to the buffer. Call this before reading from * the buffer constructed by this Encoder. */ finish() { if (this.state === 'literal') this.literal.push(this.lastValue) // Don't write anything if the only values we have seen are nulls if (this.state !== 'nulls' || this.offset > 0) this.flush() } } /** * Counterpart to RLEEncoder: reads values from an RLE-compressed sequence, * returning nulls and repeated values as required. */ class RLEDecoder extends Decoder { constructor(type, buffer) { super(buffer) this.type = type this.lastValue = undefined this.count = 0 this.state = undefined } /** * Returns false if there is still data to be read at the current decoding * position, and true if we are at the end of the buffer. */ get done() { return (this.count === 0) && (this.offset === this.buf.byteLength) } /** * Resets the cursor position, so that the next read goes back to the * beginning of the buffer. */ reset() { this.offset = 0 this.lastValue = undefined this.count = 0 this.state = undefined } /** * Returns the next value (or null) in the sequence. */ readValue() { if (this.done) return null if (this.count === 0) this.readRecord() this.count -= 1 if (this.state === 'literal') { const value = this.readRawValue() if (value === this.lastValue) throw new RangeError('Repetition of values is not allowed in literal') this.lastValue = value return value } else { return this.lastValue } } /** * Discards the next `numSkip` values in the sequence. */ skipValues(numSkip) { while (numSkip > 0 && !this.done) { if (this.count === 0) { this.count = this.readInt53() if (this.count > 0) { this.lastValue = (this.count <= numSkip) ? this.skipRawValues(1) : this.readRawValue() this.state = 'repetition' } else if (this.count < 0) { this.count = -this.count this.state = 'literal' } else { // this.count == 0 this.count = this.readUint53() this.lastValue = null this.state = 'nulls' } } const consume = Math.min(numSkip, this.count) if (this.state === 'literal') this.skipRawValues(consume) numSkip -= consume this.count -= consume } } /** * Private method, do not call from outside the class. * Reads a repetition count from the buffer and sets up the state appropriately. */ readRecord() { this.count = this.readInt53() if (this.count > 1) { const value = this.readRawValue() if ((this.state === 'repetition' || this.state === 'literal') && this.lastValue === value) { throw new RangeError('Successive repetitions with the same value are not allowed') } this.state = 'repetition' this.lastValue = value } else if (this.count === 1) { throw new RangeError('Repetition count of 1 is not allowed, use a literal instead') } else if (this.count < 0) { this.count = -this.count if (this.state === 'literal') throw new RangeError('Successive literals are not allowed') this.state = 'literal' } else { // this.count == 0 if (this.state === 'nulls') throw new RangeError('Successive null runs are not allowed') this.count = this.readUint53() if (this.count === 0) throw new RangeError('Zero-length null runs are not allowed') this.lastValue = null this.state = 'nulls' } } /** * Private method, do not call from outside the class. * Reads one value of the datatype configured on construction. */ readRawValue() { if (this.type === 'int') { return this.readInt53() } else if (this.type === 'uint') { return this.readUint53() } else if (this.type === 'utf8') { return this.readPrefixedString() } else { throw new RangeError(`Unknown RLEDecoder datatype: ${this.type}`) } } /** * Private method, do not call from outside the class. * Skips over `num` values of the datatype configured on construction. */ skipRawValues(num) { if (this.type === 'utf8') { for (let i = 0; i < num; i++) this.skip(this.readUint53()) } else { while (num > 0 && this.offset < this.buf.byteLength) { if ((this.buf[this.offset] & 0x80) === 0) num-- this.offset++ } if (num > 0) throw new RangeError('cannot skip beyond end of buffer') } } } /** * A variant of RLEEncoder: rather than storing the actual values passed to * appendValue(), this version stores only the first value, and for all * subsequent values it stores the difference to the previous value. This * encoding is good when values tend to come in sequentially incrementing runs, * because the delta between successive values is 1, and repeated values of 1 * are easily compressed with run-length encoding. * * Null values are also allowed, as with RLEEncoder. */ class DeltaEncoder extends RLEEncoder { constructor() { super('int') this.absoluteValue = 0 } /** * Appends a new integer value to the sequence. If `repetitions` is given, the value is repeated * `repetitions` times. */ appendValue(value, repetitions = 1) { if (repetitions <= 0) return if (typeof value === 'number') { super.appendValue(value - this.absoluteValue, 1) this.absoluteValue = value if (repetitions > 1) super.appendValue(0, repetitions - 1) } else { super.appendValue(value, repetitions) } } /** * Copies values from the DeltaDecoder `decoder` into this encoder. The `options` object may * contain the key `count`, indicating the number of values to copy. If not specified, copies * all remaining values in the decoder. */ copyFrom(decoder, options = {}) { if (options.sumValues) { throw new RangeError('unsupported options for DeltaEncoder.copyFrom()') } if (!(decoder instanceof DeltaDecoder)) { throw new TypeError('incompatible type of decoder') } let remaining = options.count if (remaining > 0 && decoder.done) throw new RangeError(`cannot copy ${remaining} values`) if (remaining === 0 || decoder.done) return // Copy any null values, and the first non-null value, so that appendValue() computes the // difference between the encoder's last value and the decoder's first (absolute) value. let value = decoder.readValue(), nulls = 0 this.appendValue(value) if (value === null) { nulls = decoder.count + 1 if (remaining !== undefined && remaining < nulls) nulls = remaining decoder.count -= nulls - 1 this.count += nulls - 1 if (remaining > nulls && decoder.done) throw new RangeError(`cannot copy ${remaining} values`) if (remaining === nulls || decoder.done) return // The next value read is certain to be non-null because we're not at the end of the decoder, // and a run of nulls must be followed by a run of non-nulls. if (decoder.count === 0) this.appendValue(decoder.readValue()) } // Once we have the first value, the subsequent relative values can be copied verbatim without // any further processing. Note that the first value copied by super.copyFrom() is an absolute // value, while subsequent values are relative. Thus, the sum of all of the (non-null) copied // values must equal the absolute value of the final element copied. if (remaining !== undefined) remaining -= nulls + 1 const { nonNullValues, sum } = super.copyFrom(decoder, {count: remaining, sumValues: true}) if (nonNullValues > 0) { this.absoluteValue = sum decoder.absoluteValue = sum } } } /** * Counterpart to DeltaEncoder: reads values from a delta-compressed sequence of * numbers (may include null values). */ class DeltaDecoder extends RLEDecoder { constructor(buffer) { super('int', buffer) this.absoluteValue = 0 } /** * Resets the cursor position, so that the next read goes back to the * beginning of the buffer. */ reset() { this.offset = 0 this.lastValue = undefined this.count = 0 this.state = undefined this.absoluteValue = 0 } /** * Returns the next integer (or null) value in the sequence. */ readValue() { const value = super.readValue() if (value === null) return null this.absoluteValue += value return this.absoluteValue } /** * Discards the next `numSkip` values in the sequence. */ skipValues(numSkip) { while (numSkip > 0 && !this.done) { if (this.count === 0) this.readRecord() const consume = Math.min(numSkip, this.count) if (this.state === 'literal') { for (let i = 0; i < consume; i++) { this.lastValue = this.readRawValue() this.absoluteValue += this.lastValue } } else if (this.state === 'repetition') { this.absoluteValue += consume * this.lastValue } numSkip -= consume this.count -= consume } } } /** * Encodes a sequence of boolean values by mapping it to a sequence of integers: * the number of false values, followed by the number of true values, followed * by the number of false values, and so on. Each number is encoded as a LEB128 * unsigned integer. This encoding is a bit like RLEEncoder, except that we * only encode the repetition count but not the actual value, since the values * just alternate between false and true (starting with false). */ class BooleanEncoder extends Encoder { constructor() { super() this.lastValue = false this.count = 0 } /** * Appends a new value to the sequence. If `repetitions` is given, the value is repeated * `repetitions` times. */ appendValue(value, repetitions = 1) { if (value !== false && value !== true) { throw new RangeError(`Unsupported value for BooleanEncoder: ${value}`) } if (repetitions <= 0) return if (this.lastValue === value) { this.count += repetitions } else { this.appendUint53(this.count) this.lastValue = value this.count = repetitions } } /** * Copies values from the BooleanDecoder `decoder` into this encoder. The `options` object may * contain the key `count`, indicating the number of values to copy. If not specified, copies * all remaining values in the decoder. */ copyFrom(decoder, options = {}) { if (!(decoder instanceof BooleanDecoder)) { throw new TypeError('incompatible type of decoder') } const { count } = options let remaining = (typeof count === 'number' ? count : Number.MAX_SAFE_INTEGER) if (count && remaining > 0 && decoder.done) throw new RangeError(`cannot copy ${count} values`) if (remaining === 0 || decoder.done) return // Copy one value to bring decoder and encoder state into sync, then finish that value's repetitions this.appendValue(decoder.readValue()) remaining-- const firstCopy = Math.min(decoder.count, remaining) this.count += firstCopy decoder.count -= firstCopy remaining -= firstCopy while (remaining > 0 && !decoder.done) { decoder.count = decoder.readUint53() if (decoder.count === 0) throw new RangeError('Zero-length runs are not allowed') decoder.lastValue = !decoder.lastValue this.appendUint53(this.count) const numCopied = Math.min(decoder.count, remaining) this.count = numCopied this.lastValue = decoder.lastValue decoder.count -= numCopied remaining -= numCopied } if (count && remaining > 0 && decoder.done) throw new RangeError(`cannot copy ${count} values`) } /** * Flushes any unwritten data to the buffer. Call this before reading from * the buffer constructed by this Encoder. */ finish() { if (this.count > 0) { this.appendUint53(this.count) this.count = 0 } } } /** * Counterpart to BooleanEncoder: reads boolean values from a runlength-encoded * sequence. */ class BooleanDecoder extends Decoder { constructor(buffer) { super(buffer) this.lastValue = true // is negated the first time we read a count this.firstRun = true this.count = 0 } /** * Returns false if there is still data to be read at the current decoding * position, and true if we are at the end of the buffer. */ get done() { return (this.count === 0) && (this.offset === this.buf.byteLength) } /** * Resets the cursor position, so that the next read goes back to the * beginning of the buffer. */ reset() { this.offset = 0 this.lastValue = true this.firstRun = true this.count = 0 } /** * Returns the next value in the sequence. */ readValue() { if (this.done) return false while (this.count === 0) { this.count = this.readUint53() this.lastValue = !this.lastValue if (this.count === 0 && !this.firstRun) { throw new RangeError('Zero-length runs are not allowed') } this.firstRun = false } this.count -= 1 return this.lastValue } /** * Discards the next `numSkip` values in the sequence. */ skipValues(numSkip) { while (numSkip > 0 && !this.done) { if (this.count === 0) { this.count = this.readUint53() this.lastValue = !this.lastValue if (this.count === 0 && !this.firstRun) { throw new RangeError('Zero-length runs are not allowed') } this.firstRun = false } if (this.count < numSkip) { numSkip -= this.count this.count = 0 } else { this.count -= numSkip numSkip = 0 } } } } module.exports = { stringToUtf8, utf8ToString, hexStringToBytes, bytesToHexString, Encoder, Decoder, RLEEncoder, RLEDecoder, DeltaEncoder, DeltaDecoder, BooleanEncoder, BooleanDecoder } ================================================ FILE: backend/index.js ================================================ const { init, clone, free, applyChanges, applyLocalChange, save, load, loadChanges, getPatch, getHeads, getAllChanges, getChanges, getChangesAdded, getChangeByHash, getMissingDeps } = require("./backend") const { receiveSyncMessage, generateSyncMessage, encodeSyncMessage, decodeSyncMessage, encodeSyncState, decodeSyncState, initSyncState } = require('./sync') module.exports = { init, clone, free, applyChanges, applyLocalChange, save, load, loadChanges, getPatch, getHeads, getAllChanges, getChanges, getChangesAdded, getChangeByHash, getMissingDeps, receiveSyncMessage, generateSyncMessage, encodeSyncMessage, decodeSyncMessage, encodeSyncState, decodeSyncState, initSyncState } ================================================ FILE: backend/new.js ================================================ const { parseOpId, copyObject } = require('../src/common') const { COLUMN_TYPE, VALUE_TYPE, ACTIONS, OBJECT_TYPE, DOC_OPS_COLUMNS, CHANGE_COLUMNS, DOCUMENT_COLUMNS, encoderByColumnId, decoderByColumnId, makeDecoders, decodeValue, encodeChange, decodeChangeColumns, decodeChangeMeta, decodeChanges, decodeDocumentHeader, encodeDocumentHeader } = require('./columnar') const MAX_BLOCK_SIZE = 600 // operations const BLOOM_BITS_PER_ENTRY = 10, BLOOM_NUM_PROBES = 7 // 1% false positive rate const BLOOM_FILTER_SIZE = Math.floor(BLOOM_BITS_PER_ENTRY * MAX_BLOCK_SIZE / 8) // bytes const objActorIdx = 0, objCtrIdx = 1, keyActorIdx = 2, keyCtrIdx = 3, keyStrIdx = 4, idActorIdx = 5, idCtrIdx = 6, insertIdx = 7, actionIdx = 8, valLenIdx = 9, valRawIdx = 10, predNumIdx = 13, predActorIdx = 14, predCtrIdx = 15, succNumIdx = 13, succActorIdx = 14, succCtrIdx = 15 const PRED_COLUMN_IDS = CHANGE_COLUMNS .filter(column => ['predNum', 'predActor', 'predCtr'].includes(column.columnName)) .map(column => column.columnId) /** * Updates `objectTree`, which is a tree of nested objects, so that afterwards * `objectTree[path[0]][path[1]][...] === value`. Only the root object is mutated, whereas any * nested objects are copied before updating. This means that once the root object has been * shallow-copied, this function can be used to update it without mutating the previous version. */ function deepCopyUpdate(objectTree, path, value) { if (path.length === 1) { objectTree[path[0]] = value } else { let child = Object.assign({}, objectTree[path[0]]) deepCopyUpdate(child, path.slice(1), value) objectTree[path[0]] = child } } /** * Scans a block of document operations, encoded as columns `docCols`, to find the position at which * an operation (or sequence of operations) `ops` should be applied. `actorIds` is the array that * maps actor numbers to hexadecimal actor IDs. `resumeInsertion` is true if we're performing a list * insertion and we already found the reference element in a previous block, but we reached the end * of that previous block while scanning for the actual insertion position, and so we're continuing * the scan in a subsequent block. * * Returns an object with keys: * - `found`: false if we were scanning for a reference element in a list but couldn't find it; * true otherwise. * - `skipCount`: the number of operations, counted from the start of the block, after which the * new operations should be inserted or applied. * - `visibleCount`: if modifying a list object, the number of visible (i.e. non-deleted) list * elements that precede the position where the new operations should be applied. */ function seekWithinBlock(ops, docCols, actorIds, resumeInsertion) { for (let col of docCols) col.decoder.reset() const { objActor, objCtr, keyActor, keyCtr, keyStr, idActor, idCtr, insert } = ops const [objActorD, objCtrD, /* keyActorD */, /* keyCtrD */, keyStrD, idActorD, idCtrD, insertD, actionD, /* valLenD */, /* valRawD */, /* chldActorD */, /* chldCtrD */, succNumD] = docCols.map(col => col.decoder) let skipCount = 0, visibleCount = 0, elemVisible = false, nextObjActor = null, nextObjCtr = null let nextIdActor = null, nextIdCtr = null, nextKeyStr = null, nextInsert = null, nextSuccNum = 0 // Seek to the beginning of the object being updated if (objCtr !== null && !resumeInsertion) { while (!objCtrD.done || !objActorD.done || !actionD.done) { nextObjCtr = objCtrD.readValue() nextObjActor = actorIds[objActorD.readValue()] actionD.skipValues(1) if (nextObjCtr === null || !nextObjActor || nextObjCtr < objCtr || (nextObjCtr === objCtr && nextObjActor < objActor)) { skipCount += 1 } else { break } } } if ((nextObjCtr !== objCtr || nextObjActor !== objActor) && !resumeInsertion) { return {found: true, skipCount, visibleCount} } // Seek to the appropriate key (if string key is used) if (keyStr !== null) { keyStrD.skipValues(skipCount) while (!keyStrD.done) { const objActorIndex = objActorD.readValue() nextObjActor = objActorIndex === null ? null : actorIds[objActorIndex] nextObjCtr = objCtrD.readValue() nextKeyStr = keyStrD.readValue() if (nextKeyStr !== null && nextKeyStr < keyStr && nextObjCtr === objCtr && nextObjActor === objActor) { skipCount += 1 } else { break } } return {found: true, skipCount, visibleCount} } idCtrD.skipValues(skipCount) idActorD.skipValues(skipCount) insertD.skipValues(skipCount) succNumD.skipValues(skipCount) nextIdCtr = idCtrD.readValue() nextIdActor = actorIds[idActorD.readValue()] nextInsert = insertD.readValue() nextSuccNum = succNumD.readValue() // If we are inserting into a list, an opId key is used, and we need to seek to a position *after* // the referenced operation. Moreover, we need to skip over any existing operations with a greater // opId than the new insertion, for CRDT convergence on concurrent insertions in the same place. if (insert) { // If insertion is not at the head, search for the reference element if (!resumeInsertion && keyCtr !== null && keyCtr > 0 && keyActor !== null) { skipCount += 1 while (!idCtrD.done && !idActorD.done && (nextIdCtr !== keyCtr || nextIdActor !== keyActor)) { if (nextInsert) elemVisible = false if (nextSuccNum === 0 && !elemVisible) { visibleCount += 1 elemVisible = true } nextIdCtr = idCtrD.readValue() nextIdActor = actorIds[idActorD.readValue()] nextObjCtr = objCtrD.readValue() nextObjActor = actorIds[objActorD.readValue()] nextInsert = insertD.readValue() nextSuccNum = succNumD.readValue() if (nextObjCtr === objCtr && nextObjActor === objActor) skipCount += 1; else break } if (nextObjCtr !== objCtr || nextObjActor !== objActor || nextIdCtr !== keyCtr || nextIdActor !== keyActor || !nextInsert) { return {found: false, skipCount, visibleCount} } if (nextInsert) elemVisible = false if (nextSuccNum === 0 && !elemVisible) { visibleCount += 1 elemVisible = true } // Set up the next* variables to the operation following the reference element if (idCtrD.done || idActorD.done) return {found: true, skipCount, visibleCount} nextIdCtr = idCtrD.readValue() nextIdActor = actorIds[idActorD.readValue()] nextObjCtr = objCtrD.readValue() nextObjActor = actorIds[objActorD.readValue()] nextInsert = insertD.readValue() nextSuccNum = succNumD.readValue() } // Skip over any list elements with greater ID than the new one, and any non-insertions while ((!nextInsert || nextIdCtr > idCtr || (nextIdCtr === idCtr && nextIdActor > idActor)) && nextObjCtr === objCtr && nextObjActor === objActor) { skipCount += 1 if (nextInsert) elemVisible = false if (nextSuccNum === 0 && !elemVisible) { visibleCount += 1 elemVisible = true } if (!idCtrD.done && !idActorD.done) { nextIdCtr = idCtrD.readValue() nextIdActor = actorIds[idActorD.readValue()] nextObjCtr = objCtrD.readValue() nextObjActor = actorIds[objActorD.readValue()] nextInsert = insertD.readValue() nextSuccNum = succNumD.readValue() } else { break } } } else if (keyCtr !== null && keyCtr > 0 && keyActor !== null) { // If we are updating an existing list element, seek to just before the referenced ID while ((!nextInsert || nextIdCtr !== keyCtr || nextIdActor !== keyActor) && nextObjCtr === objCtr && nextObjActor === objActor) { skipCount += 1 if (nextInsert) elemVisible = false if (nextSuccNum === 0 && !elemVisible) { visibleCount += 1 elemVisible = true } if (!idCtrD.done && !idActorD.done) { nextIdCtr = idCtrD.readValue() nextIdActor = actorIds[idActorD.readValue()] nextObjCtr = objCtrD.readValue() nextObjActor = actorIds[objActorD.readValue()] nextInsert = insertD.readValue() nextSuccNum = succNumD.readValue() } else { break } } if (nextObjCtr !== objCtr || nextObjActor !== objActor || nextIdCtr !== keyCtr || nextIdActor !== keyActor || !nextInsert) { return {found: false, skipCount, visibleCount} } } return {found: true, skipCount, visibleCount} } /** * Returns the number of list elements that should be added to a list index when skipping over the * block with index `blockIndex` in the list object with object ID consisting of actor number * `objActorNum` and counter `objCtr`. */ function visibleListElements(docState, blockIndex, objActorNum, objCtr) { const thisBlock = docState.blocks[blockIndex] const nextBlock = docState.blocks[blockIndex + 1] if (thisBlock.lastObjectActor !== objActorNum || thisBlock.lastObjectCtr !== objCtr || thisBlock.numVisible === undefined) { return 0 // If a list element is split across the block boundary, don't double-count it } else if (thisBlock.lastVisibleActor === nextBlock.firstVisibleActor && thisBlock.lastVisibleActor !== undefined && thisBlock.lastVisibleCtr === nextBlock.firstVisibleCtr && thisBlock.lastVisibleCtr !== undefined) { return thisBlock.numVisible - 1 } else { return thisBlock.numVisible } } /** * Scans the blocks of document operations to find the position where a new operation should be * inserted. Returns an object with keys: * - `blockIndex`: the index of the block into which we should insert the new operation * - `skipCount`: the number of operations, counted from the start of the block, after which the * new operations should be inserted or merged. * - `visibleCount`: if modifying a list object, the number of visible (i.e. non-deleted) list * elements that precede the position where the new operations should be applied. */ function seekToOp(docState, ops) { const { objActor, objActorNum, objCtr, keyActor, keyCtr, keyStr } = ops let blockIndex = 0, totalVisible = 0 // Skip any blocks that contain only objects with lower objectIds if (objCtr !== null) { while (blockIndex < docState.blocks.length - 1) { const blockActor = docState.blocks[blockIndex].lastObjectActor === undefined ? undefined : docState.actorIds[docState.blocks[blockIndex].lastObjectActor] const blockCtr = docState.blocks[blockIndex].lastObjectCtr if (blockCtr === null || blockCtr < objCtr || (blockCtr === objCtr && blockActor < objActor)) { blockIndex++ } else { break } } } if (keyStr !== null) { // String key is used. First skip any blocks that contain only lower keys while (blockIndex < docState.blocks.length - 1) { const { lastObjectActor, lastObjectCtr, lastKey } = docState.blocks[blockIndex] if (objCtr === lastObjectCtr && objActorNum === lastObjectActor && lastKey !== undefined && lastKey < keyStr) blockIndex++; else break } // When we have a candidate block, decode it to find the exact insertion position const {skipCount} = seekWithinBlock(ops, docState.blocks[blockIndex].columns, docState.actorIds, false) return {blockIndex, skipCount, visibleCount: 0} } else { // List operation const insertAtHead = keyCtr === null || keyCtr === 0 || keyActor === null const keyActorNum = keyActor === null ? null : docState.actorIds.indexOf(keyActor) let resumeInsertion = false while (true) { // Search for the reference element, skipping any blocks whose Bloom filter does not contain // the reference element. We only do this if not inserting at the head (in which case there is // no reference element), or if we already found the reference element in an earlier block (in // which case we have resumeInsertion === true). The latter case arises with concurrent // insertions at the same position, and so we have to scan beyond the reference element to // find the actual insertion position, and that further scan crosses a block boundary. if (!insertAtHead && !resumeInsertion) { while (blockIndex < docState.blocks.length - 1 && docState.blocks[blockIndex].lastObjectActor === objActorNum && docState.blocks[blockIndex].lastObjectCtr === objCtr && !bloomFilterContains(docState.blocks[blockIndex].bloom, keyActorNum, keyCtr)) { // If we reach the end of the list object without a Bloom filter hit, the reference element // doesn't exist if (docState.blocks[blockIndex].lastObjectCtr > objCtr) { throw new RangeError(`Reference element not found: ${keyCtr}@${keyActor}`) } // Add up number of visible list elements in any blocks we skip, for list index computation totalVisible += visibleListElements(docState, blockIndex, objActorNum, objCtr) blockIndex++ } } // We have a candidate block. Decode it to see whether it really contains the reference element const {found, skipCount, visibleCount} = seekWithinBlock(ops, docState.blocks[blockIndex].columns, docState.actorIds, resumeInsertion) if (blockIndex === docState.blocks.length - 1 || docState.blocks[blockIndex].lastObjectActor !== objActorNum || docState.blocks[blockIndex].lastObjectCtr !== objCtr) { // Last block: if we haven't found the reference element by now, it's an error if (found) { return {blockIndex, skipCount, visibleCount: totalVisible + visibleCount} } else { throw new RangeError(`Reference element not found: ${keyCtr}@${keyActor}`) } } else if (found && skipCount < docState.blocks[blockIndex].numOps) { // The insertion position lies within the current block return {blockIndex, skipCount, visibleCount: totalVisible + visibleCount} } // Reference element not found and there are still blocks left ==> it was probably a false positive. // Reference element found, but we skipped all the way to the end of the block ==> we need to // continue scanning the next block to find the actual insertion position. // Either way, go back round the loop again to skip blocks until the next Bloom filter hit. resumeInsertion = found && ops.insert totalVisible += visibleListElements(docState, blockIndex, objActorNum, objCtr) blockIndex++ } } } /** * Updates Bloom filter `bloom`, given as a Uint8Array, to contain the list element ID consisting of * counter `elemIdCtr` and actor number `elemIdActor`. We don't actually bother computing a hash * function, since those two integers serve perfectly fine as input. We turn the two integers into a * sequence of probe indexes using the triple hashing algorithm from the following paper: * * Peter C. Dillinger and Panagiotis Manolios. Bloom Filters in Probabilistic Verification. * 5th International Conference on Formal Methods in Computer-Aided Design (FMCAD), November 2004. * http://www.ccis.northeastern.edu/home/pete/pub/bloom-filters-verification.pdf */ function bloomFilterAdd(bloom, elemIdActor, elemIdCtr) { let modulo = 8 * bloom.byteLength, x = elemIdCtr % modulo, y = elemIdActor % modulo // Use one step of FNV-1a to compute a third value from the two inputs. // Taken from http://www.isthe.com/chongo/tech/comp/fnv/index.html // The prime is just over 2^24, so elemIdCtr can be up to about 2^29 = 500 million before the // result of the multiplication exceeds 2^53. And even if it does exceed 2^53 and loses precision, // that shouldn't be a problem as it should still be deterministic, and the Bloom filter // computation only needs to be internally consistent within this library. let z = ((elemIdCtr ^ elemIdActor) * 16777619 >>> 0) % modulo for (let i = 0; i < BLOOM_NUM_PROBES; i++) { bloom[x >>> 3] |= 1 << (x & 7) x = (x + y) % modulo y = (y + z) % modulo } } /** * Returns true if the list element ID consisting of counter `elemIdCtr` and actor number * `elemIdActor` is likely to be contained in the Bloom filter `bloom`. */ function bloomFilterContains(bloom, elemIdActor, elemIdCtr) { let modulo = 8 * bloom.byteLength, x = elemIdCtr % modulo, y = elemIdActor % modulo let z = ((elemIdCtr ^ elemIdActor) * 16777619 >>> 0) % modulo // See comments in the bloomFilterAdd function for an explanation for (let i = 0; i < BLOOM_NUM_PROBES; i++) { if ((bloom[x >>> 3] & (1 << (x & 7))) === 0) { return false } x = (x + y) % modulo y = (y + z) % modulo } return true } /** * Reads the relevant columns of a block of operations and updates that block to contain the * metadata we need to efficiently figure out where to insert new operations. */ function updateBlockMetadata(block) { block.bloom = new Uint8Array(BLOOM_FILTER_SIZE) block.numOps = 0 block.lastKey = undefined block.numVisible = undefined block.lastObjectActor = undefined block.lastObjectCtr = undefined block.firstVisibleActor = undefined block.firstVisibleCtr = undefined block.lastVisibleActor = undefined block.lastVisibleCtr = undefined for (let col of block.columns) col.decoder.reset() const [objActorD, objCtrD, keyActorD, keyCtrD, keyStrD, idActorD, idCtrD, insertD, /* actionD */, /* valLenD */, /* valRawD */, /* chldActorD */, /* chldCtrD */, succNumD] = block.columns.map(col => col.decoder) while (!idCtrD.done) { block.numOps += 1 const objActor = objActorD.readValue(), objCtr = objCtrD.readValue() const keyActor = keyActorD.readValue(), keyCtr = keyCtrD.readValue(), keyStr = keyStrD.readValue() const idActor = idActorD.readValue(), idCtr = idCtrD.readValue() const insert = insertD.readValue(), succNum = succNumD.readValue() if (block.lastObjectActor !== objActor || block.lastObjectCtr !== objCtr) { block.numVisible = 0 block.lastObjectActor = objActor block.lastObjectCtr = objCtr } if (keyStr !== null) { // Map key: for each object, record the highest key contained in the block block.lastKey = keyStr } else if (insert || keyCtr !== null) { // List element block.lastKey = undefined const elemIdActor = insert ? idActor : keyActor const elemIdCtr = insert ? idCtr : keyCtr bloomFilterAdd(block.bloom, elemIdActor, elemIdCtr) // If the list element is visible, update the block metadata accordingly if (succNum === 0) { if (block.firstVisibleActor === undefined) block.firstVisibleActor = elemIdActor if (block.firstVisibleCtr === undefined) block.firstVisibleCtr = elemIdCtr if (block.lastVisibleActor !== elemIdActor || block.lastVisibleCtr !== elemIdCtr) { block.numVisible += 1 block.lastVisibleActor = elemIdActor block.lastVisibleCtr = elemIdCtr } } } } } /** * Updates a block's metadata based on an operation being added to a block. */ function addBlockOperation(block, op, actorIds, isChangeOp) { if (op[keyStrIdx] !== null) { // TODO this comparison should use UTF-8 encoding, not JavaScript's UTF-16 if (block.lastObjectCtr === op[objCtrIdx] && block.lastObjectActor === op[objActorIdx] && (block.lastKey === undefined || block.lastKey < op[keyStrIdx])) { block.lastKey = op[keyStrIdx] } } else { // List element const elemIdActor = op[insertIdx] ? op[idActorIdx] : op[keyActorIdx] const elemIdCtr = op[insertIdx] ? op[idCtrIdx] : op[keyCtrIdx] bloomFilterAdd(block.bloom, elemIdActor, elemIdCtr) // Set lastVisible on the assumption that this is the last op in the block; if there are further // ops after this one in the block, lastVisible will be overwritten again later. if (op[succNumIdx] === 0 || isChangeOp) { if (block.firstVisibleActor === undefined) block.firstVisibleActor = elemIdActor if (block.firstVisibleCtr === undefined) block.firstVisibleCtr = elemIdCtr block.lastVisibleActor = elemIdActor block.lastVisibleCtr = elemIdCtr } } // Keep track of the largest objectId contained within a block if (block.lastObjectCtr === undefined || op[objActorIdx] !== null && op[objCtrIdx] !== null && (block.lastObjectCtr === null || block.lastObjectCtr < op[objCtrIdx] || (block.lastObjectCtr === op[objCtrIdx] && actorIds[block.lastObjectActor] < actorIds[op[objActorIdx]]))) { block.lastObjectActor = op[objActorIdx] block.lastObjectCtr = op[objCtrIdx] block.lastKey = (op[keyStrIdx] !== null ? op[keyStrIdx] : undefined) block.numVisible = 0 } } /** * Takes a block containing too many operations, and splits it into a sequence of adjacent blocks of * roughly equal size. */ function splitBlock(block) { for (let col of block.columns) col.decoder.reset() // Make each of the resulting blocks between 50% and 80% full (leaving a bit of space in each // block so that it doesn't get split again right away the next time an operation is added). // The upper bound cannot be lower than 75% since otherwise we would end up with a block less than // 50% full when going from two to three blocks. const numBlocks = Math.ceil(block.numOps / (0.8 * MAX_BLOCK_SIZE)) let blocks = [], opsSoFar = 0 for (let i = 1; i <= numBlocks; i++) { const opsToCopy = Math.ceil(i * block.numOps / numBlocks) - opsSoFar const encoders = block.columns.map(col => ({columnId: col.columnId, encoder: encoderByColumnId(col.columnId)})) copyColumns(encoders, block.columns, opsToCopy) const decoders = encoders.map(col => { const decoder = decoderByColumnId(col.columnId, col.encoder.buffer) return {columnId: col.columnId, decoder} }) const newBlock = {columns: decoders} updateBlockMetadata(newBlock) blocks.push(newBlock) opsSoFar += opsToCopy } return blocks } /** * Takes an array of blocks and concatenates the corresponding columns across all of the blocks. */ function concatBlocks(blocks) { const encoders = blocks[0].columns.map(col => ({columnId: col.columnId, encoder: encoderByColumnId(col.columnId)})) for (let block of blocks) { for (let col of block.columns) col.decoder.reset() copyColumns(encoders, block.columns, block.numOps) } return encoders } /** * Copies `count` rows from the set of input columns `inCols` to the set of output columns * `outCols`. The input columns are given as an array of `{columnId, decoder}` objects, and the * output columns are given as an array of `{columnId, encoder}` objects. Both are sorted in * increasing order of columnId. If there is no matching input column for a given output column, it * is filled in with `count` blank values (according to the column type). */ function copyColumns(outCols, inCols, count) { if (count === 0) return let inIndex = 0, lastGroup = -1, lastCardinality = 0, valueColumn = -1, valueBytes = 0 for (let outCol of outCols) { while (inIndex < inCols.length && inCols[inIndex].columnId < outCol.columnId) inIndex++ let inCol = null if (inIndex < inCols.length && inCols[inIndex].columnId === outCol.columnId && inCols[inIndex].decoder.buf.byteLength > 0) { inCol = inCols[inIndex].decoder } const colCount = (outCol.columnId >> 4 === lastGroup) ? lastCardinality : count if (outCol.columnId % 8 === COLUMN_TYPE.GROUP_CARD) { lastGroup = outCol.columnId >> 4 if (inCol) { lastCardinality = outCol.encoder.copyFrom(inCol, {count, sumValues: true}).sum } else { outCol.encoder.appendValue(0, count) lastCardinality = 0 } } else if (outCol.columnId % 8 === COLUMN_TYPE.VALUE_LEN) { if (inCol) { if (inIndex + 1 === inCols.length || inCols[inIndex + 1].columnId !== outCol.columnId + 1) { throw new RangeError('VALUE_LEN column without accompanying VALUE_RAW column') } valueColumn = outCol.columnId + 1 valueBytes = outCol.encoder.copyFrom(inCol, {count: colCount, sumValues: true, sumShift: 4}).sum } else { outCol.encoder.appendValue(null, colCount) valueColumn = outCol.columnId + 1 valueBytes = 0 } } else if (outCol.columnId % 8 === COLUMN_TYPE.VALUE_RAW) { if (outCol.columnId !== valueColumn) { throw new RangeError('VALUE_RAW column without accompanying VALUE_LEN column') } if (valueBytes > 0) { outCol.encoder.appendRawBytes(inCol.readRawBytes(valueBytes)) } } else { // ACTOR_ID, INT_RLE, INT_DELTA, BOOLEAN, or STRING_RLE if (inCol) { outCol.encoder.copyFrom(inCol, {count: colCount}) } else { const blankValue = (outCol.columnId % 8 === COLUMN_TYPE.BOOLEAN) ? false : null outCol.encoder.appendValue(blankValue, colCount) } } } } /** * Parses one operation from a set of columns. The argument `columns` contains a list of objects * with `columnId` and `decoder` properties. Returns an array in which the i'th element is the * value read from the i'th column in `columns`. Does not interpret datatypes; the only * interpretation of values is that if `actorTable` is given, a value `v` in a column of type * ACTOR_ID is replaced with `actorTable[v]`. */ function readOperation(columns, actorTable) { let operation = [], colValue, lastGroup = -1, lastCardinality = 0, valueColumn = -1, valueBytes = 0 for (let col of columns) { if (col.columnId % 8 === COLUMN_TYPE.VALUE_RAW) { if (col.columnId !== valueColumn) throw new RangeError('unexpected VALUE_RAW column') colValue = col.decoder.readRawBytes(valueBytes) } else if (col.columnId % 8 === COLUMN_TYPE.GROUP_CARD) { lastGroup = col.columnId >> 4 lastCardinality = col.decoder.readValue() || 0 colValue = lastCardinality } else if (col.columnId >> 4 === lastGroup) { colValue = [] if (col.columnId % 8 === COLUMN_TYPE.VALUE_LEN) { valueColumn = col.columnId + 1 valueBytes = 0 } for (let i = 0; i < lastCardinality; i++) { let value = col.decoder.readValue() if (col.columnId % 8 === COLUMN_TYPE.ACTOR_ID && actorTable && typeof value === 'number') { value = actorTable[value] } if (col.columnId % 8 === COLUMN_TYPE.VALUE_LEN) { valueBytes += colValue >>> 4 } colValue.push(value) } } else { colValue = col.decoder.readValue() if (col.columnId % 8 === COLUMN_TYPE.ACTOR_ID && actorTable && typeof colValue === 'number') { colValue = actorTable[colValue] } if (col.columnId % 8 === COLUMN_TYPE.VALUE_LEN) { valueColumn = col.columnId + 1 valueBytes = colValue >>> 4 } } operation.push(colValue) } return operation } /** * Appends `operation`, in the form returned by `readOperation()`, to the columns in `outCols`. The * argument `inCols` provides metadata about the types of columns in `operation`; the value * `operation[i]` comes from the column `inCols[i]`. */ function appendOperation(outCols, inCols, operation) { let inIndex = 0, lastGroup = -1, lastCardinality = 0 for (let outCol of outCols) { while (inIndex < inCols.length && inCols[inIndex].columnId < outCol.columnId) inIndex++ if (inIndex < inCols.length && inCols[inIndex].columnId === outCol.columnId) { const colValue = operation[inIndex] if (outCol.columnId % 8 === COLUMN_TYPE.GROUP_CARD) { lastGroup = outCol.columnId >> 4 lastCardinality = colValue outCol.encoder.appendValue(colValue) } else if (outCol.columnId >> 4 === lastGroup) { if (!Array.isArray(colValue) || colValue.length !== lastCardinality) { throw new RangeError('bad group value') } for (let v of colValue) outCol.encoder.appendValue(v) } else if (outCol.columnId % 8 === COLUMN_TYPE.VALUE_RAW) { if (colValue) outCol.encoder.appendRawBytes(colValue) } else { outCol.encoder.appendValue(colValue) } } else if (outCol.columnId % 8 === COLUMN_TYPE.GROUP_CARD) { lastGroup = outCol.columnId >> 4 lastCardinality = 0 outCol.encoder.appendValue(0) } else if (outCol.columnId % 8 !== COLUMN_TYPE.VALUE_RAW) { const count = (outCol.columnId >> 4 === lastGroup) ? lastCardinality : 1 let blankValue = null if (outCol.columnId % 8 === COLUMN_TYPE.BOOLEAN) blankValue = false if (outCol.columnId % 8 === COLUMN_TYPE.VALUE_LEN) blankValue = 0 outCol.encoder.appendValue(blankValue, count) } } } /** * Parses the next operation from block `blockIndex` of the document. Returns an object of the form * `{docOp, blockIndex}` where `docOp` is an operation in the form returned by `readOperation()`, * and `blockIndex` is the block number to use on the next call (it moves on to the next block when * we reach the end of the current block). `docOp` is null if there are no more operations. */ function readNextDocOp(docState, blockIndex) { let block = docState.blocks[blockIndex] if (!block.columns[actionIdx].decoder.done) { return {docOp: readOperation(block.columns), blockIndex} } else if (blockIndex === docState.blocks.length - 1) { return {docOp: null, blockIndex} } else { blockIndex += 1 block = docState.blocks[blockIndex] for (let col of block.columns) col.decoder.reset() return {docOp: readOperation(block.columns), blockIndex} } } /** * Parses the next operation from a sequence of changes. `changeState` serves as the state of this * pseudo-iterator, and it is mutated to reflect the new operation. In particular, * `changeState.nextOp` is set to the operation that was read, and `changeState.done` is set to true * when we have finished reading the last operation in the last change. */ function readNextChangeOp(docState, changeState) { // If we've finished reading one change, move to the next change that contains at least one op while (changeState.changeIndex < changeState.changes.length - 1 && (!changeState.columns || changeState.columns[actionIdx].decoder.done)) { changeState.changeIndex += 1 const change = changeState.changes[changeState.changeIndex] changeState.columns = makeDecoders(change.columns, CHANGE_COLUMNS) changeState.opCtr = change.startOp // If it's an empty change (no ops), set its maxOp here since it won't be set below if (changeState.columns[actionIdx].decoder.done) { change.maxOp = change.startOp - 1 } // Update docState based on the information in the change updateBlockColumns(docState, changeState.columns) const {actorIds, actorTable} = getActorTable(docState.actorIds, change) docState.actorIds = actorIds changeState.actorTable = actorTable changeState.actorIndex = docState.actorIds.indexOf(change.actorIds[0]) } // Reached the end of the last change? if (changeState.columns[actionIdx].decoder.done) { changeState.done = true changeState.nextOp = null return } changeState.nextOp = readOperation(changeState.columns, changeState.actorTable) changeState.nextOp[idActorIdx] = changeState.actorIndex changeState.nextOp[idCtrIdx] = changeState.opCtr changeState.changes[changeState.changeIndex].maxOp = changeState.opCtr if (changeState.opCtr > docState.maxOp) docState.maxOp = changeState.opCtr changeState.opCtr += 1 const op = changeState.nextOp if ((op[objCtrIdx] === null && op[objActorIdx] !== null) || (op[objCtrIdx] !== null && op[objActorIdx] === null)) { throw new RangeError(`Mismatched object reference: (${op[objCtrIdx]}, ${op[objActorIdx]})`) } if ((op[keyCtrIdx] === null && op[keyActorIdx] !== null) || (op[keyCtrIdx] === 0 && op[keyActorIdx] !== null) || (op[keyCtrIdx] > 0 && op[keyActorIdx] === null)) { throw new RangeError(`Mismatched operation key: (${op[keyCtrIdx]}, ${op[keyActorIdx]})`) } } function emptyObjectPatch(objectId, type) { if (type === 'list' || type === 'text') { return {objectId, type, edits: []} } else { return {objectId, type, props: {}} } } /** * Returns true if the two given operation IDs have the same actor ID, and the counter of `id2` is * exactly `delta` greater than the counter of `id1`. */ function opIdDelta(id1, id2, delta = 1) { const parsed1 = parseOpId(id1), parsed2 = parseOpId(id2) return parsed1.actorId === parsed2.actorId && parsed1.counter + delta === parsed2.counter } /** * Appends a list edit operation (insert, update, remove) to an array of existing operations. If the * last existing operation can be extended (as a multi-op), we do that. */ function appendEdit(existingEdits, nextEdit) { if (existingEdits.length === 0) { existingEdits.push(nextEdit) return } let lastEdit = existingEdits[existingEdits.length - 1] if (lastEdit.action === 'insert' && nextEdit.action === 'insert' && lastEdit.index === nextEdit.index - 1 && lastEdit.value.type === 'value' && nextEdit.value.type === 'value' && lastEdit.elemId === lastEdit.opId && nextEdit.elemId === nextEdit.opId && opIdDelta(lastEdit.elemId, nextEdit.elemId, 1) && lastEdit.value.datatype === nextEdit.value.datatype && typeof lastEdit.value.value === typeof nextEdit.value.value) { lastEdit.action = 'multi-insert' if (nextEdit.value.datatype) lastEdit.datatype = nextEdit.value.datatype lastEdit.values = [lastEdit.value.value, nextEdit.value.value] delete lastEdit.value delete lastEdit.opId } else if (lastEdit.action === 'multi-insert' && nextEdit.action === 'insert' && lastEdit.index + lastEdit.values.length === nextEdit.index && nextEdit.value.type === 'value' && nextEdit.elemId === nextEdit.opId && opIdDelta(lastEdit.elemId, nextEdit.elemId, lastEdit.values.length) && lastEdit.datatype === nextEdit.value.datatype && typeof lastEdit.values[0] === typeof nextEdit.value.value) { lastEdit.values.push(nextEdit.value.value) } else if (lastEdit.action === 'remove' && nextEdit.action === 'remove' && lastEdit.index === nextEdit.index) { lastEdit.count += nextEdit.count } else { existingEdits.push(nextEdit) } } /** * `edits` is an array of (SingleInsertEdit | MultiInsertEdit | UpdateEdit | RemoveEdit) list edits * for a patch. This function appends an UpdateEdit to this array. A conflict is represented by * having several consecutive edits with the same index, and this can be realised by calling * `appendUpdate` several times for the same list element. On the first such call, `firstUpdate` * must be true. * * It is possible that coincidentally the previous edit (potentially arising from a different * change) is for the same index. If this is the case, to avoid accidentally treating consecutive * updates for the same index as a conflict, we remove the previous edit for the same index. This is * safe because the previous edit is overwritten by the new edit being appended, and we know that * it's for the same list elements because there are no intervening insertions/deletions that could * have changed the indexes. */ function appendUpdate(edits, index, elemId, opId, value, firstUpdate) { let insert = false if (firstUpdate) { // Pop all edits for the same index off the end of the edits array. This sequence may begin with // either an insert or an update. If it's an insert, we remember that fact, and use it below. while (!insert && edits.length > 0) { const lastEdit = edits[edits.length - 1] if ((lastEdit.action === 'insert' || lastEdit.action === 'update') && lastEdit.index === index) { edits.pop() insert = (lastEdit.action === 'insert') } else if (lastEdit.action === 'multi-insert' && lastEdit.index + lastEdit.values.length - 1 === index) { lastEdit.values.pop() insert = true } else { break } } } // If we popped an insert edit off the edits array, we need to turn the new update into an insert // in order to ensure the list element still gets inserted (just with a new value). if (insert) { appendEdit(edits, {action: 'insert', index, elemId, opId, value}) } else { appendEdit(edits, {action: 'update', index, opId, value}) } } /** * `edits` is an array of (SingleInsertEdit | MultiInsertEdit | UpdateEdit | RemoveEdit) list edits * for a patch. We assume that there is a suffix of this array that consists of an insertion at * position `index`, followed by zero or more UpdateEdits at the same index. This function rewrites * that suffix to be all updates instead. This is needed because sometimes when generating a patch * we think we are performing a list insertion, but then it later turns out that there was already * an existing value at that list element, and so we actually need to do an update, not an insert. * * If the suffix is preceded by one or more updates at the same index, those earlier updates are * removed by `appendUpdate()` to ensure we don't inadvertently treat them as part of the same * conflict. */ function convertInsertToUpdate(edits, index, elemId) { let updates = [] while (edits.length > 0) { let lastEdit = edits[edits.length - 1] if (lastEdit.action === 'insert') { if (lastEdit.index !== index) throw new RangeError('last edit has unexpected index') updates.unshift(edits.pop()) break } else if (lastEdit.action === 'update') { if (lastEdit.index !== index) throw new RangeError('last edit has unexpected index') updates.unshift(edits.pop()) } else { // It's impossible to encounter a remove edit here because the state machine in // updatePatchProperty() ensures that a property can have either an insert or a remove edit, // but not both. It's impossible to encounter a multi-insert here because multi-inserts always // have equal elemId and opId (i.e. they can only be used for the operation that first inserts // an element, but not for any subsequent assignments to that list element); moreover, // convertInsertToUpdate is only called if an insert action is followed by a non-overwritten // document op. The fact that there is a non-overwritten document op after another op on the // same list element implies that the original insertion op for that list element must be // overwritten, and thus the original insertion op cannot have given rise to a multi-insert. throw new RangeError('last edit has unexpected action') } } // Now take the edits we popped off and push them back onto the list again let firstUpdate = true for (let update of updates) { appendUpdate(edits, index, elemId, update.opId, update.value, firstUpdate) firstUpdate = false } } /** * Updates `patches` to reflect the operation `op` within the document with state `docState`. * Can be called multiple times if there are multiple operations for the same property (e.g. due * to a conflict). `propState` is an object that carries over state between such successive * invocations for the same property. If the current object is a list, `listIndex` is the index * into that list (counting only visible elements). If the operation `op` was already previously * in the document, `oldSuccNum` is the value of `op[succNumIdx]` before the current change was * applied (allowing us to determine whether this operation was overwritten or deleted in the * current change). `oldSuccNum` must be undefined if the operation came from the current change. * If we are creating an incremental patch as a result of applying one or more changes, `newBlock` * is the block to which the operations are getting written; we will update the metadata on this * block. `newBlock` should be null if we are creating a patch for the whole document. */ function updatePatchProperty(patches, newBlock, objectId, op, docState, propState, listIndex, oldSuccNum) { const isWholeDoc = !newBlock const type = op[actionIdx] < ACTIONS.length ? OBJECT_TYPE[ACTIONS[op[actionIdx]]] : null const opId = `${op[idCtrIdx]}@${docState.actorIds[op[idActorIdx]]}` const elemIdActor = op[insertIdx] ? op[idActorIdx] : op[keyActorIdx] const elemIdCtr = op[insertIdx] ? op[idCtrIdx] : op[keyCtrIdx] const elemId = op[keyStrIdx] ? op[keyStrIdx] : `${elemIdCtr}@${docState.actorIds[elemIdActor]}` // When the change contains a new make* operation (i.e. with an even-numbered action), record the // new parent-child relationship in objectMeta. TODO: also handle link/move operations. if (op[actionIdx] % 2 === 0 && !docState.objectMeta[opId]) { docState.objectMeta[opId] = {parentObj: objectId, parentKey: elemId, opId, type, children: {}} deepCopyUpdate(docState.objectMeta, [objectId, 'children', elemId, opId], {objectId: opId, type, props: {}}) } // firstOp is true if the current operation is the first of a sequence of ops for the same key const firstOp = !propState[elemId] if (!propState[elemId]) propState[elemId] = {visibleOps: [], hasChild: false} // An operation is overwritten if it is a document operation that has at least one successor const isOverwritten = (oldSuccNum !== undefined && op[succNumIdx] > 0) // Record all visible values for the property, and whether it has any child object if (!isOverwritten) { propState[elemId].visibleOps.push(op) propState[elemId].hasChild = propState[elemId].hasChild || (op[actionIdx] % 2) === 0 // even-numbered action == make* operation } // If one or more of the values of the property is a child object, we update objectMeta to store // all of the visible values of the property (even the non-child-object values). Then, when we // subsequently process an update within that child object, we can construct the patch to // contain the conflicting values. const prevChildren = docState.objectMeta[objectId].children[elemId] if (propState[elemId].hasChild || (prevChildren && Object.keys(prevChildren).length > 0)) { let values = {} for (let visible of propState[elemId].visibleOps) { const opId = `${visible[idCtrIdx]}@${docState.actorIds[visible[idActorIdx]]}` if (ACTIONS[visible[actionIdx]] === 'set') { values[opId] = Object.assign({type: 'value'}, decodeValue(visible[valLenIdx], visible[valRawIdx])) } else if (visible[actionIdx] % 2 === 0) { const objType = visible[actionIdx] < ACTIONS.length ? OBJECT_TYPE[ACTIONS[visible[actionIdx]]] : null values[opId] = emptyObjectPatch(opId, objType) } } // Copy so that objectMeta is not modified if an exception is thrown while applying change deepCopyUpdate(docState.objectMeta, [objectId, 'children', elemId], values) } let patchKey, patchValue // For counters, increment operations are succs to the set operation that created the counter, // but in this case we want to add the values rather than overwriting them. if (isOverwritten && ACTIONS[op[actionIdx]] === 'set' && (op[valLenIdx] & 0x0f) === VALUE_TYPE.COUNTER) { // This is the initial set operation that creates a counter. Initialise the counter state // to contain all successors of the set operation. Only if we later find that each of these // successor operations is an increment, we make the counter visible in the patch. if (!propState[elemId]) propState[elemId] = {visibleOps: [], hasChild: false} if (!propState[elemId].counterStates) propState[elemId].counterStates = {} let counterStates = propState[elemId].counterStates let counterState = {opId, value: decodeValue(op[valLenIdx], op[valRawIdx]).value, succs: {}} for (let i = 0; i < op[succNumIdx]; i++) { const succOp = `${op[succCtrIdx][i]}@${docState.actorIds[op[succActorIdx][i]]}` counterStates[succOp] = counterState counterState.succs[succOp] = true } } else if (ACTIONS[op[actionIdx]] === 'inc') { // Incrementing a previously created counter. if (!propState[elemId] || !propState[elemId].counterStates || !propState[elemId].counterStates[opId]) { throw new RangeError(`increment operation ${opId} for unknown counter`) } let counterState = propState[elemId].counterStates[opId] counterState.value += decodeValue(op[valLenIdx], op[valRawIdx]).value delete counterState.succs[opId] if (Object.keys(counterState.succs).length === 0) { patchKey = counterState.opId patchValue = {type: 'value', datatype: 'counter', value: counterState.value} // TODO if the counter is in a list element, we need to add a 'remove' action when deleted } } else if (!isOverwritten) { // Add the value to the patch if it is not overwritten (i.e. if it has no succs). if (ACTIONS[op[actionIdx]] === 'set') { patchKey = opId patchValue = Object.assign({type: 'value'}, decodeValue(op[valLenIdx], op[valRawIdx])) } else if (op[actionIdx] % 2 === 0) { // even-numbered action == make* operation if (!patches[opId]) patches[opId] = emptyObjectPatch(opId, type) patchKey = opId patchValue = patches[opId] } } if (!patches[objectId]) patches[objectId] = emptyObjectPatch(objectId, docState.objectMeta[objectId].type) const patch = patches[objectId] // Updating a list or text object (with elemId key) if (op[keyStrIdx] === null) { // If we come across any document op that was previously non-overwritten/non-deleted, that // means the current list element already had a value before this change was applied, and // therefore the current element cannot be an insert. If we already registered an insert, we // have to convert it into an update. if (oldSuccNum === 0 && !isWholeDoc && propState[elemId].action === 'insert') { propState[elemId].action = 'update' convertInsertToUpdate(patch.edits, listIndex, elemId) if (newBlock && newBlock.lastObjectActor === op[objActorIdx] && newBlock.lastObjectCtr === op[objCtrIdx]) { newBlock.numVisible -= 1 } } if (patchValue) { // If the op has a non-overwritten value and it came from the change, it's an insert. // (It's not necessarily the case that op[insertIdx] is true: if a list element is concurrently // deleted and updated, the node that first processes the deletion and then the update will // observe the update as a re-insertion of the deleted list element.) if (!propState[elemId].action && (oldSuccNum === undefined || isWholeDoc)) { propState[elemId].action = 'insert' appendEdit(patch.edits, {action: 'insert', index: listIndex, elemId, opId: patchKey, value: patchValue}) if (newBlock && newBlock.lastObjectActor === op[objActorIdx] && newBlock.lastObjectCtr === op[objCtrIdx]) { newBlock.numVisible += 1 } // If the property has a value and it's not an insert, then it must be an update. // We might have previously registered it as a remove, in which case we convert it to update. } else if (propState[elemId].action === 'remove') { let lastEdit = patch.edits[patch.edits.length - 1] if (lastEdit.action !== 'remove') throw new RangeError('last edit has unexpected type') if (lastEdit.count > 1) lastEdit.count -= 1; else patch.edits.pop() propState[elemId].action = 'update' appendUpdate(patch.edits, listIndex, elemId, patchKey, patchValue, true) if (newBlock && newBlock.lastObjectActor === op[objActorIdx] && newBlock.lastObjectCtr === op[objCtrIdx]) { newBlock.numVisible += 1 } } else { // A 'normal' update appendUpdate(patch.edits, listIndex, elemId, patchKey, patchValue, !propState[elemId].action) if (!propState[elemId].action) propState[elemId].action = 'update' } } else if (oldSuccNum === 0 && !propState[elemId].action) { // If the property used to have a non-overwritten/non-deleted value, but no longer, it's a remove propState[elemId].action = 'remove' appendEdit(patch.edits, {action: 'remove', index: listIndex, count: 1}) if (newBlock && newBlock.lastObjectActor === op[objActorIdx] && newBlock.lastObjectCtr === op[objCtrIdx]) { newBlock.numVisible -= 1 } } } else if (patchValue || !isWholeDoc) { // Updating a map or table (with string key) if (firstOp || !patch.props[op[keyStrIdx]]) patch.props[op[keyStrIdx]] = {} if (patchValue) patch.props[op[keyStrIdx]][patchKey] = patchValue } } /** * Applies operations (from one or more changes) to the document by merging the sequence of change * ops into the sequence of document ops. The two inputs are `changeState` and `docState` * respectively. Assumes that the decoders of both sets of columns are at the position where we want * to start merging. `patches` is mutated to reflect the effect of the change operations. `ops` is * the operation sequence to apply (as decoded by `groupRelatedOps()`). `docState` is as * documented in `applyOps()`. If the operations are updating a list or text object, `listIndex` * is the number of visible elements that precede the position at which we start merging. * `blockIndex` is the document block number from which we are currently reading. */ function mergeDocChangeOps(patches, newBlock, outCols, changeState, docState, listIndex, blockIndex) { const firstOp = changeState.nextOp, insert = firstOp[insertIdx] const objActor = firstOp[objActorIdx], objCtr = firstOp[objCtrIdx] const objectId = objActor === null ? '_root' : `${objCtr}@${docState.actorIds[objActor]}` const idActorIndex = changeState.actorIndex, idActor = docState.actorIds[idActorIndex] let foundListElem = false, elemVisible = false, propState = {}, docOp ;({ docOp, blockIndex } = readNextDocOp(docState, blockIndex)) let docOpsConsumed = (docOp === null ? 0 : 1) let docOpOldSuccNum = (docOp === null ? 0 : docOp[succNumIdx]) let changeOp = null, changeOps = [], changeCols = [], predSeen = [], lastChangeKey = null changeState.objectIds.add(objectId) // Merge the two inputs: the sequence of ops in the doc, and the sequence of ops in the change. // At each iteration, we either output the doc's op (possibly updated based on the change's ops) // or output an op from the change. while (true) { // The array `changeOps` contains operations from the change(s) we're applying. When the array // is empty, we load changes from the change. Typically we load only a single operation at a // time, with two exceptions: 1. all operations that update the same key or list element in the // same object are put into changeOps at the same time (this is needed so that we can update the // succ columns of the document ops correctly); 2. a run of consecutive insertions is also // placed into changeOps in one go. // // When we have processed all the ops in changeOps we try to see whether there are further // operations that we can also process while we're at it. Those operations must be for the same // object, they must be for a key or list element that appears later in the document, they must // either all be insertions or all be non-insertions, and if insertions, they must be // consecutive. If these conditions are satisfied, that means the operations can be processed in // the same pass. If we encounter an operation that does not meet these conditions, we leave // changeOps empty, and this function returns after having processed any remaining document ops. // // Any operations that could not be processed in a single pass remain in changeState; applyOps // will seek to the appropriate position and then call mergeDocChangeOps again. if (changeOps.length === 0) { foundListElem = false let nextOp = changeState.nextOp while (!changeState.done && nextOp[idActorIdx] === idActorIndex && nextOp[insertIdx] === insert && nextOp[objActorIdx] === firstOp[objActorIdx] && nextOp[objCtrIdx] === firstOp[objCtrIdx]) { // Check if the operation's pred references a previous operation in changeOps const lastOp = (changeOps.length > 0) ? changeOps[changeOps.length - 1] : null let isOverwrite = false for (let i = 0; i < nextOp[predNumIdx]; i++) { for (let prevOp of changeOps) { if (nextOp[predActorIdx][i] === prevOp[idActorIdx] && nextOp[predCtrIdx][i] === prevOp[idCtrIdx]) { isOverwrite = true } } } // If any of the following `if` statements is true, we add `nextOp` to `changeOps`. If they // are all false, we break out of the loop and stop adding to `changeOps`. if (nextOp === firstOp) { // First change operation in a mergeDocChangeOps call is always used } else if (insert && lastOp !== null && nextOp[keyStrIdx] === null && nextOp[keyActorIdx] === lastOp[idActorIdx] && nextOp[keyCtrIdx] === lastOp[idCtrIdx]) { // Collect consecutive insertions } else if (!insert && lastOp !== null && nextOp[keyStrIdx] !== null && nextOp[keyStrIdx] === lastOp[keyStrIdx] && !isOverwrite) { // Collect several updates to the same key } else if (!insert && lastOp !== null && nextOp[keyStrIdx] === null && lastOp[keyStrIdx] === null && nextOp[keyActorIdx] === lastOp[keyActorIdx] && nextOp[keyCtrIdx] === lastOp[keyCtrIdx] && !isOverwrite) { // Collect several updates to the same list element } else if (!insert && lastOp === null && nextOp[keyStrIdx] === null && docOp && docOp[insertIdx] && docOp[keyStrIdx] === null && docOp[idActorIdx] === nextOp[keyActorIdx] && docOp[idCtrIdx] === nextOp[keyCtrIdx]) { // When updating/deleting list elements, keep going if the next elemId in the change // equals the next elemId in the doc (i.e. we're updating several consecutive elements) } else if (!insert && lastOp === null && nextOp[keyStrIdx] !== null && lastChangeKey !== null && lastChangeKey < nextOp[keyStrIdx]) { // Allow a single mergeDocChangeOps call to process changes to several keys in the same // object, provided that they appear in ascending order } else break lastChangeKey = (nextOp !== null) ? nextOp[keyStrIdx] : null changeOps.push(changeState.nextOp) changeCols.push(changeState.columns) predSeen.push(new Array(changeState.nextOp[predNumIdx])) readNextChangeOp(docState, changeState) nextOp = changeState.nextOp } } if (changeOps.length > 0) changeOp = changeOps[0] const inCorrectObject = docOp && docOp[objActorIdx] === changeOp[objActorIdx] && docOp[objCtrIdx] === changeOp[objCtrIdx] const keyMatches = docOp && docOp[keyStrIdx] !== null && docOp[keyStrIdx] === changeOp[keyStrIdx] const listElemMatches = docOp && docOp[keyStrIdx] === null && changeOp[keyStrIdx] === null && ((!docOp[insertIdx] && docOp[keyActorIdx] === changeOp[keyActorIdx] && docOp[keyCtrIdx] === changeOp[keyCtrIdx]) || (docOp[insertIdx] && docOp[idActorIdx] === changeOp[keyActorIdx] && docOp[idCtrIdx] === changeOp[keyCtrIdx])) // We keep going until we run out of ops in the change, except that even when we run out, we // keep going until we have processed all doc ops for the current key/list element. if (changeOps.length === 0 && !(inCorrectObject && (keyMatches || listElemMatches))) break let takeDocOp = false, takeChangeOps = 0 // The change operations come first if we are inserting list elements (seekToOp already // determines the correct insertion position), if there is no document operation, if the next // document operation is for a different object, or if the change op's string key is // lexicographically first (TODO check ordering of keys beyond the basic multilingual plane). if (insert || !inCorrectObject || (docOp[keyStrIdx] === null && changeOp[keyStrIdx] !== null) || (docOp[keyStrIdx] !== null && changeOp[keyStrIdx] !== null && changeOp[keyStrIdx] < docOp[keyStrIdx])) { // Take the operations from the change takeChangeOps = changeOps.length if (!inCorrectObject && !foundListElem && changeOp[keyStrIdx] === null && !changeOp[insertIdx]) { // This can happen if we first update one list element, then another one earlier in the // list. That is not allowed: list element updates must occur in ascending order. throw new RangeError("could not find list element with ID: " + `${changeOp[keyCtrIdx]}@${docState.actorIds[changeOp[keyActorIdx]]}`) } } else if (keyMatches || listElemMatches || foundListElem) { // The doc operation is for the same key or list element in the same object as the change // ops, so we merge them. First, if any of the change ops' `pred` matches the opId of the // document operation, we update the document operation's `succ` accordingly. for (let opIndex = 0; opIndex < changeOps.length; opIndex++) { const op = changeOps[opIndex] for (let i = 0; i < op[predNumIdx]; i++) { if (op[predActorIdx][i] === docOp[idActorIdx] && op[predCtrIdx][i] === docOp[idCtrIdx]) { // Insert into the doc op's succ list such that the lists remains sorted let j = 0 while (j < docOp[succNumIdx] && (docOp[succCtrIdx][j] < op[idCtrIdx] || docOp[succCtrIdx][j] === op[idCtrIdx] && docState.actorIds[docOp[succActorIdx][j]] < idActor)) j++ docOp[succCtrIdx].splice(j, 0, op[idCtrIdx]) docOp[succActorIdx].splice(j, 0, idActorIndex) docOp[succNumIdx]++ predSeen[opIndex][i] = true break } } } if (listElemMatches) foundListElem = true if (foundListElem && !listElemMatches) { // If the previous docOp was for the correct list element, and the current docOp is for // the wrong list element, then place the current changeOp before the docOp. takeChangeOps = changeOps.length } else if (changeOps.length === 0 || docOp[idCtrIdx] < changeOp[idCtrIdx] || (docOp[idCtrIdx] === changeOp[idCtrIdx] && docState.actorIds[docOp[idActorIdx]] < idActor)) { // When we have several operations for the same object and the same key, we want to keep // them sorted in ascending order by opId. Here we have docOp with a lower opId, so we // output it first. takeDocOp = true updatePatchProperty(patches, newBlock, objectId, docOp, docState, propState, listIndex, docOpOldSuccNum) // A deletion op in the change is represented in the document only by its entries in the // succ list of the operations it overwrites; it has no separate row in the set of ops. for (let i = changeOps.length - 1; i >= 0; i--) { let deleted = true for (let j = 0; j < changeOps[i][predNumIdx]; j++) { if (!predSeen[i][j]) deleted = false } if (ACTIONS[changeOps[i][actionIdx]] === 'del' && deleted) { changeOps.splice(i, 1) changeCols.splice(i, 1) predSeen.splice(i, 1) } } } else if (docOp[idCtrIdx] === changeOp[idCtrIdx] && docState.actorIds[docOp[idActorIdx]] === idActor) { throw new RangeError(`duplicate operation ID: ${changeOp[idCtrIdx]}@${idActor}`) } else { // The changeOp has the lower opId, so we output it first. takeChangeOps = 1 } } else { // The document operation comes first if its string key is lexicographically first, or if // we're using opId keys and the keys don't match (i.e. we scan the document until we find a // matching key). takeDocOp = true } if (takeDocOp) { appendOperation(outCols, docState.blocks[blockIndex].columns, docOp) addBlockOperation(newBlock, docOp, docState.actorIds, false) if (docOp[insertIdx] && elemVisible) { elemVisible = false listIndex++ } if (docOp[succNumIdx] === 0) elemVisible = true newBlock.numOps++ ;({ docOp, blockIndex } = readNextDocOp(docState, blockIndex)) if (docOp !== null) { docOpsConsumed++ docOpOldSuccNum = docOp[succNumIdx] } } if (takeChangeOps > 0) { for (let i = 0; i < takeChangeOps; i++) { let op = changeOps[i] // Check that we've seen all ops mentioned in `pred` (they must all have lower opIds than // the change op's own opId, so we must have seen them already) for (let j = 0; j < op[predNumIdx]; j++) { if (!predSeen[i][j]) { throw new RangeError(`no matching operation for pred: ${op[predCtrIdx][j]}@${docState.actorIds[op[predActorIdx][j]]}`) } } appendOperation(outCols, changeCols[i], op) addBlockOperation(newBlock, op, docState.actorIds, true) updatePatchProperty(patches, newBlock, objectId, op, docState, propState, listIndex) if (op[insertIdx]) { elemVisible = false listIndex++ } else { elemVisible = true } } if (takeChangeOps === changeOps.length) { changeOps.length = 0 changeCols.length = 0 predSeen.length = 0 } else { changeOps.splice(0, takeChangeOps) changeCols.splice(0, takeChangeOps) predSeen.splice(0, takeChangeOps) } newBlock.numOps += takeChangeOps } } if (docOp) { appendOperation(outCols, docState.blocks[blockIndex].columns, docOp) newBlock.numOps++ addBlockOperation(newBlock, docOp, docState.actorIds, false) } return {docOpsConsumed, blockIndex} } /** * Applies operations from the change (or series of changes) in `changeState` to the document * `docState`. Passing `changeState` to `readNextChangeOp` allows iterating over the change ops. * `docState` is an object with keys: * - `actorIds` is an array of actorIds (as hex strings) occurring in the document (values in * the document's objActor/keyActor/idActor/... columns are indexes into this array). * - `blocks` is an array of all the blocks of operations in the document. * - `objectMeta` is a map from objectId to metadata about that object. * * `docState` is mutated to contain the updated document state. * `patches` is a patch object that is mutated to reflect the operations applied by this function. */ function applyOps(patches, changeState, docState) { const [objActorNum, objCtr, keyActorNum, keyCtr, keyStr, idActorNum, idCtr, insert] = changeState.nextOp const objActor = objActorNum === null ? null : docState.actorIds[objActorNum] const keyActor = keyActorNum === null ? null : docState.actorIds[keyActorNum] const ops = { objActor, objActorNum, objCtr, keyActor, keyActorNum, keyCtr, keyStr, idActor: docState.actorIds[idActorNum], idCtr, insert, objId: objActor === null ? '_root' : `${objCtr}@${objActor}` } const {blockIndex, skipCount, visibleCount} = seekToOp(docState, ops) const block = docState.blocks[blockIndex] for (let col of block.columns) col.decoder.reset() const resetFirstVisible = (skipCount === 0) || (block.firstVisibleActor === undefined) || (!insert && block.firstVisibleActor === keyActorNum && block.firstVisibleCtr === keyCtr) const newBlock = { columns: undefined, bloom: new Uint8Array(block.bloom), numOps: skipCount, lastKey: block.lastKey, numVisible: block.numVisible, lastObjectActor: block.lastObjectActor, lastObjectCtr: block.lastObjectCtr, firstVisibleActor: resetFirstVisible ? undefined : block.firstVisibleActor, firstVisibleCtr: resetFirstVisible ? undefined : block.firstVisibleCtr, lastVisibleActor: undefined, lastVisibleCtr: undefined } // Copy the operations up to the insertion position (the first skipCount operations) const outCols = block.columns.map(col => ({columnId: col.columnId, encoder: encoderByColumnId(col.columnId)})) copyColumns(outCols, block.columns, skipCount) // Apply the operations from the change. This may cause blockIndex to move forwards if the // property being updated straddles a block boundary. const {blockIndex: lastBlockIndex, docOpsConsumed} = mergeDocChangeOps(patches, newBlock, outCols, changeState, docState, visibleCount, blockIndex) // Copy the remaining operations after the insertion position const lastBlock = docState.blocks[lastBlockIndex] let copyAfterMerge = -skipCount - docOpsConsumed for (let i = blockIndex; i <= lastBlockIndex; i++) copyAfterMerge += docState.blocks[i].numOps copyColumns(outCols, lastBlock.columns, copyAfterMerge) newBlock.numOps += copyAfterMerge for (let col of lastBlock.columns) { if (!col.decoder.done) throw new RangeError(`excess ops in column ${col.columnId}`) } newBlock.columns = outCols.map(col => { const decoder = decoderByColumnId(col.columnId, col.encoder.buffer) return {columnId: col.columnId, decoder} }) if (blockIndex === lastBlockIndex && newBlock.numOps <= MAX_BLOCK_SIZE) { // The result is just one output block if (copyAfterMerge > 0 && block.lastVisibleActor !== undefined && block.lastVisibleCtr !== undefined) { // It's possible that none of the ops after the merge point are visible, in which case the // lastVisible may not be strictly correct, because it may refer to an operation before the // merge point rather than a list element inserted by the current change. However, this doesn't // matter, because the only purpose for which we need it is to check whether one block ends with // the same visible element as the next block starts with (to avoid double-counting its index); // if the last list element of a block is invisible, the exact value of lastVisible doesn't // matter since it will be different from the next block's firstVisible in any case. newBlock.lastVisibleActor = block.lastVisibleActor newBlock.lastVisibleCtr = block.lastVisibleCtr } docState.blocks[blockIndex] = newBlock } else { // Oversized output block must be split into smaller blocks const newBlocks = splitBlock(newBlock) docState.blocks.splice(blockIndex, lastBlockIndex - blockIndex + 1, ...newBlocks) } } /** * Updates the columns in a document's operation blocks to contain all the columns in a change * (including any column types we don't recognise, which have been generated by a future version * of Automerge). */ function updateBlockColumns(docState, changeCols) { // Check that the columns of a change appear at the index at which we expect them to be if (changeCols[objActorIdx ].columnId !== CHANGE_COLUMNS[objActorIdx ].columnId || CHANGE_COLUMNS[objActorIdx ].columnName !== 'objActor' || changeCols[objCtrIdx ].columnId !== CHANGE_COLUMNS[objCtrIdx ].columnId || CHANGE_COLUMNS[objCtrIdx ].columnName !== 'objCtr' || changeCols[keyActorIdx ].columnId !== CHANGE_COLUMNS[keyActorIdx ].columnId || CHANGE_COLUMNS[keyActorIdx ].columnName !== 'keyActor' || changeCols[keyCtrIdx ].columnId !== CHANGE_COLUMNS[keyCtrIdx ].columnId || CHANGE_COLUMNS[keyCtrIdx ].columnName !== 'keyCtr' || changeCols[keyStrIdx ].columnId !== CHANGE_COLUMNS[keyStrIdx ].columnId || CHANGE_COLUMNS[keyStrIdx ].columnName !== 'keyStr' || changeCols[idActorIdx ].columnId !== CHANGE_COLUMNS[idActorIdx ].columnId || CHANGE_COLUMNS[idActorIdx ].columnName !== 'idActor' || changeCols[idCtrIdx ].columnId !== CHANGE_COLUMNS[idCtrIdx ].columnId || CHANGE_COLUMNS[idCtrIdx ].columnName !== 'idCtr' || changeCols[insertIdx ].columnId !== CHANGE_COLUMNS[insertIdx ].columnId || CHANGE_COLUMNS[insertIdx ].columnName !== 'insert' || changeCols[actionIdx ].columnId !== CHANGE_COLUMNS[actionIdx ].columnId || CHANGE_COLUMNS[actionIdx ].columnName !== 'action' || changeCols[valLenIdx ].columnId !== CHANGE_COLUMNS[valLenIdx ].columnId || CHANGE_COLUMNS[valLenIdx ].columnName !== 'valLen' || changeCols[valRawIdx ].columnId !== CHANGE_COLUMNS[valRawIdx ].columnId || CHANGE_COLUMNS[valRawIdx ].columnName !== 'valRaw' || changeCols[predNumIdx ].columnId !== CHANGE_COLUMNS[predNumIdx ].columnId || CHANGE_COLUMNS[predNumIdx ].columnName !== 'predNum' || changeCols[predActorIdx].columnId !== CHANGE_COLUMNS[predActorIdx].columnId || CHANGE_COLUMNS[predActorIdx].columnName !== 'predActor' || changeCols[predCtrIdx ].columnId !== CHANGE_COLUMNS[predCtrIdx ].columnId || CHANGE_COLUMNS[predCtrIdx ].columnName !== 'predCtr') { throw new RangeError('unexpected columnId') } // Check if there any columns in the change that are not in the document, apart from pred* const docCols = docState.blocks[0].columns if (!changeCols.every(changeCol => PRED_COLUMN_IDS.includes(changeCol.columnId) || docCols.find(docCol => docCol.columnId === changeCol.columnId))) { let allCols = docCols.map(docCol => ({columnId: docCol.columnId})) for (let changeCol of changeCols) { const { columnId } = changeCol if (!PRED_COLUMN_IDS.includes(columnId) && !docCols.find(docCol => docCol.columnId === columnId)) { allCols.push({columnId}) } } allCols.sort((a, b) => a.columnId - b.columnId) for (let blockIndex = 0; blockIndex < docState.blocks.length; blockIndex++) { let block = copyObject(docState.blocks[blockIndex]) block.columns = makeDecoders(block.columns.map(col => ({columnId: col.columnId, buffer: col.decoder.buf})), allCols) docState.blocks[blockIndex] = block } } } /** * Takes a decoded change header, including an array of actorIds. Returns an object of the form * `{actorIds, actorTable}`, where `actorIds` is an updated array of actorIds appearing in the * document (including the new change's actorId). `actorTable` is an array of integers where * `actorTable[i]` contains the document's actor index for the actor that has index `i` in the * change (`i == 0` is the author of the change). */ function getActorTable(actorIds, change) { if (actorIds.indexOf(change.actorIds[0]) < 0) { if (change.seq !== 1) { throw new RangeError(`Seq ${change.seq} is the first change for actor ${change.actorIds[0]}`) } // Use concat, not push, so that the original array is not mutated actorIds = actorIds.concat([change.actorIds[0]]) } const actorTable = [] // translate from change's actor index to doc's actor index for (let actorId of change.actorIds) { const index = actorIds.indexOf(actorId) if (index < 0) { throw new RangeError(`actorId ${actorId} is not known to document`) } actorTable.push(index) } return {actorIds, actorTable} } /** * Finalises the patch for a change. `patches` is a map from objectIds to patch for that * particular object, `objectIds` is the array of IDs of objects that are created or updated in the * change, and `docState` is an object containing various bits of document state, including * `objectMeta`, a map from objectIds to metadata about that object (such as its parent in the * document tree). Mutates `patches` such that child objects are linked into their parent object, * all the way to the root object. */ function setupPatches(patches, objectIds, docState) { for (let objectId of objectIds) { let meta = docState.objectMeta[objectId], childMeta = null, patchExists = false while (true) { const hasChildren = childMeta && Object.keys(meta.children[childMeta.parentKey]).length > 0 if (!patches[objectId]) patches[objectId] = emptyObjectPatch(objectId, meta.type) if (childMeta && hasChildren) { if (meta.type === 'list' || meta.type === 'text') { // In list/text objects, parentKey is an elemID. First see if it already appears in an edit for (let edit of patches[objectId].edits) { if (edit.opId && meta.children[childMeta.parentKey][edit.opId]) { patchExists = true } } // If we need to add an edit, we first have to translate the elemId into an index if (!patchExists) { const obj = parseOpId(objectId), elem = parseOpId(childMeta.parentKey) const seekPos = { objActor: obj.actorId, objCtr: obj.counter, keyActor: elem.actorId, keyCtr: elem.counter, objActorNum: docState.actorIds.indexOf(obj.actorId), keyActorNum: docState.actorIds.indexOf(elem.actorId), keyStr: null, insert: false, objId: objectId } const { visibleCount } = seekToOp(docState, seekPos) for (let [opId, value] of Object.entries(meta.children[childMeta.parentKey])) { let patchValue = value if (value.objectId) { if (!patches[value.objectId]) patches[value.objectId] = emptyObjectPatch(value.objectId, value.type) patchValue = patches[value.objectId] } const edit = {action: 'update', index: visibleCount, opId, value: patchValue} appendEdit(patches[objectId].edits, edit) } } } else { // Non-list object: parentKey is the name of the property being updated (a string) if (!patches[objectId].props[childMeta.parentKey]) { patches[objectId].props[childMeta.parentKey] = {} } let values = patches[objectId].props[childMeta.parentKey] for (let [opId, value] of Object.entries(meta.children[childMeta.parentKey])) { if (values[opId]) { patchExists = true } else if (value.objectId) { if (!patches[value.objectId]) patches[value.objectId] = emptyObjectPatch(value.objectId, value.type) values[opId] = patches[value.objectId] } else { values[opId] = value } } } } if (patchExists || !meta.parentObj || (childMeta && !hasChildren)) break childMeta = meta objectId = meta.parentObj meta = docState.objectMeta[objectId] } } return patches } /** * Takes an array of decoded changes and applies them to a document. `docState` contains a bunch of * fields describing the document state. This function mutates `docState` to contain the updated * document state, and mutates `patches` to contain a patch to return to the frontend. Only the * top-level `docState` object is mutated; all nested objects within it are treated as immutable. * `objectIds` is mutated to contain the IDs of objects that are updated in any of the changes. * * The function detects duplicate changes that we've already applied by looking up each change's * hash in `docState.changeIndexByHash`. If we deferred the hash graph computation, that structure * will be incomplete, and we run the risk of applying the same change twice. However, we still have * the sequence numbers for detecting duplicates. If `throwExceptions` is true, we assume that the * set of change hashes is complete, and therefore a duplicate sequence number indicates illegal * behaviour. If `throwExceptions` is false, and we detect a possible sequence number reuse, we * don't throw an exception but instead enqueue all of the changes. This gives us a chance to * recompute the hash graph and eliminate duplicates before raising an error to the application. * * Returns a two-element array `[applied, enqueued]`, where `applied` is an array of changes that * have been applied to the document, and `enqueued` is an array of changes that have not yet been * applied because they are missing a dependency. */ function applyChanges(patches, decodedChanges, docState, objectIds, throwExceptions) { let heads = new Set(docState.heads), changeHashes = new Set() let clock = copyObject(docState.clock) let applied = [], enqueued = [] for (let change of decodedChanges) { // Skip any duplicate changes that we have already seen if (docState.changeIndexByHash[change.hash] !== undefined || changeHashes.has(change.hash)) continue const expectedSeq = (clock[change.actor] || 0) + 1 let causallyReady = true for (let dep of change.deps) { const depIndex = docState.changeIndexByHash[dep] if ((depIndex === undefined || depIndex === -1) && !changeHashes.has(dep)) { causallyReady = false } } if (!causallyReady) { enqueued.push(change) } else if (change.seq < expectedSeq) { if (throwExceptions) { throw new RangeError(`Reuse of sequence number ${change.seq} for actor ${change.actor}`) } else { return [[], decodedChanges] } } else if (change.seq > expectedSeq) { throw new RangeError(`Skipped sequence number ${expectedSeq} for actor ${change.actor}`) } else { clock[change.actor] = change.seq changeHashes.add(change.hash) for (let dep of change.deps) heads.delete(dep) heads.add(change.hash) applied.push(change) } } if (applied.length > 0) { let changeState = {changes: applied, changeIndex: -1, objectIds} readNextChangeOp(docState, changeState) while (!changeState.done) applyOps(patches, changeState, docState) docState.heads = [...heads].sort() docState.clock = clock } return [applied, enqueued] } /** * Scans the operations in a document and generates a patch that can be sent to the frontend to * instantiate the current state of the document. `objectMeta` is mutated to contain information * about the parent and children of each object in the document. */ function documentPatch(docState) { for (let col of docState.blocks[0].columns) col.decoder.reset() let propState = {}, docOp = null, blockIndex = 0 let patches = {_root: {objectId: '_root', type: 'map', props: {}}} let lastObjActor = null, lastObjCtr = null, objectId = '_root', elemVisible = false, listIndex = 0 while (true) { ({ docOp, blockIndex } = readNextDocOp(docState, blockIndex)) if (docOp === null) break if (docOp[objActorIdx] !== lastObjActor || docOp[objCtrIdx] !== lastObjCtr) { objectId = `${docOp[objCtrIdx]}@${docState.actorIds[docOp[objActorIdx]]}` lastObjActor = docOp[objActorIdx] lastObjCtr = docOp[objCtrIdx] propState = {} listIndex = 0 elemVisible = false } if (docOp[insertIdx] && elemVisible) { elemVisible = false listIndex++ } if (docOp[succNumIdx] === 0) elemVisible = true if (docOp[idCtrIdx] > docState.maxOp) docState.maxOp = docOp[idCtrIdx] for (let i = 0; i < docOp[succNumIdx]; i++) { if (docOp[succCtrIdx][i] > docState.maxOp) docState.maxOp = docOp[succCtrIdx][i] } updatePatchProperty(patches, null, objectId, docOp, docState, propState, listIndex, docOp[succNumIdx]) } return patches._root } /** * Takes an encoded document whose headers have been parsed using `decodeDocumentHeader()` and reads * from it the list of changes. Returns the document's current vector clock, i.e. an object mapping * each actor ID (as a hex string) to the number of changes seen from that actor. Also returns an * array of the actorIds whose most recent change has no dependents (i.e. the actors that * contributed the current heads of the document), and an array of encoders that has been * initialised to contain the columns of the changes list. */ function readDocumentChanges(doc) { const columns = makeDecoders(doc.changesColumns, DOCUMENT_COLUMNS) const actorD = columns[0].decoder, seqD = columns[1].decoder const depsNumD = columns[5].decoder, depsIndexD = columns[6].decoder if (columns[0].columnId !== DOCUMENT_COLUMNS[0].columnId || DOCUMENT_COLUMNS[0].columnName !== 'actor' || columns[1].columnId !== DOCUMENT_COLUMNS[1].columnId || DOCUMENT_COLUMNS[1].columnName !== 'seq' || columns[5].columnId !== DOCUMENT_COLUMNS[5].columnId || DOCUMENT_COLUMNS[5].columnName !== 'depsNum' || columns[6].columnId !== DOCUMENT_COLUMNS[6].columnId || DOCUMENT_COLUMNS[6].columnName !== 'depsIndex') { throw new RangeError('unexpected columnId') } let numChanges = 0, clock = {}, actorNums = [], headIndexes = new Set() while (!actorD.done) { const actorNum = actorD.readValue(), seq = seqD.readValue(), depsNum = depsNumD.readValue() const actorId = doc.actorIds[actorNum] if (seq !== 1 && seq !== clock[actorId] + 1) { throw new RangeError(`Expected seq ${clock[actorId] + 1}, got ${seq} for actor ${actorId}`) } actorNums.push(actorNum) clock[actorId] = seq headIndexes.add(numChanges) for (let j = 0; j < depsNum; j++) headIndexes.delete(depsIndexD.readValue()) numChanges++ } const headActors = [...headIndexes].map(index => doc.actorIds[actorNums[index]]).sort() for (let col of columns) col.decoder.reset() const encoders = columns.map(col => ({columnId: col.columnId, encoder: encoderByColumnId(col.columnId)})) copyColumns(encoders, columns, numChanges) return {clock, headActors, encoders, numChanges} } /** * Records the metadata about a change in the appropriate columns. */ function appendChange(columns, change, actorIds, changeIndexByHash) { appendOperation(columns, DOCUMENT_COLUMNS, [ actorIds.indexOf(change.actor), // actor change.seq, // seq change.maxOp, // maxOp change.time, // time change.message, // message change.deps.length, // depsNum change.deps.map(dep => changeIndexByHash[dep]), // depsIndex change.extraBytes ? (change.extraBytes.byteLength << 4 | VALUE_TYPE.BYTES) : VALUE_TYPE.BYTES, // extraLen change.extraBytes // extraRaw ]) } class BackendDoc { constructor(buffer) { this.maxOp = 0 this.haveHashGraph = false this.changes = [] this.changeIndexByHash = {} this.dependenciesByHash = {} this.dependentsByHash = {} this.hashesByActor = {} this.actorIds = [] this.heads = [] this.clock = {} this.queue = [] this.objectMeta = {_root: {parentObj: null, parentKey: null, opId: null, type: 'map', children: {}}} if (buffer) { const doc = decodeDocumentHeader(buffer) const {clock, headActors, encoders, numChanges} = readDocumentChanges(doc) this.binaryDoc = buffer this.changes = new Array(numChanges) this.actorIds = doc.actorIds this.heads = doc.heads this.clock = clock this.changesEncoders = encoders this.extraBytes = doc.extraBytes // If there is a single head, we can unambiguously point at the actorId and sequence number of // the head hash without having to reconstruct the hash graph if (doc.heads.length === 1 && headActors.length === 1) { this.hashesByActor[headActors[0]] = [] this.hashesByActor[headActors[0]][clock[headActors[0]] - 1] = doc.heads[0] } // The encoded document gives each change an index, and expresses dependencies in terms of // those indexes. Initialise the translation table from hash to index. if (doc.heads.length === doc.headsIndexes.length) { for (let i = 0; i < doc.heads.length; i++) { this.changeIndexByHash[doc.heads[i]] = doc.headsIndexes[i] } } else if (doc.heads.length === 1) { // If there is only one head, it must be the last change this.changeIndexByHash[doc.heads[0]] = numChanges - 1 } else { // We know the heads hashes, but not their indexes for (let head of doc.heads) this.changeIndexByHash[head] = -1 } this.blocks = [{columns: makeDecoders(doc.opsColumns, DOC_OPS_COLUMNS)}] updateBlockMetadata(this.blocks[0]) if (this.blocks[0].numOps > MAX_BLOCK_SIZE) { this.blocks = splitBlock(this.blocks[0]) } let docState = {blocks: this.blocks, actorIds: this.actorIds, objectMeta: this.objectMeta, maxOp: 0} this.initPatch = documentPatch(docState) this.maxOp = docState.maxOp } else { this.haveHashGraph = true this.changesEncoders = DOCUMENT_COLUMNS.map(col => ({columnId: col.columnId, encoder: encoderByColumnId(col.columnId)})) this.blocks = [{ columns: makeDecoders([], DOC_OPS_COLUMNS), bloom: new Uint8Array(BLOOM_FILTER_SIZE), numOps: 0, lastKey: undefined, numVisible: undefined, lastObjectActor: undefined, lastObjectCtr: undefined, firstVisibleActor: undefined, firstVisibleCtr: undefined, lastVisibleActor: undefined, lastVisibleCtr: undefined }] } } /** * Makes a copy of this BackendDoc that can be independently modified. */ clone() { if (!this.haveHashGraph) this.computeHashGraph() let copy = new BackendDoc() copy.maxOp = this.maxOp copy.haveHashGraph = this.haveHashGraph copy.changes = this.changes.slice() copy.changeIndexByHash = copyObject(this.changeIndexByHash) copy.dependenciesByHash = copyObject(this.dependenciesByHash) copy.dependentsByHash = Object.entries(this.dependentsByHash).reduce((acc, [k, v]) => { acc[k] = v.slice(); return acc }, {}) copy.hashesByActor = Object.entries(this.hashesByActor).reduce((acc, [k, v]) => { acc[k] = v.slice(); return acc }, {}) copy.actorIds = this.actorIds // immutable, no copying needed copy.heads = this.heads // immutable, no copying needed copy.clock = this.clock // immutable, no copying needed copy.blocks = this.blocks // immutable, no copying needed copy.objectMeta = this.objectMeta // immutable, no copying needed copy.queue = this.queue // immutable, no copying needed return copy } /** * Parses the changes given as Uint8Arrays in `changeBuffers`, and applies them to the current * document. Returns a patch to apply to the frontend. If an exception is thrown, the document * object is not modified. */ applyChanges(changeBuffers, isLocal = false) { if (changeBuffers instanceof Uint8Array) { throw new TypeError('applyChanges takes an array of Uint8Arrays, not just a single Uint8Array') } if (!Array.isArray(changeBuffers)) { throw new TypeError('applyChanges takes an array of Uint8Arrays') } // decoded change has the form { actor, seq, startOp, time, message, deps, actorIds, hash, columns, buffer } let decodedChanges = changeBuffers.map(buffer => { const decoded = decodeChangeColumns(buffer) decoded.buffer = buffer return decoded }) let patches = {_root: {objectId: '_root', type: 'map', props: {}}} let docState = { maxOp: this.maxOp, changeIndexByHash: this.changeIndexByHash, actorIds: this.actorIds, heads: this.heads, clock: this.clock, blocks: this.blocks.slice(), objectMeta: Object.assign({}, this.objectMeta) } let queue = (this.queue.length === 0) ? decodedChanges : decodedChanges.concat(this.queue) let allApplied = [], objectIds = new Set() while (true) { const [applied, enqueued] = applyChanges(patches, queue, docState, objectIds, this.haveHashGraph) queue = enqueued for (let i = 0; i < applied.length; i++) { docState.changeIndexByHash[applied[i].hash] = this.changes.length + allApplied.length + i } if (applied.length > 0) allApplied = allApplied.concat(applied) if (queue.length === 0) break // If we are missing a dependency, and we haven't computed the hash graph yet, first compute // the hashes to see if we actually have it already if (applied.length === 0) { if (this.haveHashGraph) break this.computeHashGraph() docState.changeIndexByHash = this.changeIndexByHash } } setupPatches(patches, objectIds, docState) // Update the document state only if `applyChanges` does not throw an exception for (let change of allApplied) { this.changes.push(change.buffer) if (!this.hashesByActor[change.actor]) this.hashesByActor[change.actor] = [] this.hashesByActor[change.actor][change.seq - 1] = change.hash this.changeIndexByHash[change.hash] = this.changes.length - 1 this.dependenciesByHash[change.hash] = change.deps this.dependentsByHash[change.hash] = [] for (let dep of change.deps) { if (!this.dependentsByHash[dep]) this.dependentsByHash[dep] = [] this.dependentsByHash[dep].push(change.hash) } appendChange(this.changesEncoders, change, docState.actorIds, this.changeIndexByHash) } this.maxOp = docState.maxOp this.actorIds = docState.actorIds this.heads = docState.heads this.clock = docState.clock this.blocks = docState.blocks this.objectMeta = docState.objectMeta this.queue = queue this.binaryDoc = null this.initPatch = null let patch = { maxOp: this.maxOp, clock: this.clock, deps: this.heads, pendingChanges: this.queue.length, diffs: patches._root } if (isLocal && decodedChanges.length === 1) { patch.actor = decodedChanges[0].actor patch.seq = decodedChanges[0].seq } return patch } /** * Reconstructs the full change history of a document, and initialises the variables that allow us * to traverse the hash graph of changes and their dependencies. When a compressed document is * loaded we defer the computation of this hash graph to make loading faster, but if the hash * graph is later needed (e.g. for the sync protocol), this function fills it in. */ computeHashGraph() { const binaryDoc = this.save() this.haveHashGraph = true this.changes = [] this.changeIndexByHash = {} this.dependenciesByHash = {} this.dependentsByHash = {} this.hashesByActor = {} this.clock = {} for (let change of decodeChanges([binaryDoc])) { const binaryChange = encodeChange(change) // TODO: avoid decoding and re-encoding again this.changes.push(binaryChange) this.changeIndexByHash[change.hash] = this.changes.length - 1 this.dependenciesByHash[change.hash] = change.deps this.dependentsByHash[change.hash] = [] for (let dep of change.deps) this.dependentsByHash[dep].push(change.hash) if (change.seq === 1) this.hashesByActor[change.actor] = [] this.hashesByActor[change.actor].push(change.hash) const expectedSeq = (this.clock[change.actor] || 0) + 1 if (change.seq !== expectedSeq) { throw new RangeError(`Expected seq ${expectedSeq}, got seq ${change.seq} from actor ${change.actor}`) } this.clock[change.actor] = change.seq } } /** * Returns all the changes that need to be sent to another replica. `haveDeps` is a list of change * hashes (as hex strings) of the heads that the other replica has. The changes in `haveDeps` and * any of their transitive dependencies will not be returned; any changes later than or concurrent * to the hashes in `haveDeps` will be returned. If `haveDeps` is an empty array, all changes are * returned. Throws an exception if any of the given hashes are not known to this replica. */ getChanges(haveDeps) { if (!this.haveHashGraph) this.computeHashGraph() // If the other replica has nothing, return all changes in history order if (haveDeps.length === 0) { return this.changes.slice() } // Fast path for the common case where all new changes depend only on haveDeps let stack = [], seenHashes = {}, toReturn = [] for (let hash of haveDeps) { seenHashes[hash] = true const successors = this.dependentsByHash[hash] if (!successors) throw new RangeError(`hash not found: ${hash}`) stack.push(...successors) } // Depth-first traversal of the hash graph to find all changes that depend on `haveDeps` while (stack.length > 0) { const hash = stack.pop() seenHashes[hash] = true toReturn.push(hash) if (!this.dependenciesByHash[hash].every(dep => seenHashes[dep])) { // If a change depends on a hash we have not seen, abort the traversal and fall back to the // slower algorithm. This will sometimes abort even if all new changes depend on `haveDeps`, // because our depth-first traversal is not necessarily a topological sort of the graph. break } stack.push(...this.dependentsByHash[hash]) } // If the traversal above has encountered all the heads, and was not aborted early due to // a missing dependency, then the set of changes it has found is complete, so we can return it if (stack.length === 0 && this.heads.every(head => seenHashes[head])) { return toReturn.map(hash => this.changes[this.changeIndexByHash[hash]]) } // If we haven't encountered all of the heads, we have to search harder. This will happen if // changes were added that are concurrent to `haveDeps` stack = haveDeps.slice() seenHashes = {} while (stack.length > 0) { const hash = stack.pop() if (!seenHashes[hash]) { const deps = this.dependenciesByHash[hash] if (!deps) throw new RangeError(`hash not found: ${hash}`) stack.push(...deps) seenHashes[hash] = true } } return this.changes.filter(change => !seenHashes[decodeChangeMeta(change, true).hash]) } /** * Returns all changes that are present in this BackendDoc, but not present in the `other` * BackendDoc. */ getChangesAdded(other) { if (!this.haveHashGraph) this.computeHashGraph() // Depth-first traversal from the heads through the dependency graph, // until we reach a change that is already present in opSet1 let stack = this.heads.slice(), seenHashes = {}, toReturn = [] while (stack.length > 0) { const hash = stack.pop() if (!seenHashes[hash] && other.changeIndexByHash[hash] === undefined) { seenHashes[hash] = true toReturn.push(hash) stack.push(...this.dependenciesByHash[hash]) } } // Return those changes in the reverse of the order in which the depth-first search // found them. This is not necessarily a topological sort, but should usually be close. return toReturn.reverse().map(hash => this.changes[this.changeIndexByHash[hash]]) } getChangeByHash(hash) { if (!this.haveHashGraph) this.computeHashGraph() return this.changes[this.changeIndexByHash[hash]] } /** * Returns the hashes of any missing dependencies, i.e. where we have tried to apply a change that * has a dependency on a change we have not seen. * * If the argument `heads` is given (an array of hexadecimal strings representing hashes as * returned by `getHeads()`), this function also ensures that all of those hashes resolve to * either a change that has been applied to the document, or that has been enqueued for later * application once missing dependencies have arrived. Any missing heads hashes are included in * the returned array. */ getMissingDeps(heads = []) { if (!this.haveHashGraph) this.computeHashGraph() let allDeps = new Set(heads), inQueue = new Set() for (let change of this.queue) { inQueue.add(change.hash) for (let dep of change.deps) allDeps.add(dep) } let missing = [] for (let hash of allDeps) { if (this.changeIndexByHash[hash] === undefined && !inQueue.has(hash)) missing.push(hash) } return missing.sort() } /** * Serialises the current document state into a single byte array. */ save() { if (this.binaryDoc) return this.binaryDoc // Getting the byte array for the changes columns finalises their encoders, after which we can // no longer append values to them. We therefore copy their data over to fresh encoders. const newEncoders = this.changesEncoders.map(col => ({columnId: col.columnId, encoder: encoderByColumnId(col.columnId)})) const decoders = this.changesEncoders.map(col => { const decoder = decoderByColumnId(col.columnId, col.encoder.buffer) return {columnId: col.columnId, decoder} }) copyColumns(newEncoders, decoders, this.changes.length) this.binaryDoc = encodeDocumentHeader({ changesColumns: this.changesEncoders, opsColumns: concatBlocks(this.blocks), actorIds: this.actorIds, // TODO: sort actorIds (requires transforming all actorId columns in opsColumns) heads: this.heads, headsIndexes: this.heads.map(hash => this.changeIndexByHash[hash]), extraBytes: this.extraBytes }) this.changesEncoders = newEncoders return this.binaryDoc } /** * Returns a patch from which we can initialise the current state of the backend. */ getPatch() { const objectMeta = {_root: {parentObj: null, parentKey: null, opId: null, type: 'map', children: {}}} const docState = {blocks: this.blocks, actorIds: this.actorIds, objectMeta, maxOp: 0} const diffs = this.initPatch ? this.initPatch : documentPatch(docState) return { maxOp: this.maxOp, clock: this.clock, deps: this.heads, pendingChanges: this.queue.length, diffs } } } module.exports = { MAX_BLOCK_SIZE, BackendDoc, bloomFilterContains } ================================================ FILE: backend/sync.js ================================================ /** * Implementation of the data synchronisation protocol that brings a local and a remote document * into the same state. This is typically used when two nodes have been disconnected for some time, * and need to exchange any changes that happened while they were disconnected. The two nodes that * are syncing could be client and server, or server and client, or two peers with symmetric roles. * * The protocol is based on this paper: Martin Kleppmann and Heidi Howard. Byzantine Eventual * Consistency and the Fundamental Limits of Peer-to-Peer Databases. https://arxiv.org/abs/2012.00472 * * The protocol assumes that every time a node successfully syncs with another node, it remembers * the current heads (as returned by `Backend.getHeads()`) after the last sync with that node. The * next time we try to sync with the same node, we start from the assumption that the other node's * document version is no older than the outcome of the last sync, so we only need to exchange any * changes that are more recent than the last sync. This assumption may not be true if the other * node did not correctly persist its state (perhaps it crashed before writing the result of the * last sync to disk), and we fall back to sending the entire document in this case. */ const Backend = require('./backend') const { hexStringToBytes, bytesToHexString, Encoder, Decoder } = require('./encoding') const { decodeChangeMeta } = require('./columnar') const { copyObject } = require('../src/common') const HASH_SIZE = 32 // 256 bits = 32 bytes const MESSAGE_TYPE_SYNC = 0x42 // first byte of a sync message, for identification const PEER_STATE_TYPE = 0x43 // first byte of an encoded peer state, for identification // These constants correspond to a 1% false positive rate. The values can be changed without // breaking compatibility of the network protocol, since the parameters used for a particular // Bloom filter are encoded in the wire format. const BITS_PER_ENTRY = 10, NUM_PROBES = 7 /** * A Bloom filter implementation that can be serialised to a byte array for transmission * over a network. The entries that are added are assumed to already be SHA-256 hashes, * so this implementation does not perform its own hashing. */ class BloomFilter { constructor (arg) { if (Array.isArray(arg)) { // arg is an array of SHA256 hashes in hexadecimal encoding this.numEntries = arg.length this.numBitsPerEntry = BITS_PER_ENTRY this.numProbes = NUM_PROBES this.bits = new Uint8Array(Math.ceil(this.numEntries * this.numBitsPerEntry / 8)) for (let hash of arg) this.addHash(hash) } else if (arg instanceof Uint8Array) { if (arg.byteLength === 0) { this.numEntries = 0 this.numBitsPerEntry = 0 this.numProbes = 0 this.bits = arg } else { const decoder = new Decoder(arg) this.numEntries = decoder.readUint32() this.numBitsPerEntry = decoder.readUint32() this.numProbes = decoder.readUint32() this.bits = decoder.readRawBytes(Math.ceil(this.numEntries * this.numBitsPerEntry / 8)) } } else { throw new TypeError('invalid argument') } } /** * Returns the Bloom filter state, encoded as a byte array. */ get bytes() { if (this.numEntries === 0) return new Uint8Array(0) const encoder = new Encoder() encoder.appendUint32(this.numEntries) encoder.appendUint32(this.numBitsPerEntry) encoder.appendUint32(this.numProbes) encoder.appendRawBytes(this.bits) return encoder.buffer } /** * Given a SHA-256 hash (as hex string), returns an array of probe indexes indicating which bits * in the Bloom filter need to be tested or set for this particular entry. We do this by * interpreting the first 12 bytes of the hash as three little-endian 32-bit unsigned integers, * and then using triple hashing to compute the probe indexes. The algorithm comes from: * * Peter C. Dillinger and Panagiotis Manolios. Bloom Filters in Probabilistic Verification. * 5th International Conference on Formal Methods in Computer-Aided Design (FMCAD), November 2004. * http://www.ccis.northeastern.edu/home/pete/pub/bloom-filters-verification.pdf */ getProbes(hash) { const hashBytes = hexStringToBytes(hash), modulo = 8 * this.bits.byteLength if (hashBytes.byteLength !== 32) throw new RangeError(`Not a 256-bit hash: ${hash}`) // on the next three lines, the right shift means interpret value as unsigned let x = ((hashBytes[0] | hashBytes[1] << 8 | hashBytes[2] << 16 | hashBytes[3] << 24) >>> 0) % modulo let y = ((hashBytes[4] | hashBytes[5] << 8 | hashBytes[6] << 16 | hashBytes[7] << 24) >>> 0) % modulo let z = ((hashBytes[8] | hashBytes[9] << 8 | hashBytes[10] << 16 | hashBytes[11] << 24) >>> 0) % modulo const probes = [x] for (let i = 1; i < this.numProbes; i++) { x = (x + y) % modulo y = (y + z) % modulo probes.push(x) } return probes } /** * Sets the Bloom filter bits corresponding to a given SHA-256 hash (given as hex string). */ addHash(hash) { for (let probe of this.getProbes(hash)) { this.bits[probe >>> 3] |= 1 << (probe & 7) } } /** * Tests whether a given SHA-256 hash (given as hex string) is contained in the Bloom filter. */ containsHash(hash) { if (this.numEntries === 0) return false for (let probe of this.getProbes(hash)) { if ((this.bits[probe >>> 3] & (1 << (probe & 7))) === 0) { return false } } return true } } /** * Encodes a sorted array of SHA-256 hashes (as hexadecimal strings) into a byte array. */ function encodeHashes(encoder, hashes) { if (!Array.isArray(hashes)) throw new TypeError('hashes must be an array') encoder.appendUint32(hashes.length) for (let i = 0; i < hashes.length; i++) { if (i > 0 && hashes[i - 1] >= hashes[i]) throw new RangeError('hashes must be sorted') const bytes = hexStringToBytes(hashes[i]) if (bytes.byteLength !== HASH_SIZE) throw new TypeError('heads hashes must be 256 bits') encoder.appendRawBytes(bytes) } } /** * Decodes a byte array in the format returned by encodeHashes(), and returns its content as an * array of hex strings. */ function decodeHashes(decoder) { let length = decoder.readUint32(), hashes = [] for (let i = 0; i < length; i++) { hashes.push(bytesToHexString(decoder.readRawBytes(HASH_SIZE))) } return hashes } /** * Takes a sync message of the form `{heads, need, have, changes}` and encodes it as a byte array for * transmission. */ function encodeSyncMessage(message) { const encoder = new Encoder() encoder.appendByte(MESSAGE_TYPE_SYNC) encodeHashes(encoder, message.heads) encodeHashes(encoder, message.need) encoder.appendUint32(message.have.length) for (let have of message.have) { encodeHashes(encoder, have.lastSync) encoder.appendPrefixedBytes(have.bloom) } encoder.appendUint32(message.changes.length) for (let change of message.changes) { encoder.appendPrefixedBytes(change) } return encoder.buffer } /** * Takes a binary-encoded sync message and decodes it into the form `{heads, need, have, changes}`. */ function decodeSyncMessage(bytes) { const decoder = new Decoder(bytes) const messageType = decoder.readByte() if (messageType !== MESSAGE_TYPE_SYNC) { throw new RangeError(`Unexpected message type: ${messageType}`) } const heads = decodeHashes(decoder) const need = decodeHashes(decoder) const haveCount = decoder.readUint32() let message = {heads, need, have: [], changes: []} for (let i = 0; i < haveCount; i++) { const lastSync = decodeHashes(decoder) const bloom = decoder.readPrefixedBytes(decoder) message.have.push({lastSync, bloom}) } const changeCount = decoder.readUint32() for (let i = 0; i < changeCount; i++) { const change = decoder.readPrefixedBytes() message.changes.push(change) } // Ignore any trailing bytes -- they can be used for extensions by future versions of the protocol return message } /** * Takes a SyncState and encodes as a byte array those parts of the state that should persist across * an application restart or disconnect and reconnect. The ephemeral parts of the state that should * be cleared on reconnect are not encoded. */ function encodeSyncState(syncState) { const encoder = new Encoder() encoder.appendByte(PEER_STATE_TYPE) encodeHashes(encoder, syncState.sharedHeads) return encoder.buffer } /** * Takes a persisted peer state as encoded by `encodeSyncState` and decodes it into a SyncState * object. The parts of the peer state that were not encoded are initialised with default values. */ function decodeSyncState(bytes) { const decoder = new Decoder(bytes) const recordType = decoder.readByte() if (recordType !== PEER_STATE_TYPE) { throw new RangeError(`Unexpected record type: ${recordType}`) } const sharedHeads = decodeHashes(decoder) return Object.assign(initSyncState(), { sharedHeads }) } /** * Constructs a Bloom filter containing all changes that are not one of the hashes in * `lastSync` or its transitive dependencies. In other words, the filter contains those * changes that have been applied since the version identified by `lastSync`. Returns * an object of the form `{lastSync, bloom}` as required for the `have` field of a sync * message. */ function makeBloomFilter(backend, lastSync) { const newChanges = Backend.getChanges(backend, lastSync) const hashes = newChanges.map(change => decodeChangeMeta(change, true).hash) return {lastSync, bloom: new BloomFilter(hashes).bytes} } /** * Call this function when a sync message is received from another node. The `message` argument * needs to already have been decoded using `decodeSyncMessage()`. This function determines the * changes that we need to send to the other node in response. Returns an array of changes (as * byte arrays). */ function getChangesToSend(backend, have, need) { if (have.length === 0) { return need.map(hash => Backend.getChangeByHash(backend, hash)).filter(change => change !== undefined) } let lastSyncHashes = {}, bloomFilters = [] for (let h of have) { for (let hash of h.lastSync) lastSyncHashes[hash] = true bloomFilters.push(new BloomFilter(h.bloom)) } // Get all changes that were added since the last sync const changes = Backend.getChanges(backend, Object.keys(lastSyncHashes)) .map(change => decodeChangeMeta(change, true)) let changeHashes = {}, dependents = {}, hashesToSend = {} for (let change of changes) { changeHashes[change.hash] = true // For each change, make a list of changes that depend on it for (let dep of change.deps) { if (!dependents[dep]) dependents[dep] = [] dependents[dep].push(change.hash) } // Exclude any change hashes contained in one or more Bloom filters if (bloomFilters.every(bloom => !bloom.containsHash(change.hash))) { hashesToSend[change.hash] = true } } // Include any changes that depend on a Bloom-negative change let stack = Object.keys(hashesToSend) while (stack.length > 0) { const hash = stack.pop() if (dependents[hash]) { for (let dep of dependents[hash]) { if (!hashesToSend[dep]) { hashesToSend[dep] = true stack.push(dep) } } } } // Include any explicitly requested changes let changesToSend = [] for (let hash of need) { hashesToSend[hash] = true if (!changeHashes[hash]) { // Change is not among those returned by getMissingChanges()? const change = Backend.getChangeByHash(backend, hash) if (change) changesToSend.push(change) } } // Return changes in the order they were returned by getMissingChanges() for (let change of changes) { if (hashesToSend[change.hash]) changesToSend.push(change.change) } return changesToSend } function initSyncState() { return { sharedHeads: [], lastSentHeads: [], theirHeads: null, theirNeed: null, theirHave: null, sentHashes: {}, } } function compareArrays(a, b) { return (a.length === b.length) && a.every((v, i) => v === b[i]) } /** * Given a backend and what we believe to be the state of our peer, generate a message which tells * them about we have and includes any changes we believe they need */ function generateSyncMessage(backend, syncState) { if (!backend) { throw new Error("generateSyncMessage called with no Automerge document") } if (!syncState) { throw new Error("generateSyncMessage requires a syncState, which can be created with initSyncState()") } let { sharedHeads, lastSentHeads, theirHeads, theirNeed, theirHave, sentHashes } = syncState const ourHeads = Backend.getHeads(backend) // Hashes to explicitly request from the remote peer: any missing dependencies of unapplied // changes, and any of the remote peer's heads that we don't know about const ourNeed = Backend.getMissingDeps(backend, theirHeads || []) // There are two reasons why ourNeed may be nonempty: 1. we might be missing dependencies due to // Bloom filter false positives; 2. we might be missing heads that the other peer mentioned // because they (intentionally) only sent us a subset of changes. In case 1, we leave the `have` // field of the message empty because we just want to fill in the missing dependencies for now. // In case 2, or if ourNeed is empty, we send a Bloom filter to request any unsent changes. let ourHave = [] if (!theirHeads || ourNeed.every(hash => theirHeads.includes(hash))) { ourHave = [makeBloomFilter(backend, sharedHeads)] } // Fall back to a full re-sync if the sender's last sync state includes hashes // that we don't know. This could happen if we crashed after the last sync and // failed to persist changes that the other node already sent us. if (theirHave && theirHave.length > 0) { const lastSync = theirHave[0].lastSync if (!lastSync.every(hash => Backend.getChangeByHash(backend, hash))) { // we need to queue them to send us a fresh sync message, the one they sent is uninteligible so we don't know what they need const resetMsg = {heads: ourHeads, need: [], have: [{ lastSync: [], bloom: new Uint8Array(0) }], changes: []} return [syncState, encodeSyncMessage(resetMsg)] } } // XXX: we should limit ourselves to only sending a subset of all the messages, probably limited by a total message size // these changes should ideally be RLE encoded but we haven't implemented that yet. let changesToSend = Array.isArray(theirHave) && Array.isArray(theirNeed) ? getChangesToSend(backend, theirHave, theirNeed) : [] // If the heads are equal, we're in sync and don't need to do anything further const headsUnchanged = Array.isArray(lastSentHeads) && compareArrays(ourHeads, lastSentHeads) const headsEqual = Array.isArray(theirHeads) && compareArrays(ourHeads, theirHeads) if (headsUnchanged && headsEqual && changesToSend.length === 0) { // no need to send a sync message if we know we're synced! return [syncState, null] } // TODO: this recomputes the SHA-256 hash of each change; we should restructure this to avoid the // unnecessary recomputation changesToSend = changesToSend.filter(change => !sentHashes[decodeChangeMeta(change, true).hash]) // Regular response to a sync message: send any changes that the other node // doesn't have. We leave the "have" field empty because the previous message // generated by `syncStart` already indicated what changes we have. const syncMessage = {heads: ourHeads, have: ourHave, need: ourNeed, changes: changesToSend} if (changesToSend.length > 0) { sentHashes = copyObject(sentHashes) for (const change of changesToSend) { sentHashes[decodeChangeMeta(change, true).hash] = true } } syncState = Object.assign({}, syncState, {lastSentHeads: ourHeads, sentHashes}) return [syncState, encodeSyncMessage(syncMessage)] } /** * Computes the heads that we share with a peer after we have just received some changes from that * peer and applied them. This may not be sufficient to bring our heads in sync with the other * peer's heads, since they may have only sent us a subset of their outstanding changes. * * `myOldHeads` are the local heads before the most recent changes were applied, `myNewHeads` are * the local heads after those changes were applied, and `ourOldSharedHeads` is the previous set of * shared heads. Applying the changes will have replaced some heads with others, but some heads may * have remained unchanged (because they are for branches on which no changes have been added). Any * such unchanged heads remain in the sharedHeads. Any sharedHeads that were replaced by applying * changes are also replaced as sharedHeads. This is safe because if we received some changes from * another peer, that means that peer had those changes, and therefore we now both know about them. */ function advanceHeads(myOldHeads, myNewHeads, ourOldSharedHeads) { const newHeads = myNewHeads.filter((head) => !myOldHeads.includes(head)) const commonHeads = ourOldSharedHeads.filter((head) => myNewHeads.includes(head)) const advancedHeads = [...new Set([...newHeads, ...commonHeads])].sort() return advancedHeads } /** * Given a backend, a message message and the state of our peer, apply any changes, update what * we believe about the peer, and (if there were applied changes) produce a patch for the frontend */ function receiveSyncMessage(backend, oldSyncState, binaryMessage) { if (!backend) { throw new Error("generateSyncMessage called with no Automerge document") } if (!oldSyncState) { throw new Error("generateSyncMessage requires a syncState, which can be created with initSyncState()") } let { sharedHeads, lastSentHeads, sentHashes } = oldSyncState, patch = null const message = decodeSyncMessage(binaryMessage) const beforeHeads = Backend.getHeads(backend) // If we received changes, we try to apply them to the document. There may still be missing // dependencies due to Bloom filter false positives, in which case the backend will enqueue the // changes without applying them. The set of changes may also be incomplete if the sender decided // to break a large set of changes into chunks. if (message.changes.length > 0) { [backend, patch] = Backend.applyChanges(backend, message.changes) sharedHeads = advanceHeads(beforeHeads, Backend.getHeads(backend), sharedHeads) } // If heads are equal, indicate we don't need to send a response message if (message.changes.length === 0 && compareArrays(message.heads, beforeHeads)) { lastSentHeads = message.heads } // If all of the remote heads are known to us, that means either our heads are equal, or we are // ahead of the remote peer. In this case, take the remote heads to be our shared heads. const knownHeads = message.heads.filter(head => Backend.getChangeByHash(backend, head)) if (knownHeads.length === message.heads.length) { sharedHeads = message.heads // If the remote peer has lost all its data, reset our state to perform a full resync if (message.heads.length === 0) { lastSentHeads = [] sentHashes = [] } } else { // If some remote heads are unknown to us, we add all the remote heads we know to // sharedHeads, but don't remove anything from sharedHeads. This might cause sharedHeads to // contain some redundant hashes (where one hash is actually a transitive dependency of // another), but this will be cleared up as soon as we know all the remote heads. sharedHeads = [...new Set(knownHeads.concat(sharedHeads))].sort() } const syncState = { sharedHeads, // what we have in common to generate an efficient bloom filter lastSentHeads, theirHave: message.have, // the information we need to calculate the changes they need theirHeads: message.heads, theirNeed: message.need, sentHashes } return [backend, syncState, patch] } module.exports = { receiveSyncMessage, generateSyncMessage, encodeSyncMessage, decodeSyncMessage, initSyncState, encodeSyncState, decodeSyncState, BloomFilter // BloomFilter is a private API, exported only for testing purposes } ================================================ FILE: backend/util.js ================================================ function backendState(backend) { if (backend.frozen) { throw new Error( 'Attempting to use an outdated Automerge document that has already been updated. ' + 'Please use the latest document state, or call Automerge.clone() if you really ' + 'need to use this old document state.' ) } return backend.state } module.exports = { backendState } ================================================ FILE: frontend/apply_patch.js ================================================ const { isObject, copyObject, parseOpId } = require('../src/common') const { OBJECT_ID, CONFLICTS, ELEM_IDS } = require('./constants') const { instantiateText } = require('./text') const { instantiateTable } = require('./table') const { Counter } = require('./counter') /** * Reconstructs the value from the patch object `patch`. */ function getValue(patch, object, updated) { if (patch.objectId) { // If the objectId of the existing object does not match the objectId in the patch, // that means the patch is replacing the object with a new one made from scratch if (object && object[OBJECT_ID] !== patch.objectId) { object = undefined } return interpretPatch(patch, object, updated) } else if (patch.datatype === 'timestamp') { // Timestamp: value is milliseconds since 1970 epoch return new Date(patch.value) } else if (patch.datatype === 'counter') { return new Counter(patch.value) } else { // Primitive value (int, uint, float64, string, boolean, or null) return patch.value } } /** * Compares two strings, interpreted as Lamport timestamps of the form * 'counter@actorId'. Returns 1 if ts1 is greater, or -1 if ts2 is greater. */ function lamportCompare(ts1, ts2) { const regex = /^(\d+)@(.*)$/ const time1 = regex.test(ts1) ? parseOpId(ts1) : {counter: 0, actorId: ts1} const time2 = regex.test(ts2) ? parseOpId(ts2) : {counter: 0, actorId: ts2} if (time1.counter < time2.counter) return -1 if (time1.counter > time2.counter) return 1 if (time1.actorId < time2.actorId) return -1 if (time1.actorId > time2.actorId) return 1 return 0 } /** * `props` is an object of the form: * `{key1: {opId1: {...}, opId2: {...}}, key2: {opId3: {...}}}` * where the outer object is a mapping from property names to inner objects, * and the inner objects are a mapping from operation ID to sub-patch. * This function interprets that structure and updates the objects `object` and * `conflicts` to reflect it. For each key, the greatest opId (by Lamport TS * order) is chosen as the default resolution; that op's value is assigned * to `object[key]`. Moreover, all the opIds and values are packed into a * conflicts object of the form `{opId1: value1, opId2: value2}` and assigned * to `conflicts[key]`. If there is no conflict, the conflicts object contains * just a single opId-value mapping. */ function applyProperties(props, object, conflicts, updated) { if (!props) return for (let key of Object.keys(props)) { const values = {}, opIds = Object.keys(props[key]).sort(lamportCompare).reverse() for (let opId of opIds) { const subpatch = props[key][opId] if (conflicts[key] && conflicts[key][opId]) { values[opId] = getValue(subpatch, conflicts[key][opId], updated) } else { values[opId] = getValue(subpatch, undefined, updated) } } if (opIds.length === 0) { delete object[key] delete conflicts[key] } else { object[key] = values[opIds[0]] conflicts[key] = values } } } /** * Creates a writable copy of an immutable map object. If `originalObject` * is undefined, creates an empty object with ID `objectId`. */ function cloneMapObject(originalObject, objectId) { const object = copyObject(originalObject) const conflicts = copyObject(originalObject ? originalObject[CONFLICTS] : undefined) Object.defineProperty(object, OBJECT_ID, {value: objectId}) Object.defineProperty(object, CONFLICTS, {value: conflicts}) return object } /** * Updates the map object `obj` according to the modifications described in * `patch`, or creates a new object if `obj` is undefined. Mutates `updated` * to map the objectId to the new object, and returns the new object. */ function updateMapObject(patch, obj, updated) { const objectId = patch.objectId if (!updated[objectId]) { updated[objectId] = cloneMapObject(obj, objectId) } const object = updated[objectId] applyProperties(patch.props, object, object[CONFLICTS], updated) return object } /** * Updates the table object `obj` according to the modifications described in * `patch`, or creates a new object if `obj` is undefined. Mutates `updated` * to map the objectId to the new object, and returns the new object. */ function updateTableObject(patch, obj, updated) { const objectId = patch.objectId if (!updated[objectId]) { updated[objectId] = obj ? obj._clone() : instantiateTable(objectId) } const object = updated[objectId] for (let key of Object.keys(patch.props || {})) { const opIds = Object.keys(patch.props[key]) if (opIds.length === 0) { object.remove(key) } else if (opIds.length === 1) { const subpatch = patch.props[key][opIds[0]] object._set(key, getValue(subpatch, object.byId(key), updated), opIds[0]) } else { throw new RangeError('Conflicts are not supported on properties of a table') } } return object } /** * Creates a writable copy of an immutable list object. If `originalList` is * undefined, creates an empty list with ID `objectId`. */ function cloneListObject(originalList, objectId) { const list = originalList ? originalList.slice() : [] // slice() makes a shallow clone const conflicts = (originalList && originalList[CONFLICTS]) ? originalList[CONFLICTS].slice() : [] const elemIds = (originalList && originalList[ELEM_IDS]) ? originalList[ELEM_IDS].slice() : [] Object.defineProperty(list, OBJECT_ID, {value: objectId}) Object.defineProperty(list, CONFLICTS, {value: conflicts}) Object.defineProperty(list, ELEM_IDS, {value: elemIds}) return list } /** * Updates the list object `obj` according to the modifications described in * `patch`, or creates a new object if `obj` is undefined. Mutates `updated` * to map the objectId to the new object, and returns the new object. */ function updateListObject(patch, obj, updated) { const objectId = patch.objectId if (!updated[objectId]) { updated[objectId] = cloneListObject(obj, objectId) } const list = updated[objectId], conflicts = list[CONFLICTS], elemIds = list[ELEM_IDS] for (let i = 0; i < patch.edits.length; i++) { const edit = patch.edits[i] if (edit.action === 'insert' || edit.action === 'update') { const oldValue = conflicts[edit.index] && conflicts[edit.index][edit.opId] let lastValue = getValue(edit.value, oldValue, updated) let values = {[edit.opId]: lastValue} // Successive updates for the same index are an indication of a conflict on that list element. // Edits are sorted in increasing order by Lamport timestamp, so the last value (with the // greatest timestamp) is the default resolution of the conflict. while (i < patch.edits.length - 1 && patch.edits[i + 1].index === edit.index && patch.edits[i + 1].action === 'update') { i++ const conflict = patch.edits[i] const oldValue2 = conflicts[conflict.index] && conflicts[conflict.index][conflict.opId] lastValue = getValue(conflict.value, oldValue2, updated) values[conflict.opId] = lastValue } if (edit.action === 'insert') { list.splice(edit.index, 0, lastValue) conflicts.splice(edit.index, 0, values) elemIds.splice(edit.index, 0, edit.elemId) } else { list[edit.index] = lastValue conflicts[edit.index] = values } } else if (edit.action === 'multi-insert') { const startElemId = parseOpId(edit.elemId), newElems = [], newValues = [], newConflicts = [] const datatype = edit.datatype edit.values.forEach((value, index) => { const elemId = `${startElemId.counter + index}@${startElemId.actorId}` value = getValue({ value, datatype }, undefined, updated) newValues.push(value) newConflicts.push({[elemId]: {value, datatype, type: 'value'}}) newElems.push(elemId) }) list.splice(edit.index, 0, ...newValues) conflicts.splice(edit.index, 0, ...newConflicts) elemIds.splice(edit.index, 0, ...newElems) } else if (edit.action === 'remove') { list.splice(edit.index, edit.count) conflicts.splice(edit.index, edit.count) elemIds.splice(edit.index, edit.count) } } return list } /** * Updates the text object `obj` according to the modifications described in * `patch`, or creates a new object if `obj` is undefined. Mutates `updated` * to map the objectId to the new object, and returns the new object. */ function updateTextObject(patch, obj, updated) { const objectId = patch.objectId let elems if (updated[objectId]) { elems = updated[objectId].elems } else if (obj) { elems = obj.elems.slice() } else { elems = [] } for (const edit of patch.edits) { if (edit.action === 'insert') { const value = getValue(edit.value, undefined, updated) const elem = {elemId: edit.elemId, pred: [edit.opId], value} elems.splice(edit.index, 0, elem) } else if (edit.action === 'multi-insert') { const startElemId = parseOpId(edit.elemId) const datatype = edit.datatype const newElems = edit.values.map((value, index) => { value = getValue({ datatype, value }, undefined, updated) const elemId = `${startElemId.counter + index}@${startElemId.actorId}` return {elemId, pred: [elemId], value} }) elems.splice(edit.index, 0, ...newElems) } else if (edit.action === 'update') { const elemId = elems[edit.index].elemId const value = getValue(edit.value, elems[edit.index].value, updated) elems[edit.index] = {elemId, pred: [edit.opId], value} } else if (edit.action === 'remove') { elems.splice(edit.index, edit.count) } } updated[objectId] = instantiateText(objectId, elems) return updated[objectId] } /** * Applies the patch object `patch` to the read-only document object `obj`. * Clones a writable copy of `obj` and places it in `updated` (indexed by * objectId), if that has not already been done. Returns the updated object. */ function interpretPatch(patch, obj, updated) { // Return original object if it already exists and isn't being modified if (isObject(obj) && (!patch.props || Object.keys(patch.props).length === 0) && (!patch.edits || patch.edits.length === 0) && !updated[patch.objectId]) { return obj } if (patch.type === 'map') { return updateMapObject(patch, obj, updated) } else if (patch.type === 'table') { return updateTableObject(patch, obj, updated) } else if (patch.type === 'list') { return updateListObject(patch, obj, updated) } else if (patch.type === 'text') { return updateTextObject(patch, obj, updated) } else { throw new TypeError(`Unknown object type: ${patch.type}`) } } /** * Creates a writable copy of the immutable document root object `root`. */ function cloneRootObject(root) { if (root[OBJECT_ID] !== '_root') { throw new RangeError(`Not the root object: ${root[OBJECT_ID]}`) } return cloneMapObject(root, '_root') } module.exports = { interpretPatch, cloneRootObject } ================================================ FILE: frontend/constants.js ================================================ // Properties of the document root object const OPTIONS = Symbol('_options') // object containing options passed to init() const CACHE = Symbol('_cache') // map from objectId to immutable object const STATE = Symbol('_state') // object containing metadata about current state (e.g. sequence numbers) // Properties of all Automerge objects const OBJECT_ID = Symbol('_objectId') // the object ID of the current object (string) const CONFLICTS = Symbol('_conflicts') // map or list (depending on object type) of conflicts const CHANGE = Symbol('_change') // the context object on proxy objects used in change callback const ELEM_IDS = Symbol('_elemIds') // list containing the element ID of each list element module.exports = { OPTIONS, CACHE, STATE, OBJECT_ID, CONFLICTS, CHANGE, ELEM_IDS } ================================================ FILE: frontend/context.js ================================================ const { CACHE, OBJECT_ID, CONFLICTS, ELEM_IDS, STATE } = require('./constants') const { interpretPatch } = require('./apply_patch') const { Text } = require('./text') const { Table } = require('./table') const { Counter, getWriteableCounter } = require('./counter') const { Int, Uint, Float64 } = require('./numbers') const { isObject, parseOpId, createArrayOfNulls } = require('../src/common') const uuid = require('../src/uuid') /** * An instance of this class is passed to `rootObjectProxy()`. The methods are * called by proxy object mutation functions to query the current object state * and to apply the requested changes. */ class Context { constructor (doc, actorId, applyPatch) { this.actorId = actorId this.nextOpNum = doc[STATE].maxOp + 1 this.cache = doc[CACHE] this.updated = {} this.ops = [] this.applyPatch = applyPatch ? applyPatch : interpretPatch } /** * Adds an operation object to the list of changes made in the current context. */ addOp(operation) { this.ops.push(operation) if (operation.action === 'set' && operation.values) { this.nextOpNum += operation.values.length } else if (operation.action === 'del' && operation.multiOp) { this.nextOpNum += operation.multiOp } else { this.nextOpNum += 1 } } /** * Returns the operation ID of the next operation to be added to the context. */ nextOpId() { return `${this.nextOpNum}@${this.actorId}` } /** * Takes a value and returns an object describing the value (in the format used by patches). */ getValueDescription(value) { if (!['object', 'boolean', 'number', 'string'].includes(typeof value)) { throw new TypeError(`Unsupported type of value: ${typeof value}`) } if (isObject(value)) { if (value instanceof Date) { // Date object, represented as milliseconds since epoch return {type: 'value', value: value.getTime(), datatype: 'timestamp'} } else if (value instanceof Int) { return {type: 'value', value: value.value, datatype: 'int'} } else if (value instanceof Uint) { return {type: 'value', value: value.value, datatype: 'uint'} } else if (value instanceof Float64) { return {type: 'value', value: value.value, datatype: 'float64'} } else if (value instanceof Counter) { // Counter object return {type: 'value', value: value.value, datatype: 'counter'} } else { // Nested object (map, list, text, or table) const objectId = value[OBJECT_ID], type = this.getObjectType(objectId) if (!objectId) { throw new RangeError(`Object ${JSON.stringify(value)} has no objectId`) } if (type === 'list' || type === 'text') { return {objectId, type, edits: []} } else { return {objectId, type, props: {}} } } } else if (typeof value === 'number') { if (Number.isInteger(value) && value <= Number.MAX_SAFE_INTEGER && value >= Number.MIN_SAFE_INTEGER) { return {type: 'value', value, datatype: 'int'} } else { return {type: 'value', value, datatype: 'float64'} } } else { // Primitive value (string, boolean, or null) return {type: 'value', value} } } /** * Builds the values structure describing a single property in a patch. Finds all the values of * property `key` of `object` (there might be multiple values in the case of a conflict), and * returns an object that maps operation IDs to descriptions of values. */ getValuesDescriptions(path, object, key) { if (object instanceof Table) { // Table objects don't have conflicts, since rows are identified by their unique objectId const value = object.byId(key) const opId = object.opIds[key] return value ? {[opId]: this.getValueDescription(value)} : {} } else if (object instanceof Text) { // Text objects don't support conflicts const value = object.get(key) const elemId = object.getElemId(key) return value ? {[elemId]: this.getValueDescription(value)} : {} } else { // Map or list objects const conflicts = object[CONFLICTS][key], values = {} if (!conflicts) { throw new RangeError(`No children at key ${key} of path ${JSON.stringify(path)}`) } for (let opId of Object.keys(conflicts)) { values[opId] = this.getValueDescription(conflicts[opId]) } return values } } /** * Returns the value at property `key` of object `object`. In the case of a conflict, returns * the value whose assignment operation has the ID `opId`. */ getPropertyValue(object, key, opId) { if (object instanceof Table) { return object.byId(key) } else if (object instanceof Text) { return object.get(key) } else { return object[CONFLICTS][key][opId] } } /** * Recurses along `path` into the patch object `patch`, creating nodes along the way as needed * by mutating the patch object. Returns the subpatch at the given path. */ getSubpatch(patch, path) { if (path.length == 0) return patch let subpatch = patch, object = this.getObject('_root') for (let pathElem of path) { let values = this.getValuesDescriptions(path, object, pathElem.key) if (subpatch.props) { if (!subpatch.props[pathElem.key]) { subpatch.props[pathElem.key] = values } } else if (subpatch.edits) { for (const opId of Object.keys(values)) { subpatch.edits.push({action: 'update', index: pathElem.key, opId, value: values[opId]}) } } let nextOpId = null for (let opId of Object.keys(values)) { if (values[opId].objectId === pathElem.objectId) { nextOpId = opId } } if (!nextOpId) { throw new RangeError(`Cannot find path object with objectId ${pathElem.objectId}`) } subpatch = values[nextOpId] object = this.getPropertyValue(object, pathElem.key, nextOpId) } return subpatch } /** * Returns an object (not proxied) from the cache or updated set, as appropriate. */ getObject(objectId) { const object = this.updated[objectId] || this.cache[objectId] if (!object) throw new RangeError(`Target object does not exist: ${objectId}`) return object } /** * Returns a string that is either 'map', 'table', 'list', or 'text', indicating * the type of the object with ID `objectId`. */ getObjectType(objectId) { if (objectId === '_root') return 'map' const object = this.getObject(objectId) if (object instanceof Text) return 'text' if (object instanceof Table) return 'table' if (Array.isArray(object)) return 'list' return 'map' } /** * Returns the value associated with the property named `key` on the object * at path `path`. If the value is an object, returns a proxy for it. */ getObjectField(path, objectId, key) { if (!['string', 'number'].includes(typeof key)) return const object = this.getObject(objectId) if (object[key] instanceof Counter) { return getWriteableCounter(object[key].value, this, path, objectId, key) } else if (isObject(object[key])) { const childId = object[key][OBJECT_ID] const subpath = path.concat([{key, objectId: childId}]) // The instantiateObject function is added to the context object by rootObjectProxy() return this.instantiateObject(subpath, childId) } else { return object[key] } } /** * Recursively creates Automerge versions of all the objects and nested objects in `value`, * constructing a patch and operations that describe the object tree. The new object is * assigned to the property `key` in the object with ID `obj`. If the object is a list or * text, `key` must be set to the list index being updated, and `elemId` must be set to the * elemId of the element being updated. If `insert` is true, we insert a new list element * (or text character) at index `key`, and `elemId` must be the elemId of the immediate * predecessor element (or the string '_head' if inserting at index 0). If the assignment * overwrites a previous value at this key/element, `pred` must be set to the array of the * prior operations we are overwriting (empty array if there is no existing value). */ createNestedObjects(obj, key, value, insert, pred, elemId) { if (value[OBJECT_ID]) { throw new RangeError('Cannot create a reference to an existing document object') } const objectId = this.nextOpId() if (value instanceof Text) { // Create a new Text object this.addOp(elemId ? {action: 'makeText', obj, elemId, insert, pred} : {action: 'makeText', obj, key, insert, pred}) const subpatch = {objectId, type: 'text', edits: []} this.insertListItems(subpatch, 0, [...value], true) return subpatch } else if (value instanceof Table) { // Create a new Table object if (value.count > 0) { throw new RangeError('Assigning a non-empty Table object is not supported') } this.addOp(elemId ? {action: 'makeTable', obj, elemId, insert, pred} : {action: 'makeTable', obj, key, insert, pred}) return {objectId, type: 'table', props: {}} } else if (Array.isArray(value)) { // Create a new list object this.addOp(elemId ? {action: 'makeList', obj, elemId, insert, pred} : {action: 'makeList', obj, key, insert, pred}) const subpatch = {objectId, type: 'list', edits: []} this.insertListItems(subpatch, 0, value, true) return subpatch } else { // Create a new map object this.addOp(elemId ? {action: 'makeMap', obj, elemId, insert, pred} : {action: 'makeMap', obj, key, insert, pred}) let props = {} for (let nested of Object.keys(value).sort()) { const opId = this.nextOpId() const valuePatch = this.setValue(objectId, nested, value[nested], false, []) props[nested] = {[opId]: valuePatch} } return {objectId, type: 'map', props} } } /** * Records an assignment to a particular key in a map, or a particular index in a list. * `objectId` is the ID of the object being modified, `key` is the property name or list * index being updated, and `value` is the new value being assigned. If `insert` is true, * a new list element is inserted at index `key`, and `value` is assigned to that new list * element. `pred` is an array of opIds for previous values of the property being assigned, * which are overwritten by this operation. If the object being modified is a list or text, * `elemId` is the element ID of the list element being updated (if insert=false), or the * element ID of the list element immediately preceding the insertion (if insert=true). * * Returns a patch describing the new value. The return value is of the form * `{objectId, type, props}` if `value` is an object, or `{value, datatype}` if it is a * primitive value. For string, number, boolean, or null the datatype is omitted. */ setValue(objectId, key, value, insert, pred, elemId) { if (!objectId) { throw new RangeError('setValue needs an objectId') } if (key === '') { throw new RangeError('The key of a map entry must not be an empty string') } if (isObject(value) && !(value instanceof Date) && !(value instanceof Counter) && !(value instanceof Int) && !(value instanceof Uint) && !(value instanceof Float64)) { // Nested object (map, list, text, or table) return this.createNestedObjects(objectId, key, value, insert, pred, elemId) } else { // Date or counter object, or primitive value (number, string, boolean, or null) const description = this.getValueDescription(value) const op = {action: 'set', obj: objectId, insert, value: description.value, pred} if (elemId) op.elemId = elemId; else op.key = key if (description.datatype) op.datatype = description.datatype this.addOp(op) return description } } /** * Constructs a new patch, calls `callback` with the subpatch at the location `path`, * and then immediately applies the patch to the document. */ applyAtPath(path, callback) { let diff = {objectId: '_root', type: 'map', props: {}} callback(this.getSubpatch(diff, path)) this.applyPatch(diff, this.cache._root, this.updated) } /** * Updates the map object at path `path`, setting the property with name * `key` to `value`. */ setMapKey(path, key, value) { if (typeof key !== 'string') { throw new RangeError(`The key of a map entry must be a string, not ${typeof key}`) } const objectId = path.length === 0 ? '_root' : path[path.length - 1].objectId const object = this.getObject(objectId) if (object[key] instanceof Counter) { throw new RangeError('Cannot overwrite a Counter object; use .increment() or .decrement() to change its value.') } // If the assigned field value is the same as the existing value, and // the assignment does not resolve a conflict, do nothing if (object[key] !== value || Object.keys(object[CONFLICTS][key] || {}).length > 1 || value === undefined) { this.applyAtPath(path, subpatch => { const pred = getPred(object, key) const opId = this.nextOpId() const valuePatch = this.setValue(objectId, key, value, false, pred) subpatch.props[key] = {[opId]: valuePatch} }) } } /** * Updates the map object at path `path`, deleting the property `key`. */ deleteMapKey(path, key) { const objectId = path.length === 0 ? '_root' : path[path.length - 1].objectId const object = this.getObject(objectId) if (object[key] !== undefined) { const pred = getPred(object, key) this.addOp({action: 'del', obj: objectId, key, insert: false, pred}) this.applyAtPath(path, subpatch => { subpatch.props[key] = {} }) } } /** * Inserts a sequence of new list elements `values` into a list, starting at position `index`. * `newObject` is true if we are creating a new list object, and false if we are updating an * existing one. `subpatch` is the patch for the list object being modified. Mutates * `subpatch` to reflect the sequence of values. */ insertListItems(subpatch, index, values, newObject) { const list = newObject ? [] : this.getObject(subpatch.objectId) if (index < 0 || index > list.length) { throw new RangeError(`List index ${index} is out of bounds for list of length ${list.length}`) } if (values.length === 0) return let elemId = getElemId(list, index, true) const allPrimitive = values.every(v => typeof v === 'string' || typeof v === 'number' || typeof v === 'boolean' || v === null || (isObject(v) && (v instanceof Date || v instanceof Counter || v instanceof Int || v instanceof Uint || v instanceof Float64))) const allValueDescriptions = allPrimitive ? values.map(v => this.getValueDescription(v)) : [] const allDatatypesSame = allValueDescriptions.every(t => t.datatype === allValueDescriptions[0].datatype) if (allPrimitive && allDatatypesSame && values.length > 1) { const nextElemId = this.nextOpId() const datatype = allValueDescriptions[0].datatype const values = allValueDescriptions.map(v => v.value) const op = {action: 'set', obj: subpatch.objectId, elemId, insert: true, values, pred: []} const edit = {action: 'multi-insert', elemId: nextElemId, index, values} if (datatype) { op.datatype = datatype edit.datatype = datatype } this.addOp(op) subpatch.edits.push(edit) } else { for (let offset = 0; offset < values.length; offset++) { let nextElemId = this.nextOpId() const valuePatch = this.setValue(subpatch.objectId, index + offset, values[offset], true, [], elemId) elemId = nextElemId subpatch.edits.push({action: 'insert', index: index + offset, elemId, opId: elemId, value: valuePatch}) } } } /** * Updates the list object at path `path`, replacing the current value at * position `index` with the new value `value`. */ setListIndex(path, index, value) { const objectId = path.length === 0 ? '_root' : path[path.length - 1].objectId const list = this.getObject(objectId) // Assignment past the end of the list => insert nulls followed by new value if (index >= list.length) { const insertions = createArrayOfNulls(index - list.length) insertions.push(value) return this.splice(path, list.length, 0, insertions) } if (list[index] instanceof Counter) { throw new RangeError('Cannot overwrite a Counter object; use .increment() or .decrement() to change its value.') } // If the assigned list element value is the same as the existing value, and // the assignment does not resolve a conflict, do nothing if (list[index] !== value || Object.keys(list[CONFLICTS][index] || {}).length > 1 || value === undefined) { this.applyAtPath(path, subpatch => { const pred = getPred(list, index) const opId = this.nextOpId() const valuePatch = this.setValue(objectId, index, value, false, pred, getElemId(list, index)) subpatch.edits.push({action: 'update', index, opId, value: valuePatch}) }) } } /** * Updates the list object at path `path`, deleting `deletions` list elements starting from * list index `start`, and inserting the list of new elements `insertions` at that position. */ splice(path, start, deletions, insertions) { const objectId = path.length === 0 ? '_root' : path[path.length - 1].objectId let list = this.getObject(objectId) if (start < 0 || deletions < 0 || start > list.length - deletions) { throw new RangeError(`${deletions} deletions starting at index ${start} are out of bounds for list of length ${list.length}`) } if (deletions === 0 && insertions.length === 0) return let patch = {diffs: {objectId: '_root', type: 'map', props: {}}} let subpatch = this.getSubpatch(patch.diffs, path) if (deletions > 0) { let op, lastElemParsed, lastPredParsed for (let i = 0; i < deletions; i++) { if (this.getObjectField(path, objectId, start + i) instanceof Counter) { // This may seem bizarre, but it's really fiddly to implement deletion of counters from // lists, and I doubt anyone ever needs to do this, so I'm just going to throw an // exception for now. The reason is: a counter is created by a set operation with counter // datatype, and subsequent increment ops are successors to the set operation. Normally, a // set operation with successor indicates a value that has been overwritten, so a set // operation with successors is normally invisible. Counters are an exception, because the // increment operations don't make the set operation invisible. When a counter appears in // a map, this is not too bad: if all successors are increments, then the counter remains // visible; if one or more successors are deletions, it goes away. However, when deleting // a list element, we have the additional challenge that we need to distinguish between a // list element that is being deleted by the current change (in which case we need to put // a 'remove' action in the patch's edits for that list) and a list element that was // already deleted previously (in which case the patch should not reflect the deletion). // This can be done, but as I said, it's fiddly. If someone wants to pick this up in the // future, hopefully the above description will be enough to get you started. Good luck! throw new TypeError('Unsupported operation: deleting a counter from a list') } // Any sequences of deletions with consecutive elemId and pred values get combined into a // single multiOp; any others become individual deletion operations. This optimisation only // kicks in if the user deletes a sequence of elements at once (in a single call to splice); // it might be nice to also detect such runs of deletions in the case where the user deletes // a sequence of list elements one by one. const thisElem = getElemId(list, start + i), thisElemParsed = parseOpId(thisElem) const thisPred = getPred(list, start + i) const thisPredParsed = (thisPred.length === 1) ? parseOpId(thisPred[0]) : undefined if (op && lastElemParsed && lastPredParsed && thisPredParsed && lastElemParsed.actorId === thisElemParsed.actorId && lastElemParsed.counter + 1 === thisElemParsed.counter && lastPredParsed.actorId === thisPredParsed.actorId && lastPredParsed.counter + 1 === thisPredParsed.counter) { op.multiOp = (op.multiOp || 1) + 1 } else { if (op) this.addOp(op) op = {action: 'del', obj: objectId, elemId: thisElem, insert: false, pred: thisPred} } lastElemParsed = thisElemParsed lastPredParsed = thisPredParsed } this.addOp(op) subpatch.edits.push({action: 'remove', index: start, count: deletions}) } if (insertions.length > 0) { this.insertListItems(subpatch, start, insertions, false) } this.applyPatch(patch.diffs, this.cache._root, this.updated) } /** * Updates the table object at path `path`, adding a new entry `row`. * Returns the objectId of the new row. */ addTableRow(path, row) { if (!isObject(row) || Array.isArray(row)) { throw new TypeError('A table row must be an object') } if (row[OBJECT_ID]) { throw new TypeError('Cannot reuse an existing object as table row') } if (row.id) { throw new TypeError('A table row must not have an "id" property; it is generated automatically') } const id = uuid() const valuePatch = this.setValue(path[path.length - 1].objectId, id, row, false, []) this.applyAtPath(path, subpatch => { subpatch.props[id] = {[valuePatch.objectId]: valuePatch} }) return id } /** * Updates the table object at path `path`, deleting the row with ID `rowId`. * `pred` is the opId of the operation that originally created the row. */ deleteTableRow(path, rowId, pred) { const objectId = path[path.length - 1].objectId, table = this.getObject(objectId) if (table.byId(rowId)) { this.addOp({action: 'del', obj: objectId, key: rowId, insert: false, pred: [pred]}) this.applyAtPath(path, subpatch => { subpatch.props[rowId] = {} }) } } /** * Adds the integer `delta` to the value of the counter located at property * `key` in the object at path `path`. */ increment(path, key, delta) { const objectId = path.length === 0 ? '_root' : path[path.length - 1].objectId const object = this.getObject(objectId) if (!(object[key] instanceof Counter)) { throw new TypeError('Only counter values can be incremented') } // TODO what if there is a conflicting value on the same key as the counter? const type = this.getObjectType(objectId) const value = object[key].value + delta const opId = this.nextOpId() const pred = getPred(object, key) if (type === 'list' || type === 'text') { const elemId = getElemId(object, key, false) this.addOp({action: 'inc', obj: objectId, elemId, value: delta, insert: false, pred}) } else { this.addOp({action: 'inc', obj: objectId, key, value: delta, insert: false, pred}) } this.applyAtPath(path, subpatch => { if (type === 'list' || type === 'text') { subpatch.edits.push({action: 'update', index: key, opId, value: {value, datatype: 'counter'}}) } else { subpatch.props[key] = {[opId]: {value, datatype: 'counter'}} } }) } } function getPred(object, key) { if (object instanceof Table) { return [object.opIds[key]] } else if (object instanceof Text) { return object.elems[key].pred } else if (object[CONFLICTS]) { return object[CONFLICTS][key] ? Object.keys(object[CONFLICTS][key]) : [] } else { return [] } } function getElemId(list, index, insert = false) { if (insert) { if (index === 0) return '_head' index -= 1 } if (list[ELEM_IDS]) return list[ELEM_IDS][index] if (list.getElemId) return list.getElemId(index) throw new RangeError(`Cannot find elemId at list index ${index}`) } module.exports = { Context } ================================================ FILE: frontend/counter.js ================================================ /** * The most basic CRDT: an integer value that can be changed only by * incrementing and decrementing. Since addition of integers is commutative, * the value trivially converges. */ class Counter { constructor(value) { this.value = value || 0 Object.freeze(this) } /** * A peculiar JavaScript language feature from its early days: if the object * `x` has a `valueOf()` method that returns a number, you can use numerical * operators on the object `x` directly, such as `x + 1` or `x < 4`. * This method is also called when coercing a value to a string by * concatenating it with another string, as in `x + ''`. * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object/valueOf */ valueOf() { return this.value } /** * Returns the counter value as a decimal string. If `x` is a counter object, * this method is called e.g. when you do `['value: ', x].join('')` or when * you use string interpolation: `value: ${x}`. */ toString() { return this.valueOf().toString() } /** * Returns the counter value, so that a JSON serialization of an Automerge * document represents the counter simply as an integer. */ toJSON() { return this.value } } /** * An instance of this class is used when a counter is accessed within a change * callback. */ class WriteableCounter extends Counter { /** * Increases the value of the counter by `delta`. If `delta` is not given, * increases the value of the counter by 1. */ increment(delta) { delta = typeof delta === 'number' ? delta : 1 this.context.increment(this.path, this.key, delta) this.value += delta return this.value } /** * Decreases the value of the counter by `delta`. If `delta` is not given, * decreases the value of the counter by 1. */ decrement(delta) { return this.increment(typeof delta === 'number' ? -delta : -1) } } /** * Returns an instance of `WriteableCounter` for use in a change callback. * `context` is the proxy context that keeps track of the mutations. * `objectId` is the ID of the object containing the counter, and `key` is * the property name (key in map, or index in list) where the counter is * located. */ function getWriteableCounter(value, context, path, objectId, key) { const instance = Object.create(WriteableCounter.prototype) instance.value = value instance.context = context instance.path = path instance.objectId = objectId instance.key = key return instance } module.exports = { Counter, getWriteableCounter } ================================================ FILE: frontend/index.js ================================================ const { OPTIONS, CACHE, STATE, OBJECT_ID, CONFLICTS, CHANGE, ELEM_IDS } = require('./constants') const { isObject, copyObject } = require('../src/common') const uuid = require('../src/uuid') const { interpretPatch, cloneRootObject } = require('./apply_patch') const { rootObjectProxy } = require('./proxies') const { Context } = require('./context') const { Text } = require('./text') const { Table } = require('./table') const { Counter } = require('./counter') const { Float64, Int, Uint } = require('./numbers') const { Observable } = require('./observable') /** * Actor IDs must consist only of hexadecimal digits so that they can be encoded * compactly in binary form. */ function checkActorId(actorId) { if (typeof actorId !== 'string') { throw new TypeError(`Unsupported type of actorId: ${typeof actorId}`) } if (!/^[0-9a-f]+$/.test(actorId)) { throw new RangeError('actorId must consist only of lowercase hex digits') } if (actorId.length % 2 !== 0) { throw new RangeError('actorId must consist of an even number of digits') } } /** * Takes a set of objects that have been updated (in `updated`) and an updated state object * `state`, and returns a new immutable document root object based on `doc` that reflects * those updates. */ function updateRootObject(doc, updated, state) { let newDoc = updated._root if (!newDoc) { newDoc = cloneRootObject(doc[CACHE]._root) updated._root = newDoc } Object.defineProperty(newDoc, OPTIONS, {value: doc[OPTIONS]}) Object.defineProperty(newDoc, CACHE, {value: updated}) Object.defineProperty(newDoc, STATE, {value: state}) if (doc[OPTIONS].freeze) { for (let objectId of Object.keys(updated)) { if (updated[objectId] instanceof Table) { updated[objectId]._freeze() } else if (updated[objectId] instanceof Text) { Object.freeze(updated[objectId].elems) Object.freeze(updated[objectId]) } else { Object.freeze(updated[objectId]) Object.freeze(updated[objectId][CONFLICTS]) } } } for (let objectId of Object.keys(doc[CACHE])) { if (!updated[objectId]) { updated[objectId] = doc[CACHE][objectId] } } if (doc[OPTIONS].freeze) { Object.freeze(updated) } return newDoc } /** * Adds a new change request to the list of pending requests, and returns an * updated document root object. * The details of the change are taken from the context object `context`. * `options` contains properties that may affect how the change is processed; in * particular, the `message` property of `options` is an optional human-readable * string describing the change. */ function makeChange(doc, context, options) { const actor = getActorId(doc) if (!actor) { throw new Error('Actor ID must be initialized with setActorId() before making a change') } const state = copyObject(doc[STATE]) state.seq += 1 const change = { actor, seq: state.seq, startOp: state.maxOp + 1, deps: state.deps, time: (options && typeof options.time === 'number') ? options.time : Math.round(new Date().getTime() / 1000), message: (options && typeof options.message === 'string') ? options.message : '', ops: context.ops } if (doc[OPTIONS].backend) { const [backendState, patch, binaryChange] = doc[OPTIONS].backend.applyLocalChange(state.backendState, change) state.backendState = backendState state.lastLocalChange = binaryChange // NOTE: When performing a local change, the patch is effectively applied twice -- once by the // context invoking interpretPatch as soon as any change is made, and the second time here // (after a round-trip through the backend). This is perhaps more robust, as changes only take // effect in the form processed by the backend, but the downside is a performance cost. // Should we change this? const newDoc = applyPatchToDoc(doc, patch, state, true) const patchCallback = options && options.patchCallback || doc[OPTIONS].patchCallback if (patchCallback) patchCallback(patch, doc, newDoc, true, [binaryChange]) return [newDoc, change] } else { const queuedRequest = {actor, seq: change.seq, before: doc} state.requests = state.requests.concat([queuedRequest]) state.maxOp = state.maxOp + countOps(change.ops) state.deps = [] return [updateRootObject(doc, context ? context.updated : {}, state), change] } } function countOps(ops) { let count = 0 for (const op of ops) { if (op.action === 'set' && op.values) { count += op.values.length } else { count += 1 } } return count } /** * Returns the binary encoding of the last change made by the local actor. */ function getLastLocalChange(doc) { return doc[STATE] && doc[STATE].lastLocalChange ? doc[STATE].lastLocalChange : null } /** * Applies the changes described in `patch` to the document with root object * `doc`. The state object `state` is attached to the new root object. * `fromBackend` should be set to `true` if the patch came from the backend, * and to `false` if the patch is a transient local (optimistically applied) * change from the frontend. */ function applyPatchToDoc(doc, patch, state, fromBackend) { const actor = getActorId(doc) const updated = {} interpretPatch(patch.diffs, doc, updated) if (fromBackend) { if (!patch.clock) throw new RangeError('patch is missing clock field') if (patch.clock[actor] && patch.clock[actor] > state.seq) { state.seq = patch.clock[actor] } state.clock = patch.clock state.deps = patch.deps state.maxOp = Math.max(state.maxOp, patch.maxOp) } return updateRootObject(doc, updated, state) } /** * Creates an empty document object with no changes. */ function init(options) { if (typeof options === 'string') { options = {actorId: options} } else if (typeof options === 'undefined') { options = {} } else if (!isObject(options)) { throw new TypeError(`Unsupported value for init() options: ${options}`) } if (!options.deferActorId) { if (options.actorId === undefined) { options.actorId = uuid() } checkActorId(options.actorId) } if (options.observable) { const patchCallback = options.patchCallback, observable = options.observable options.patchCallback = (patch, before, after, local, changes) => { if (patchCallback) patchCallback(patch, before, after, local, changes) observable.patchCallback(patch, before, after, local, changes) } } const root = {}, cache = {_root: root} const state = {seq: 0, maxOp: 0, requests: [], clock: {}, deps: []} if (options.backend) { state.backendState = options.backend.init() state.lastLocalChange = null } Object.defineProperty(root, OBJECT_ID, {value: '_root'}) Object.defineProperty(root, OPTIONS, {value: Object.freeze(options)}) Object.defineProperty(root, CONFLICTS, {value: Object.freeze({})}) Object.defineProperty(root, CACHE, {value: Object.freeze(cache)}) Object.defineProperty(root, STATE, {value: Object.freeze(state)}) return Object.freeze(root) } /** * Returns a new document object initialized with the given state. */ function from(initialState, options) { return change(init(options), 'Initialization', doc => Object.assign(doc, initialState)) } /** * Changes a document `doc` according to actions taken by the local user. * `options` is an object that can contain the following properties: * - `message`: an optional descriptive string that is attached to the change. * If `options` is a string, it is treated as `message`. * * The actual change is made within the callback function `callback`, which is * given a mutable version of the document as argument. Returns a two-element * array `[doc, request]` where `doc` is the updated document, and `request` * is the change request to send to the backend. If nothing was actually * changed, returns the original `doc` and a `null` change request. */ function change(doc, options, callback) { if (doc[OBJECT_ID] !== '_root') { throw new TypeError('The first argument to Automerge.change must be the document root') } if (doc[CHANGE]) { throw new TypeError('Calls to Automerge.change cannot be nested') } if (typeof options === 'function' && callback === undefined) { [options, callback] = [callback, options] } if (typeof options === 'string') { options = {message: options} } if (options !== undefined && !isObject(options)) { throw new TypeError('Unsupported type of options') } const actorId = getActorId(doc) if (!actorId) { throw new Error('Actor ID must be initialized with setActorId() before making a change') } const context = new Context(doc, actorId) callback(rootObjectProxy(context)) if (Object.keys(context.updated).length === 0) { // If the callback didn't change anything, return the original document object unchanged return [doc, null] } else { return makeChange(doc, context, options) } } /** * Triggers a new change request on the document `doc` without actually * modifying its data. `options` is an object as described in the documentation * for the `change` function. This function can be useful for acknowledging the * receipt of some message (as it's incorported into the `deps` field of the * change). Returns a two-element array `[doc, request]` where `doc` is the * updated document, and `request` is the change request to send to the backend. */ function emptyChange(doc, options) { if (doc[OBJECT_ID] !== '_root') { throw new TypeError('The first argument to Automerge.emptyChange must be the document root') } if (typeof options === 'string') { options = {message: options} } if (options !== undefined && !isObject(options)) { throw new TypeError('Unsupported type of options') } const actorId = getActorId(doc) if (!actorId) { throw new Error('Actor ID must be initialized with setActorId() before making a change') } return makeChange(doc, new Context(doc, actorId), options) } /** * Applies `patch` to the document root object `doc`. This patch must come * from the backend; it may be the result of a local change or a remote change. * If it is the result of a local change, the `seq` field from the change * request should be included in the patch, so that we can match them up here. */ function applyPatch(doc, patch, backendState = undefined) { if (doc[OBJECT_ID] !== '_root') { throw new TypeError('The first argument to Frontend.applyPatch must be the document root') } const state = copyObject(doc[STATE]) if (doc[OPTIONS].backend) { if (!backendState) { throw new RangeError('applyPatch must be called with the updated backend state') } state.backendState = backendState return applyPatchToDoc(doc, patch, state, true) } let baseDoc if (state.requests.length > 0) { baseDoc = state.requests[0].before if (patch.actor === getActorId(doc)) { if (state.requests[0].seq !== patch.seq) { throw new RangeError(`Mismatched sequence number: patch ${patch.seq} does not match next request ${state.requests[0].seq}`) } state.requests = state.requests.slice(1) } else { state.requests = state.requests.slice() } } else { baseDoc = doc state.requests = [] } let newDoc = applyPatchToDoc(baseDoc, patch, state, true) if (state.requests.length === 0) { return newDoc } else { state.requests[0] = copyObject(state.requests[0]) state.requests[0].before = newDoc return updateRootObject(doc, {}, state) } } /** * Returns the Automerge object ID of the given object. */ function getObjectId(object) { return object[OBJECT_ID] } /** * Returns the object with the given Automerge object ID. Note: when called * within a change callback, the returned object is read-only (not a mutable * proxy object). */ function getObjectById(doc, objectId) { // It would be nice to return a proxied object in a change callback. // However, that requires knowing the path from the root to the current // object, which we don't have if we jumped straight to the object by its ID. // If we maintained an index from object ID to parent ID we could work out the path. if (doc[CHANGE]) { throw new TypeError('Cannot use getObjectById in a change callback') } return doc[CACHE][objectId] } /** * Returns the Automerge actor ID of the given document. */ function getActorId(doc) { return doc[STATE].actorId || doc[OPTIONS].actorId } /** * Sets the Automerge actor ID on the document object `doc`, returning a * document object with updated metadata. */ function setActorId(doc, actorId) { checkActorId(actorId) const state = Object.assign({}, doc[STATE], {actorId}) return updateRootObject(doc, {}, state) } /** * Fetches the conflicts on the property `key` of `object`, which may be any * object in a document. If `object` is a list, then `key` must be a list * index; if `object` is a map, then `key` must be a property name. */ function getConflicts(object, key) { if (object[CONFLICTS] && object[CONFLICTS][key] && Object.keys(object[CONFLICTS][key]).length > 1) { return object[CONFLICTS][key] } } /** * Returns the backend state associated with the document `doc` (only used if * a backend implementation is passed to `init()`). */ function getBackendState(doc, callerName = null, argPos = 'first') { if (doc[OBJECT_ID] !== '_root') { // Most likely cause of passing an array here is forgetting to deconstruct the return value of // Automerge.applyChanges(). const extraMsg = Array.isArray(doc) ? '. Note: Automerge.applyChanges now returns an array.' : '' if (callerName) { throw new TypeError(`The ${argPos} argument to Automerge.${callerName} must be the document root${extraMsg}`) } else { throw new TypeError(`Argument is not an Automerge document root${extraMsg}`) } } return doc[STATE].backendState } /** * Given an array or text object from an Automerge document, returns an array * containing the unique element ID of each list element/character. */ function getElementIds(list) { if (list instanceof Text) { return list.elems.map(elem => elem.elemId) } else { return list[ELEM_IDS] } } module.exports = { init, from, change, emptyChange, applyPatch, getObjectId, getObjectById, getActorId, setActorId, getConflicts, getLastLocalChange, getBackendState, getElementIds, Text, Table, Counter, Observable, Float64, Int, Uint } ================================================ FILE: frontend/numbers.js ================================================ // Convience classes to allow users to stricly specify the number type they want class Int { constructor(value) { if (!(Number.isInteger(value) && value <= Number.MAX_SAFE_INTEGER && value >= Number.MIN_SAFE_INTEGER)) { throw new RangeError(`Value ${value} cannot be a uint`) } this.value = value Object.freeze(this) } } class Uint { constructor(value) { if (!(Number.isInteger(value) && value <= Number.MAX_SAFE_INTEGER && value >= 0)) { throw new RangeError(`Value ${value} cannot be a uint`) } this.value = value Object.freeze(this) } } class Float64 { constructor(value) { if (typeof value !== 'number') { throw new RangeError(`Value ${value} cannot be a float64`) } this.value = value || 0.0 Object.freeze(this) } } module.exports = { Int, Uint, Float64 } ================================================ FILE: frontend/observable.js ================================================ const { OBJECT_ID, CONFLICTS } = require('./constants') /** * Allows an application to register a callback when a particular object in * a document changes. * * NOTE: This API is experimental and may change without warning in minor releases. */ class Observable { constructor() { this.observers = {} // map from objectId to array of observers for that object } /** * Called by an Automerge document when `patch` is applied. `before` is the * state of the document before the patch, and `after` is the state after * applying it. `local` is true if the update is a result of locally calling * `Automerge.change()`, and false otherwise. `changes` is an array of * changes that were applied to the document (as Uint8Arrays). */ patchCallback(patch, before, after, local, changes) { this._objectUpdate(patch.diffs, before, after, local, changes) } /** * Recursively walks a patch and calls the callbacks for all objects that * appear in the patch. */ _objectUpdate(diff, before, after, local, changes) { if (!diff.objectId) return if (this.observers[diff.objectId]) { for (let callback of this.observers[diff.objectId]) { callback(diff, before, after, local, changes) } } if (diff.type === 'map' && diff.props) { for (const propName of Object.keys(diff.props)) { for (const opId of Object.keys(diff.props[propName])) { this._objectUpdate(diff.props[propName][opId], before && before[CONFLICTS] && before[CONFLICTS][propName] && before[CONFLICTS][propName][opId], after && after[CONFLICTS] && after[CONFLICTS][propName] && after[CONFLICTS][propName][opId], local, changes) } } } else if (diff.type === 'table' && diff.props) { for (const rowId of Object.keys(diff.props)) { for (const opId of Object.keys(diff.props[rowId])) { this._objectUpdate(diff.props[rowId][opId], before && before.byId(rowId), after && after.byId(rowId), local, changes) } } } else if (diff.type === 'list' && diff.edits) { let offset = 0 for (const edit of diff.edits) { if (edit.action === 'insert') { offset -= 1 this._objectUpdate(edit.value, undefined, after && after[CONFLICTS] && after[CONFLICTS][edit.index] && after[CONFLICTS][edit.index][edit.elemId], local, changes) } else if (edit.action === 'multi-insert') { offset -= edit.values.length } else if (edit.action === 'update') { this._objectUpdate(edit.value, before && before[CONFLICTS] && before[CONFLICTS][edit.index + offset] && before[CONFLICTS][edit.index + offset][edit.opId], after && after[CONFLICTS] && after[CONFLICTS][edit.index] && after[CONFLICTS][edit.index][edit.opId], local, changes) } else if (edit.action === 'remove') { offset += edit.count } } } else if (diff.type === 'text' && diff.edits) { let offset = 0 for (const edit of diff.edits) { if (edit.action === 'insert') { offset -= 1 this._objectUpdate(edit.value, undefined, after && after.get(edit.index), local, changes) } else if (edit.action === 'multi-insert') { offset -= edit.values.length } else if (edit.action === 'update') { this._objectUpdate(edit.value, before && before.get(edit.index + offset), after && after.get(edit.index), local, changes) } else if (edit.action === 'remove') { offset += edit.count } } } } /** * Call this to register a callback that will get called whenever a particular * object in a document changes. The callback is passed five arguments: the * part of the patch describing the update to that object, the old state of * the object, the new state of the object, a boolean that is true if the * change is the result of calling `Automerge.change()` locally, and the array * of binary changes applied to the document. */ observe(object, callback) { const objectId = object[OBJECT_ID] if (!objectId) throw new TypeError('The observed object must be part of an Automerge document') if (!this.observers[objectId]) this.observers[objectId] = [] this.observers[objectId].push(callback) } } module.exports = { Observable } ================================================ FILE: frontend/proxies.js ================================================ const { OBJECT_ID, CHANGE, STATE } = require('./constants') const { isObject, createArrayOfNulls } = require('../src/common') const { Text } = require('./text') const { Table } = require('./table') function parseListIndex(key) { if (typeof key === 'string' && /^[0-9]+$/.test(key)) key = parseInt(key, 10) if (typeof key !== 'number') { throw new TypeError('A list index must be a number, but you passed ' + JSON.stringify(key)) } if (key < 0 || isNaN(key) || key === Infinity || key === -Infinity) { throw new RangeError('A list index must be positive, but you passed ' + key) } return key } function listMethods(context, listId, path) { const methods = { deleteAt(index, numDelete) { context.splice(path, parseListIndex(index), numDelete || 1, []) return this }, fill(value, start, end) { let list = context.getObject(listId) for (let index = parseListIndex(start || 0); index < parseListIndex(end || list.length); index++) { context.setListIndex(path, index, value) } return this }, indexOf(o, start = 0) { const id = isObject(o) ? o[OBJECT_ID] : undefined if (id) { const list = context.getObject(listId) for (let index = start; index < list.length; index++) { if (list[index][OBJECT_ID] === id) { return index } } return -1 } else { return context.getObject(listId).indexOf(o, start) } }, insertAt(index, ...values) { context.splice(path, parseListIndex(index), 0, values) return this }, pop() { let list = context.getObject(listId) if (list.length == 0) return const last = context.getObjectField(path, listId, list.length - 1) context.splice(path, list.length - 1, 1, []) return last }, push(...values) { let list = context.getObject(listId) context.splice(path, list.length, 0, values) // need to getObject() again because the list object above may be immutable return context.getObject(listId).length }, shift() { let list = context.getObject(listId) if (list.length == 0) return const first = context.getObjectField(path, listId, 0) context.splice(path, 0, 1, []) return first }, splice(start, deleteCount, ...values) { let list = context.getObject(listId) start = parseListIndex(start) if (deleteCount === undefined || deleteCount > list.length - start) { deleteCount = list.length - start } const deleted = [] for (let n = 0; n < deleteCount; n++) { deleted.push(context.getObjectField(path, listId, start + n)) } context.splice(path, start, deleteCount, values) return deleted }, unshift(...values) { context.splice(path, 0, 0, values) return context.getObject(listId).length } } for (let iterator of ['entries', 'keys', 'values']) { let list = context.getObject(listId) methods[iterator] = () => list[iterator]() } // Read-only methods that can delegate to the JavaScript built-in implementations for (let method of ['concat', 'every', 'filter', 'find', 'findIndex', 'forEach', 'includes', 'join', 'lastIndexOf', 'map', 'reduce', 'reduceRight', 'slice', 'some', 'toLocaleString', 'toString']) { methods[method] = (...args) => { const list = context.getObject(listId) .map((item, index) => context.getObjectField(path, listId, index)) return list[method](...args) } } return methods } const MapHandler = { get (target, key) { const { context, objectId, path } = target if (key === OBJECT_ID) return objectId if (key === CHANGE) return context if (key === STATE) return {actorId: context.actorId} return context.getObjectField(path, objectId, key) }, set (target, key, value) { const { context, path, readonly } = target if (Array.isArray(readonly) && readonly.indexOf(key) >= 0) { throw new RangeError(`Object property "${key}" cannot be modified`) } context.setMapKey(path, key, value) return true }, deleteProperty (target, key) { const { context, path, readonly } = target if (Array.isArray(readonly) && readonly.indexOf(key) >= 0) { throw new RangeError(`Object property "${key}" cannot be modified`) } context.deleteMapKey(path, key) return true }, has (target, key) { const { context, objectId } = target return [OBJECT_ID, CHANGE].includes(key) || (key in context.getObject(objectId)) }, getOwnPropertyDescriptor (target, key) { const { context, objectId } = target const object = context.getObject(objectId) if (key in object) { return { configurable: true, enumerable: true, value: context.getObjectField(objectId, key) } } }, ownKeys (target) { const { context, objectId } = target return Object.keys(context.getObject(objectId)) } } const ListHandler = { get (target, key) { const [context, objectId, path] = target if (key === Symbol.iterator) return context.getObject(objectId)[Symbol.iterator] if (key === OBJECT_ID) return objectId if (key === CHANGE) return context if (key === 'length') return context.getObject(objectId).length if (typeof key === 'string' && /^[0-9]+$/.test(key)) { return context.getObjectField(path, objectId, parseListIndex(key)) } return listMethods(context, objectId, path)[key] }, set (target, key, value) { const [context, objectId, path] = target if (key === 'length') { if (typeof value !== 'number') { throw new RangeError("Invalid array length") } const length = context.getObject(objectId).length if (length > value) { context.splice(path, value, length - value, []) } else { context.splice(path, length, 0, createArrayOfNulls(value - length)) } } else { context.setListIndex(path, parseListIndex(key), value) } return true }, deleteProperty (target, key) { const [context, /* objectId */, path] = target context.splice(path, parseListIndex(key), 1, []) return true }, has (target, key) { const [context, objectId, /* path */] = target if (typeof key === 'string' && /^[0-9]+$/.test(key)) { return parseListIndex(key) < context.getObject(objectId).length } return ['length', OBJECT_ID, CHANGE].includes(key) }, getOwnPropertyDescriptor (target, key) { const [context, objectId, /* path */] = target const object = context.getObject(objectId) if (key === 'length') return {writable: true, value: object.length} if (key === OBJECT_ID) return {configurable: false, enumerable: false, value: objectId} if (typeof key === 'string' && /^[0-9]+$/.test(key)) { const index = parseListIndex(key) if (index < object.length) return { configurable: true, enumerable: true, value: context.getObjectField(objectId, index) } } }, ownKeys (target) { const [context, objectId, /* path */] = target const object = context.getObject(objectId) let keys = ['length'] for (let key of Object.keys(object)) keys.push(key) return keys } } function mapProxy(context, objectId, path, readonly) { return new Proxy({context, objectId, path, readonly}, MapHandler) } function listProxy(context, objectId, path) { return new Proxy([context, objectId, path], ListHandler) } /** * Instantiates a proxy object for the given `objectId`. * This function is added as a method to the context object by rootObjectProxy(). * When it is called, `this` is the context object. * `readonly` is a list of map property names that cannot be modified. */ function instantiateProxy(path, objectId, readonly) { const object = this.getObject(objectId) if (Array.isArray(object)) { return listProxy(this, objectId, path) } else if (object instanceof Text || object instanceof Table) { return object.getWriteable(this, path) } else { return mapProxy(this, objectId, path, readonly) } } function rootObjectProxy(context) { context.instantiateObject = instantiateProxy return mapProxy(context, '_root', []) } module.exports = { rootObjectProxy } ================================================ FILE: frontend/table.js ================================================ const { OBJECT_ID, CONFLICTS } = require('./constants') const { isObject, copyObject } = require('../src/common') function compareRows(properties, row1, row2) { for (let prop of properties) { if (row1[prop] === row2[prop]) continue if (typeof row1[prop] === 'number' && typeof row2[prop] === 'number') { return row1[prop] - row2[prop] } else { const prop1 = '' + row1[prop], prop2 = '' + row2[prop] if (prop1 === prop2) continue if (prop1 < prop2) return -1; else return +1 } } return 0 } /** * A relational-style unordered collection of records (rows). Each row is an * object that maps column names to values. The set of rows is represented by * a map from UUID to row object. */ class Table { /** * This constructor is used by application code when creating a new Table * object within a change callback. */ constructor() { this.entries = Object.freeze({}) this.opIds = Object.freeze({}) Object.freeze(this) } /** * Looks up a row in the table by its unique ID. */ byId(id) { return this.entries[id] } /** * Returns an array containing the unique IDs of all rows in the table, in no * particular order. */ get ids() { return Object.keys(this.entries).filter(key => { const entry = this.entries[key] return isObject(entry) && entry.id === key }) } /** * Returns the number of rows in the table. */ get count() { return this.ids.length } /** * Returns an array containing all of the rows in the table, in no particular * order. */ get rows() { return this.ids.map(id => this.byId(id)) } /** * The standard JavaScript `filter()` method, which passes each row to the * callback function and returns all rows for which the it returns true. */ filter(callback, thisArg) { return this.rows.filter(callback, thisArg) } /** * The standard JavaScript `find()` method, which passes each row to the * callback function and returns the first row for which it returns true. */ find(callback, thisArg) { return this.rows.find(callback, thisArg) } /** * The standard JavaScript `map()` method, which passes each row to the * callback function and returns a list of its return values. */ map(callback, thisArg) { return this.rows.map(callback, thisArg) } /** * Returns the list of rows, sorted by one of the following: * - If a function argument is given, it compares rows as per * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/sort#Description * - If a string argument is given, it is interpreted as a column name and * rows are sorted according to that column. * - If an array of strings is given, it is interpreted as a list of column * names, and rows are sorted lexicographically by those columns. * - If no argument is given, it sorts by row ID by default. */ sort(arg) { if (typeof arg === 'function') { return this.rows.sort(arg) } else if (typeof arg === 'string') { return this.rows.sort((row1, row2) => compareRows([arg], row1, row2)) } else if (Array.isArray(arg)) { return this.rows.sort((row1, row2) => compareRows(arg, row1, row2)) } else if (arg === undefined) { return this.rows.sort((row1, row2) => compareRows(['id'], row1, row2)) } else { throw new TypeError(`Unsupported sorting argument: ${arg}`) } } /** * When iterating over a table, you get all rows in the table, in no * particular order. */ [Symbol.iterator] () { let rows = this.rows, index = -1 return { next () { index += 1 if (index < rows.length) { return {done: false, value: rows[index]} } else { return {done: true} } } } } /** * Returns a shallow clone of this object. This clone is used while applying * a patch to the table, and `freeze()` is called on it when we have finished * applying the patch. */ _clone() { if (!this[OBJECT_ID]) { throw new RangeError('clone() requires the objectId to be set') } return instantiateTable(this[OBJECT_ID], copyObject(this.entries), copyObject(this.opIds)) } /** * Sets the entry with key `id` to `value`. `opId` is the ID of the operation * performing this assignment. This method is for internal use only; it is * not part of the public API of Automerge.Table. */ _set(id, value, opId) { if (Object.isFrozen(this.entries)) { throw new Error('A table can only be modified in a change function') } if (isObject(value) && !Array.isArray(value)) { Object.defineProperty(value, 'id', {value: id, enumerable: true}) } this.entries[id] = value this.opIds[id] = opId } /** * Removes the row with unique ID `id` from the table. */ remove(id) { if (Object.isFrozen(this.entries)) { throw new Error('A table can only be modified in a change function') } delete this.entries[id] delete this.opIds[id] } /** * Makes this object immutable. This is called after a change has been made. */ _freeze() { Object.freeze(this.entries) Object.freeze(this.opIds) Object.freeze(this) } /** * Returns a writeable instance of this table. This instance is returned when * the table is accessed within a change callback. `context` is the proxy * context that keeps track of the mutations. */ getWriteable(context, path) { if (!this[OBJECT_ID]) { throw new RangeError('getWriteable() requires the objectId to be set') } const instance = Object.create(WriteableTable.prototype) instance[OBJECT_ID] = this[OBJECT_ID] instance.context = context instance.entries = this.entries instance.opIds = this.opIds instance.path = path return instance } /** * Returns an object containing the table entries, indexed by objectID, * for serializing an Automerge document to JSON. */ toJSON() { const rows = {} for (let id of this.ids) rows[id] = this.byId(id) return rows } } /** * An instance of this class is used when a table is accessed within a change * callback. */ class WriteableTable extends Table { /** * Returns a proxied version of the row with ID `id`. This row object can be * modified within a change callback. */ byId(id) { if (isObject(this.entries[id]) && this.entries[id].id === id) { const objectId = this.entries[id][OBJECT_ID] const path = this.path.concat([{key: id, objectId}]) return this.context.instantiateObject(path, objectId, ['id']) } } /** * Adds a new row to the table. The row is given as a map from * column name to value. Returns the objectId of the new row. */ add(row) { return this.context.addTableRow(this.path, row) } /** * Removes the row with ID `id` from the table. Throws an exception if the row * does not exist in the table. */ remove(id) { if (isObject(this.entries[id]) && this.entries[id].id === id) { this.context.deleteTableRow(this.path, id, this.opIds[id]) } else { throw new RangeError(`There is no row with ID ${id} in this table`) } } } /** * This function is used to instantiate a Table object in the context of * applying a patch (see apply_patch.js). */ function instantiateTable(objectId, entries, opIds) { const instance = Object.create(Table.prototype) if (!objectId) { throw new RangeError('instantiateTable requires an objectId to be given') } instance[OBJECT_ID] = objectId instance[CONFLICTS] = Object.freeze({}) instance.entries = entries || {} instance.opIds = opIds || {} return instance } module.exports = { Table, instantiateTable } ================================================ FILE: frontend/text.js ================================================ const { OBJECT_ID } = require('./constants') const { isObject } = require('../src/common') class Text { constructor (text) { if (typeof text === 'string') { const elems = [...text].map(value => ({value})) return instantiateText(undefined, elems) // eslint-disable-line } else if (Array.isArray(text)) { const elems = text.map(value => ({value})) return instantiateText(undefined, elems) // eslint-disable-line } else if (text === undefined) { return instantiateText(undefined, []) // eslint-disable-line } else { throw new TypeError(`Unsupported initial value for Text: ${text}`) } } get length () { return this.elems.length } get (index) { const value = this.elems[index].value if (this.context && isObject(value)) { const objectId = value[OBJECT_ID] const path = this.path.concat([{key: index, objectId}]) return this.context.instantiateObject(path, objectId) } else { return value } } getElemId (index) { return this.elems[index].elemId } /** * Iterates over the text elements character by character, including any * inline objects. */ [Symbol.iterator] () { let elems = this.elems, index = -1 return { next () { index += 1 if (index < elems.length) { return {done: false, value: elems[index].value} } else { return {done: true} } } } } /** * Returns the content of the Text object as a simple string, ignoring any * non-character elements. */ toString() { // Concatting to a string is faster than creating an array and then // .join()ing for small (<100KB) arrays. // https://jsperf.com/join-vs-loop-w-type-test let str = '' for (const elem of this.elems) { if (typeof elem.value === 'string') str += elem.value } return str } /** * Returns the content of the Text object as a sequence of strings, * interleaved with non-character elements. * * For example, the value ['a', 'b', {x: 3}, 'c', 'd'] has spans: * => ['ab', {x: 3}, 'cd'] */ toSpans() { let spans = [] let chars = '' for (const elem of this.elems) { if (typeof elem.value === 'string') { chars += elem.value } else { if (chars.length > 0) { spans.push(chars) chars = '' } spans.push(elem.value) } } if (chars.length > 0) { spans.push(chars) } return spans } /** * Returns the content of the Text object as a simple string, so that the * JSON serialization of an Automerge document represents text nicely. */ toJSON() { return this.toString() } /** * Returns a writeable instance of this object. This instance is returned when * the text object is accessed within a change callback. `context` is the * proxy context that keeps track of the mutations. */ getWriteable(context, path) { if (!this[OBJECT_ID]) { throw new RangeError('getWriteable() requires the objectId to be set') } const instance = instantiateText(this[OBJECT_ID], this.elems) instance.context = context instance.path = path return instance } /** * Updates the list item at position `index` to a new value `value`. */ set (index, value) { if (this.context) { this.context.setListIndex(this.path, index, value) } else if (!this[OBJECT_ID]) { this.elems[index].value = value } else { throw new TypeError('Automerge.Text object cannot be modified outside of a change block') } return this } /** * Inserts new list items `values` starting at position `index`. */ insertAt(index, ...values) { if (this.context) { this.context.splice(this.path, index, 0, values) } else if (!this[OBJECT_ID]) { this.elems.splice(index, 0, ...values.map(value => ({value}))) } else { throw new TypeError('Automerge.Text object cannot be modified outside of a change block') } return this } /** * Deletes `numDelete` list items starting at position `index`. * if `numDelete` is not given, one item is deleted. */ deleteAt(index, numDelete = 1) { if (this.context) { this.context.splice(this.path, index, numDelete, []) } else if (!this[OBJECT_ID]) { this.elems.splice(index, numDelete) } else { throw new TypeError('Automerge.Text object cannot be modified outside of a change block') } return this } } // Read-only methods that can delegate to the JavaScript built-in array for (let method of ['concat', 'every', 'filter', 'find', 'findIndex', 'forEach', 'includes', 'indexOf', 'join', 'lastIndexOf', 'map', 'reduce', 'reduceRight', 'slice', 'some', 'toLocaleString']) { Text.prototype[method] = function (...args) { const array = [...this] return array[method](...args) } } function instantiateText(objectId, elems) { const instance = Object.create(Text.prototype) instance[OBJECT_ID] = objectId instance.elems = elems return instance } module.exports = { Text, instantiateText } ================================================ FILE: karma.conf.js ================================================ const path = require('path') const webpack = require('webpack') const webpackConfig = require('./webpack.config.js') // Karma-Webpack needs these gone delete webpackConfig.entry delete webpackConfig.output.filename // Don't mix dist/ webpackConfig.output.path = path.join(webpackConfig.output.path, 'test') // You're importing *a lot* of Node-specific code so the bundle is huge... webpackConfig.plugins = [ new webpack.DefinePlugin({ 'process.env.TEST_DIST': JSON.stringify(process.env.TEST_DIST) || '1', 'process.env.NODE_DEBUG': false, }), ...(webpackConfig.plugins || []), ] module.exports = function(config) { config.set({ frameworks: ['webpack', 'mocha', 'karma-typescript'], files: [ { pattern: 'test/*test*.js', watched: false }, { pattern: 'test/*test*.ts' }, ], preprocessors: { 'test/*test*.js': ['webpack'], 'test/*test*.ts': ['karma-typescript'], }, webpack: webpackConfig, browsers: ['Chrome', 'Firefox', 'Safari'], singleRun: true, // Webpack can handle Typescript via ts-loader karmaTypescriptConfig: { tsconfig: './tsconfig.json', bundlerOptions: { resolve: { alias: { automerge: './src/automerge.js' } } }, compilerOptions: { allowJs: true, sourceMap: true, } } }) } ================================================ FILE: karma.sauce.js ================================================ const path = require('path') const webpack = require('webpack') const webpackConfig = require("./webpack.config.js") // Karma-Webpack needs these gone delete webpackConfig.entry delete webpackConfig.output.filename // Don't mix dist/ webpackConfig.output.path = path.join(webpackConfig.output.path, 'test') // You're importing *a lot* of Node-specific code so the bundle is huge... webpackConfig.plugins = [ new webpack.DefinePlugin({ 'process.env.TEST_DIST': JSON.stringify(process.env.TEST_DIST) || '1', 'process.env.NODE_DEBUG': false, }), ...(webpackConfig.plugins || []), ] module.exports = function(config) { if (!process.env.SAUCE_USERNAME || !process.env.SAUCE_ACCESS_KEY) { console.log('Make sure the SAUCE_USERNAME and SAUCE_ACCESS_KEY environment variables are set.') // eslint-disable-line process.exit(1) } // Browsers to run on Sauce Labs // Check out https://saucelabs.com/platforms for all browser/OS combos const customLaunchers = { sl_chrome: { base: 'SauceLabs', browserName: 'chrome', platform: 'Windows 10', version: 'latest' }, sl_firefox: { base: 'SauceLabs', browserName: 'firefox', platform: 'Windows 10', version: 'latest' }, sl_edge: { base: 'SauceLabs', browserName: 'MicrosoftEdge', platform: 'Windows 10', version: 'latest' }, sl_safari_mac: { base: 'SauceLabs', browserName: 'safari', platform: 'macOS 10.15', version: 'latest' } } config.set({ frameworks: ['webpack', 'mocha', 'karma-typescript'], files: [ { pattern: 'test/*test*.js', watched: false }, { pattern: 'test/*test*.ts' }, ], preprocessors: { 'test/*test*.js': ['webpack'], 'test/*test*.ts': ['karma-typescript'], }, webpack: webpackConfig, karmaTypescriptConfig: { tsconfig: './tsconfig.json', bundlerOptions: { resolve: { alias: { automerge: './src/automerge.js' } } }, compilerOptions: { allowJs: true, sourceMap: true, } }, port: 9876, captureTimeout: 120000, sauceLabs: { testName: 'Automerge unit tests', startConnect: false, // Sauce Connect is started in GitHub action tunnelIdentifier: 'github-action-tunnel' }, customLaunchers, browsers: Object.keys(customLaunchers), reporters: ['progress', 'saucelabs'], singleRun: true }) } ================================================ FILE: package.json ================================================ { "name": "automerge", "version": "1.0.1-preview.7", "description": "Data structures for building collaborative applications", "main": "src/automerge.js", "browser": "dist/automerge.js", "types": "@types/automerge/index.d.ts", "scripts": { "browsertest": "karma start", "coverage": "nyc --reporter=html --reporter=text mocha", "test": "mocha", "testwasm": "mocha --file test/wasm.js", "build": "webpack && copyfiles --flat @types/automerge/index.d.ts dist", "prepublishOnly": "npm run-script build", "lint": "eslint ." }, "author": "", "repository": { "type": "git", "url": "git+ssh://git@github.com/automerge/automerge.git" }, "bugs": { "url": "https://github.com/automerge/automerge/issues" }, "homepage": "https://github.com/automerge/automerge", "license": "MIT", "files": [ "/src/**", "/frontend/**", "/backend/**", "/test/**", "/@types/**", "/dist/**", "/img/**", "/*.md", "/LICENSE", "/.babelrc", "/.eslintrc.json", "/.mocharc.yaml", "/karma.*.js", "/tsconfig.json", "/webpack.config.js" ], "dependencies": { "fast-sha256": "^1.3.0", "pako": "^2.0.3", "uuid": "^3.4.0" }, "devDependencies": { "@types/mocha": "^8.2.1", "@types/node": "^14.14.31", "copyfiles": "^2.4.1", "eslint": "^7.24.0", "eslint-plugin-compat": "^3.9.0", "karma": "^6.1.1", "karma-chrome-launcher": "^3.1.0", "karma-firefox-launcher": "^2.1.0", "karma-mocha": "^2.0.1", "karma-safari-launcher": "^1.0.0", "karma-sauce-launcher": "^4.3.5", "karma-typescript": "^5.4.0", "karma-webpack": "^5.0.0", "mocha": "^8.3.0", "nyc": "^15.1.0", "sinon": "^9.2.4", "ts-node": "^9.1.1", "tsconfig-paths": "^3.9.0", "typescript": "^4.1.5", "watchify": "^4.0.0", "webpack": "^5.24.0", "webpack-cli": "^4.5.0" }, "resolutions": { "karma-sauce-launcher/selenium-webdriver": "4.0.0-alpha.7" }, "browserslist": { "production": [ "defaults", "not IE 11", "maintained node versions" ], "web": [ "defaults", "not IE 11" ] } } ================================================ FILE: src/automerge.js ================================================ const uuid = require('./uuid') const Frontend = require('../frontend') const { OPTIONS } = require('../frontend/constants') const { encodeChange, decodeChange } = require('../backend/columnar') const { isObject } = require('./common') let backend = require('../backend') // mutable: can be overridden with setDefaultBackend() /** * Automerge.* API * The functions in this file constitute the publicly facing Automerge API which combines * the features of the Frontend (a document interface) and the backend (CRDT operations) */ function init(options) { if (typeof options === 'string') { options = {actorId: options} } else if (typeof options === 'undefined') { options = {} } else if (!isObject(options)) { throw new TypeError(`Unsupported options for init(): ${options}`) } return Frontend.init(Object.assign({backend}, options)) } /** * Returns a new document object initialized with the given state. */ function from(initialState, options) { const changeOpts = {message: 'Initialization'} return change(init(options), changeOpts, doc => Object.assign(doc, initialState)) } function change(doc, options, callback) { const [newDoc] = Frontend.change(doc, options, callback) return newDoc } function emptyChange(doc, options) { const [newDoc] = Frontend.emptyChange(doc, options) return newDoc } function clone(doc, options = {}) { const state = backend.clone(Frontend.getBackendState(doc, 'clone')) return applyPatch(init(options), backend.getPatch(state), state, [], options) } function free(doc) { backend.free(Frontend.getBackendState(doc, 'free')) } function load(data, options = {}) { const state = backend.load(data) return applyPatch(init(options), backend.getPatch(state), state, [data], options) } function save(doc) { return backend.save(Frontend.getBackendState(doc, 'save')) } function merge(localDoc, remoteDoc) { const localState = Frontend.getBackendState(localDoc, 'merge') const remoteState = Frontend.getBackendState(remoteDoc, 'merge', 'second') const changes = backend.getChangesAdded(localState, remoteState) const [updatedDoc] = applyChanges(localDoc, changes) return updatedDoc } function getChanges(oldDoc, newDoc) { const oldState = Frontend.getBackendState(oldDoc, 'getChanges') const newState = Frontend.getBackendState(newDoc, 'getChanges', 'second') return backend.getChanges(newState, backend.getHeads(oldState)) } function getAllChanges(doc) { return backend.getAllChanges(Frontend.getBackendState(doc, 'getAllChanges')) } function applyPatch(doc, patch, backendState, changes, options) { const newDoc = Frontend.applyPatch(doc, patch, backendState) const patchCallback = options.patchCallback || doc[OPTIONS].patchCallback if (patchCallback) { patchCallback(patch, doc, newDoc, false, changes) } return newDoc } function applyChanges(doc, changes, options = {}) { const oldState = Frontend.getBackendState(doc, 'applyChanges') const [newState, patch] = backend.applyChanges(oldState, changes) return [applyPatch(doc, patch, newState, changes, options), patch] } function equals(val1, val2) { if (!isObject(val1) || !isObject(val2)) return val1 === val2 const keys1 = Object.keys(val1).sort(), keys2 = Object.keys(val2).sort() if (keys1.length !== keys2.length) return false for (let i = 0; i < keys1.length; i++) { if (keys1[i] !== keys2[i]) return false if (!equals(val1[keys1[i]], val2[keys2[i]])) return false } return true } function getHistory(doc) { const actor = Frontend.getActorId(doc) const history = getAllChanges(doc) return history.map((change, index) => ({ get change () { return decodeChange(change) }, get snapshot () { const state = backend.loadChanges(backend.init(), history.slice(0, index + 1)) return Frontend.applyPatch(init(actor), backend.getPatch(state), state) } }) ) } function generateSyncMessage(doc, syncState) { const state = Frontend.getBackendState(doc, 'generateSyncMessage') return backend.generateSyncMessage(state, syncState) } function receiveSyncMessage(doc, oldSyncState, message) { const oldBackendState = Frontend.getBackendState(doc, 'receiveSyncMessage') const [backendState, syncState, patch] = backend.receiveSyncMessage(oldBackendState, oldSyncState, message) if (!patch) return [doc, syncState, patch] // The patchCallback is passed as argument all changes that are applied. // We get those from the sync message if a patchCallback is present. let changes = null if (doc[OPTIONS].patchCallback) { changes = backend.decodeSyncMessage(message).changes } return [applyPatch(doc, patch, backendState, changes, {}), syncState, patch] } function initSyncState() { return backend.initSyncState() } /** * Replaces the default backend implementation with a different one. * This allows you to switch to using the Rust/WebAssembly implementation. */ function setDefaultBackend(newBackend) { backend = newBackend } module.exports = { init, from, change, emptyChange, clone, free, load, save, merge, getChanges, getAllChanges, applyChanges, encodeChange, decodeChange, equals, getHistory, uuid, Frontend, setDefaultBackend, generateSyncMessage, receiveSyncMessage, initSyncState, get Backend() { return backend } } for (let name of ['getObjectId', 'getObjectById', 'getActorId', 'setActorId', 'getConflicts', 'getLastLocalChange', 'Text', 'Table', 'Counter', 'Observable', 'Int', 'Uint', 'Float64']) { module.exports[name] = Frontend[name] } ================================================ FILE: src/common.js ================================================ function isObject(obj) { return typeof obj === 'object' && obj !== null } /** * Returns a shallow copy of the object `obj`. Faster than `Object.assign({}, obj)`. * https://jsperf.com/cloning-large-objects/1 */ function copyObject(obj) { if (!isObject(obj)) return {} let copy = {} for (let key of Object.keys(obj)) { copy[key] = obj[key] } return copy } /** * Takes a string in the form that is used to identify operations (a counter concatenated * with an actor ID, separated by an `@` sign) and returns an object `{counter, actorId}`. */ function parseOpId(opId) { const match = /^(\d+)@(.*)$/.exec(opId || '') if (!match) { throw new RangeError(`Not a valid opId: ${opId}`) } return {counter: parseInt(match[1], 10), actorId: match[2]} } /** * Returns true if the two byte arrays contain the same data, false if not. */ function equalBytes(array1, array2) { if (!(array1 instanceof Uint8Array) || !(array2 instanceof Uint8Array)) { throw new TypeError('equalBytes can only compare Uint8Arrays') } if (array1.byteLength !== array2.byteLength) return false for (let i = 0; i < array1.byteLength; i++) { if (array1[i] !== array2[i]) return false } return true } /** * Creates an array containing the value `null` repeated `length` times. */ function createArrayOfNulls(length) { const array = new Array(length) for (let i = 0; i < length; i++) array[i] = null return array } module.exports = { isObject, copyObject, parseOpId, equalBytes, createArrayOfNulls } ================================================ FILE: src/uuid.js ================================================ const { v4: uuid } = require('uuid') function defaultFactory() { return uuid().replace(/-/g, '') } let factory = defaultFactory function makeUuid() { return factory() } makeUuid.setFactory = newFactory => { factory = newFactory } makeUuid.reset = () => { factory = defaultFactory } module.exports = makeUuid ================================================ FILE: test/backend_test.js ================================================ /* eslint-disable no-unused-vars */ const assert = require('assert') const Automerge = process.env.TEST_DIST === '1' ? require('../dist/automerge') : require('../src/automerge') const Backend = Automerge.Backend const { encodeChange, decodeChange } = require('../backend/columnar') const uuid = require('../src/uuid') function hash(change) { return decodeChange(encodeChange(change)).hash } describe('Automerge.Backend', () => { describe('incremental diffs', () => { it('should assign to a key in a map', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'set', obj: '_root', key: 'bird', value: 'magpie', pred: []} ]} const s0 = Backend.init() const [s1, patch1] = Backend.applyChanges(s0, [encodeChange(change1)]) assert.deepStrictEqual(patch1, { clock: {[actor]: 1}, deps: [hash(change1)], maxOp: 1, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: { bird: {[`1@${actor}`]: {type: 'value', value: 'magpie'}} }} }) }) it('should increment a key in a map', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'set', obj: '_root', key: 'counter', value: 1, datatype: 'counter', pred: []} ]} const change2 = {actor, seq: 2, startOp: 2, time: 0, deps: [hash(change1)], ops: [ {action: 'inc', obj: '_root', key: 'counter', value: 2, pred: [`1@${actor}`]} ]} const s0 = Backend.init() const [s1, patch1] = Backend.applyChanges(s0, [encodeChange(change1)]) const [s2, patch2] = Backend.applyChanges(s1, [encodeChange(change2)]) assert.deepStrictEqual(patch2, { clock: {[actor]: 2}, deps: [hash(change2)], maxOp: 2, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: { counter: {[`1@${actor}`]: {type: 'value', value: 3, datatype: 'counter'}} }} }) }) it('should make a conflict on assignment to the same key', () => { const change1 = {actor: '111111', seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'set', obj: '_root', key: 'bird', value: 'magpie', pred: []} ]} const change2 = {actor: '222222', seq: 1, startOp: 2, time: 0, deps: [hash(change1)], ops: [ {action: 'set', obj: '_root', key: 'bird', value: 'blackbird', pred: []} ]} const s0 = Backend.init() const [s1, patch1] = Backend.applyChanges(s0, [encodeChange(change1)]) const [s2, patch2] = Backend.applyChanges(s1, [encodeChange(change2)]) assert.deepStrictEqual(patch2, { clock: {111111: 1, 222222: 1}, deps: [hash(change2)], maxOp: 2, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: { bird: { '1@111111': {type: 'value', value: 'magpie'}, '2@222222': {type: 'value', value: 'blackbird'} } }} }) }) it('should delete a key from a map', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'set', obj: '_root', key: 'bird', value: 'magpie', pred: []} ]} const change2 = {actor, seq: 2, startOp: 2, time: 0, deps: [hash(change1)], ops: [ {action: 'del', obj: '_root', key: 'bird', pred: [`1@${actor}`]} ]} const s0 = Backend.init() const [s1, patch1] = Backend.applyChanges(s0, [encodeChange(change1)]) const [s2, patch2] = Backend.applyChanges(s1, [encodeChange(change2)]) assert.deepStrictEqual(patch2, { clock: {[actor]: 2}, deps: [hash(change2)], maxOp: 2, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {bird: {}}} }) }) it('should create nested maps', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeMap', obj: '_root', key: 'birds', pred: []}, {action: 'set', obj: `1@${actor}`, key: 'wrens', value: 3, pred: []} ]} const s0 = Backend.init() const [s1, patch1] = Backend.applyChanges(s0, [encodeChange(change1)]) assert.deepStrictEqual(patch1, { clock: {[actor]: 1}, deps: [hash(change1)], maxOp: 2, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {birds: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'map', props: {wrens: {[`2@${actor}`]: {type: 'value', value: 3, datatype: 'int'}}} }}}} }) }) it('should assign to keys in nested maps', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeMap', obj: '_root', key: 'birds', pred: []}, {action: 'set', obj: `1@${actor}`, key: 'wrens', value: 3, pred: []} ]} const change2 = {actor, seq: 2, startOp: 3, time: 0, deps: [hash(change1)], ops: [ {action: 'set', obj: `1@${actor}`, key: 'sparrows', value: 15, pred: []} ]} const s0 = Backend.init() const [s1, patch1] = Backend.applyChanges(s0, [encodeChange(change1)]) const [s2, patch2] = Backend.applyChanges(s1, [encodeChange(change2)]) assert.deepStrictEqual(patch2, { clock: {[actor]: 2}, deps: [hash(change2)], maxOp: 3, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {birds: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'map', props: {sparrows: {[`3@${actor}`]: {type: 'value', value: 15, datatype: 'int'}}} }}}} }) }) it('should handle deletion of nested maps', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeMap', obj: '_root', key: 'birds', pred: []}, {action: 'set', obj: `1@${actor}`, key: 'wrens', value: 3, pred: []} ]} const change2 = {actor, seq: 2, startOp: 3, time: 0, deps: [hash(change1)], ops: [ {action: 'del', obj: '_root', key: 'birds', pred: [`1@${actor}`]} ]} const s0 = Backend.init() const [s1, patch1] = Backend.applyChanges(s0, [change1, change2].map(encodeChange)) assert.deepStrictEqual(patch1, { clock: {[actor]: 2}, deps: [hash(change2)], maxOp: 3, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {birds: {}}} }) }) it('should handle conflicts on nested maps', () => { const actor1 = uuid(), actor2 = uuid() const change1 = {actor: actor1, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeMap', obj: '_root', key: 'birds', pred: []}, {action: 'set', obj: `1@${actor1}`, key: 'wrens', value: 3, pred: []} ]} const change2 = {actor: actor1, seq: 2, startOp: 3, time: 0, deps: [hash(change1)], ops: [ {action: 'makeMap', obj: '_root', key: 'birds', pred: [`1@${actor1}`]}, {action: 'set', obj: `3@${actor1}`, key: 'hawks', value: 1, pred: []} ]} const change3 = {actor: actor2, seq: 1, startOp: 3, time: 0, deps: [hash(change1)], ops: [ {action: 'makeMap', obj: '_root', key: 'birds', pred: [`1@${actor1}`]}, {action: 'set', obj: `3@${actor2}`, key: 'sparrows', value: 15, pred: []} ]} const s0 = Backend.init() const [s1, patch1] = Backend.applyChanges(s0, [change1, change2, change3].map(encodeChange)) assert.deepStrictEqual(patch1, { clock: {[actor1]: 2, [actor2]: 1}, deps: [hash(change2), hash(change3)].sort(), maxOp: 4, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {birds: { [`3@${actor1}`]: {objectId: `3@${actor1}`, type: 'map', props: { hawks: {[`4@${actor1}`]: {type: 'value', value: 1, datatype: 'int'}} }}, [`3@${actor2}`]: {objectId: `3@${actor2}`, type: 'map', props: { sparrows: {[`4@${actor2}`]: {type: 'value', value: 15, datatype: 'int'}} }} }}} }) }) it('should handle updates inside conflicted map keys', () => { const actor1 = uuid(), actor2 = uuid() const change1 = {actor: actor1, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeMap', obj: '_root', key: 'birds', pred: []}, {action: 'set', obj: `1@${actor1}`, key: 'hawks', value: 1, pred: []} ]} const change2 = {actor: actor2, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeMap', obj: '_root', key: 'birds', pred: []}, {action: 'set', obj: `1@${actor2}`, key: 'sparrows', value: 15, pred: []} ]} const change3 = {actor: actor1, seq: 2, startOp: 3, time: 0, deps: [hash(change1), hash(change2)].sort(), ops: [ {action: 'set', obj: `1@${actor2}`, key: 'sparrows', value: 17, pred: [`2@${actor2}`]} ]} const s0 = Backend.init() const [s1, patch1] = Backend.applyChanges(s0, [encodeChange(change1), encodeChange(change2)]) const [s2, patch2] = Backend.applyChanges(s1, [encodeChange(change3)]) assert.deepStrictEqual(patch2, { clock: {[actor1]: 2, [actor2]: 1}, deps: [hash(change3)], maxOp: 3, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {birds: { [`1@${actor1}`]: {objectId: `1@${actor1}`, type: 'map', props: {}}, [`1@${actor2}`]: {objectId: `1@${actor2}`, type: 'map', props: { sparrows: {[`3@${actor1}`]: {type: 'value', value: 17, datatype: 'int'}} }} }}} }) }) it('should handle updates inside deleted maps', () => { const actor1 = uuid(), actor2 = uuid() const change1 = {actor: actor1, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeMap', obj: '_root', key: 'birds', pred: []}, {action: 'set', obj: `1@${actor1}`, key: 'hawks', value: 1, pred: []} ]} const change2 = {actor: actor2, seq: 1, startOp: 3, time: 0, deps: [hash(change1)], ops: [ {action: 'del', obj: '_root', key: 'birds', pred: [`1@${actor1}`]} ]} const change3 = {actor: actor1, seq: 2, startOp: 3, time: 0, deps: [hash(change1)], ops: [ {action: 'set', obj: `1@${actor1}`, key: 'hawks', value: 2, pred: [`2@${actor1}`]} ]} const s0 = Backend.init() const [s1, patch1] = Backend.applyChanges(s0, [encodeChange(change1), encodeChange(change2)]) const [s2, patch2] = Backend.applyChanges(s1, [encodeChange(change3)]) assert.deepStrictEqual(patch1, { clock: {[actor1]: 1, [actor2]: 1}, deps: [hash(change2)], maxOp: 3, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {birds: {}}} }) assert.deepStrictEqual(patch2, { clock: {[actor1]: 2, [actor2]: 1}, deps: [hash(change2), hash(change3)].sort(), maxOp: 3, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {}} }) }) it('should create lists', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeList', obj: '_root', key: 'birds', pred: []}, {action: 'set', obj: `1@${actor}`, elemId: '_head', insert: true, value: 'chaffinch', pred: []} ]} const s0 = Backend.init() const [s1, patch1] = Backend.applyChanges(s0, [encodeChange(change1)]) assert.deepStrictEqual(patch1, { clock: {[actor]: 1}, deps: [hash(change1)], maxOp: 2, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {birds: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'list', edits: [ {action: 'insert', index: 0, elemId: `2@${actor}`, opId: `2@${actor}`, value: {type: 'value', value: 'chaffinch'}} ] }}}} }) }) it('should apply updates inside lists', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeList', obj: '_root', key: 'birds', pred: []}, {action: 'set', obj: `1@${actor}`, elemId: '_head', insert: true, value: 'chaffinch', pred: []} ]} const change2 = {actor, seq: 2, startOp: 3, time: 0, deps: [hash(change1)], ops: [ {action: 'set', obj: `1@${actor}`, elemId: `2@${actor}`, value: 'greenfinch', pred: [`2@${actor}`]} ]} const s0 = Backend.init() const [s1, patch1] = Backend.applyChanges(s0, [encodeChange(change1)]) const [s2, patch2] = Backend.applyChanges(s1, [encodeChange(change2)]) assert.deepStrictEqual(patch2, { clock: {[actor]: 2}, deps: [hash(change2)], maxOp: 3, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {birds: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'list', edits: [ {action: 'update', opId: `3@${actor}`, index: 0, value: {type: 'value', value: 'greenfinch'}} ] }}}} }) }) it('should apply updates to objects inside list elements', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeList', obj: '_root', key: 'todos', pred: []}, {action: 'makeMap', obj: `1@${actor}`, elemId: '_head', insert: true, pred: []}, {action: 'set', obj: `2@${actor}`, key: 'title', value: 'buy milk', pred: []}, {action: 'set', obj: `2@${actor}`, key: 'done', value: false, pred: []} ]} // insert a new list element and update the existing list element in the same change const change2 = {actor, seq: 2, startOp: 5, time: 0, deps: [hash(change1)], ops: [ {action: 'makeMap', obj: `1@${actor}`, elemId: '_head', insert: true, pred: []}, {action: 'set', obj: `5@${actor}`, key: 'title', value: 'water plants', pred: []}, {action: 'set', obj: `5@${actor}`, key: 'done', value: false, pred: []}, {action: 'set', obj: `2@${actor}`, key: 'done', value: true, pred: [`4@${actor}`]} ]} const s0 = Backend.init() const [s1, patch1] = Backend.applyChanges(s0, [encodeChange(change1)]) const [s2, patch2] = Backend.applyChanges(s1, [encodeChange(change2)]) assert.deepStrictEqual(patch2, { clock: {[actor]: 2}, deps: [hash(change2)], maxOp: 8, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {todos: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'list', edits: [ {action: 'insert', index: 0, elemId: `5@${actor}`, opId: `5@${actor}`, value: { objectId: `5@${actor}`, type: 'map', props: { title: {[`6@${actor}`]: {type: 'value', value: 'water plants'}}, done: {[`7@${actor}`]: {type: 'value', value: false}} } }}, {action: 'update', index: 1, opId: `2@${actor}`, value: { objectId: `2@${actor}`, type: 'map', props: { done: {[`8@${actor}`]: {type: 'value', value: true}} } }} ] }}}} }) }) it('should apply updates inside conflicted list elements', () => { const actor1 = '01234567', actor2 = '89abcdef' const change1 = {actor: actor1, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeList', obj: '_root', key: 'todos', pred: []}, {action: 'makeMap', obj: `1@${actor1}`, elemId: '_head', insert: true, pred: []} ]} const change2 = {actor: actor1, seq: 2, startOp: 3, time: 0, deps: [hash(change1)], ops: [ {action: 'makeMap', obj: `1@${actor1}`, elemId: `2@${actor1}`, pred: [`2@${actor1}`]}, {action: 'set', obj: `3@${actor1}`, key: 'title', value: 'buy milk', pred: []}, {action: 'set', obj: `3@${actor1}`, key: 'done', value: false, pred: []} ]} const change3 = {actor: actor2, seq: 1, startOp: 3, time: 0, deps: [hash(change1)], ops: [ {action: 'makeMap', obj: `1@${actor1}`, elemId: `2@${actor1}`, pred: [`2@${actor1}`]}, {action: 'set', obj: `3@${actor2}`, key: 'title', value: 'water plants', pred: []}, {action: 'set', obj: `3@${actor2}`, key: 'done', value: false, pred: []} ]} const change4 = {actor: actor1, seq: 3, startOp: 6, time: 0, deps: [hash(change2), hash(change3)].sort(), ops: [ {action: 'set', obj: `3@${actor1}`, key: 'done', value: true, pred: [`5@${actor1}`]} ]} const s0 = Backend.init() const [s1, patch1] = Backend.applyChanges(s0, [change1, change2, change3].map(encodeChange)) const [s2, patch2] = Backend.applyChanges(s1, [encodeChange(change4)]) assert.deepStrictEqual(patch2, { clock: {[actor1]: 3, [actor2]: 1}, deps: [hash(change4)], maxOp: 6, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {todos: {[`1@${actor1}`]: { objectId: `1@${actor1}`, type: 'list', edits: [ {action: 'update', index: 0, opId: `3@${actor1}`, value: { objectId: `3@${actor1}`, type: 'map', props: { done: {[`6@${actor1}`]: {type: 'value', value: true}} } }}, {action: 'update', index: 0, opId: `3@${actor2}`, value: { objectId: `3@${actor2}`, type: 'map', props: {} }} ] }}}} }) }) it('should overwrite list elements', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeList', obj: '_root', key: 'todos', pred: []}, {action: 'makeMap', obj: `1@${actor}`, elemId: '_head', insert: true, pred: []}, {action: 'set', obj: `2@${actor}`, key: 'title', value: 'buy milk', pred: []}, {action: 'set', obj: `2@${actor}`, key: 'done', value: false, pred: []} ]} const change2 = {actor, seq: 2, startOp: 5, time: 0, deps: [hash(change1)], ops: [ {action: 'makeMap', obj: `1@${actor}`, elemId: `2@${actor}`, insert: false, pred: [`2@${actor}`]}, {action: 'set', obj: `5@${actor}`, key: 'title', value: 'water plants', pred: []}, {action: 'set', obj: `5@${actor}`, key: 'done', value: false, pred: []} ]} const s0 = Backend.init() const [s1, patch1] = Backend.applyChanges(s0, [encodeChange(change1), encodeChange(change2)]) assert.deepStrictEqual(patch1, { clock: {[actor]: 2}, deps: [hash(change2)], maxOp: 7, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {todos: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'list', edits: [ {action: 'insert', index: 0, elemId: `2@${actor}`, opId: `5@${actor}`, value: { objectId: `5@${actor}`, type: 'map', props: { title: {[`6@${actor}`]: {type: 'value', value: 'water plants'}}, done: {[`7@${actor}`]: {type: 'value', value: false}} } }} ] }}}} }) }) it('should delete list elements', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeList', obj: '_root', key: 'birds', pred: []}, {action: 'set', obj: `1@${actor}`, elemId: '_head', insert: true, value: 'chaffinch', pred: []} ]} const change2 = {actor, seq: 2, startOp: 3, time: 0, deps: [hash(change1)], ops: [ {action: 'del', obj: `1@${actor}`, elemId: `2@${actor}`, pred: [`2@${actor}`]} ]} const s0 = Backend.init() const [s1, patch1] = Backend.applyChanges(s0, [encodeChange(change1)]) const [s2, patch2] = Backend.applyChanges(s1, [encodeChange(change2)]) assert.deepStrictEqual(patch2, { clock: {[actor]: 2}, deps: [hash(change2)], maxOp: 3, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {birds: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'list', edits: [ {action: 'remove', index: 0, count: 1} ] }}}} }) }) it('should handle list element insertion and deletion in the same change', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeList', obj: '_root', key: 'birds', pred: []} ]} const change2 = {actor, seq: 2, startOp: 2, time: 0, deps: [hash(change1)], ops: [ {action: 'set', obj: `1@${actor}`, elemId: '_head', insert: true, value: 'chaffinch', pred: []}, {action: 'del', obj: `1@${actor}`, elemId: `2@${actor}`, pred: [`2@${actor}`]} ]} const s0 = Backend.init() const [s1, patch1] = Backend.applyChanges(s0, [encodeChange(change1)]) const [s2, patch2] = Backend.applyChanges(s1, [encodeChange(change2)]) assert.deepStrictEqual(patch2, { clock: {[actor]: 2}, deps: [hash(change2)], maxOp: 3, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {birds: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'list', edits: [ {action: 'insert', index: 0, elemId: `2@${actor}`, opId: `2@${actor}`, value: {type: 'value', value: 'chaffinch'}}, {action: 'remove', index: 0, count: 1} ] }}}} }) }) it('should handle changes within conflicted objects', () => { const actor1 = uuid(), actor2 = uuid() const change1 = {actor: actor1, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeList', obj: '_root', key: 'conflict', pred: []} ]} const change2 = {actor: actor2, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeMap', obj: '_root', key: 'conflict', pred: []} ]} const change3 = {actor: actor2, seq: 2, startOp: 2, time: 0, deps: [hash(change2)], ops: [ {action: 'set', obj: `1@${actor2}`, key: 'sparrows', value: 12, pred: []} ]} const s0 = Backend.init() const [s1, patch1] = Backend.applyChanges(s0, [encodeChange(change1)]) const [s2, patch2] = Backend.applyChanges(s1, [encodeChange(change2)]) const [s3, patch3] = Backend.applyChanges(s2, [encodeChange(change3)]) assert.deepStrictEqual(patch3, { clock: {[actor1]: 1, [actor2]: 2}, maxOp: 2, pendingChanges: 0, deps: [hash(change1), hash(change3)].sort(), diffs: {objectId: '_root', type: 'map', props: {conflict: { [`1@${actor1}`]: {objectId: `1@${actor1}`, type: 'list', edits: []}, [`1@${actor2}`]: {objectId: `1@${actor2}`, type: 'map', props: {sparrows: {[`2@${actor2}`]: {type: 'value', value: 12, datatype: 'int'}}}} }}} }) }) it('should support Date objects at the root', () => { const now = new Date() const actor = uuid(), change = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'set', obj: '_root', key: 'now', value: now.getTime(), datatype: 'timestamp', pred: []} ]} const s0 = Backend.init() const [s1, patch] = Backend.applyChanges(s0, [encodeChange(change)]) assert.deepStrictEqual(patch, { clock: {[actor]: 1}, deps: [hash(change)], maxOp: 1, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: { now: {[`1@${actor}`]: {type: 'value', value: now.getTime(), datatype: 'timestamp'}} }} }) }) it('should support Date objects in a list', () => { const now = new Date(), actor = uuid() const change = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeList', obj: '_root', key: 'list', pred: []}, {action: 'set', obj: `1@${actor}`, elemId: '_head', insert: true, value: now.getTime(), datatype: 'timestamp', pred: []} ]} const s0 = Backend.init() const [s1, patch] = Backend.applyChanges(s0, [encodeChange(change)]) assert.deepStrictEqual(patch, { clock: {[actor]: 1}, deps: [hash(change)], maxOp: 2, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {list: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'list', edits: [ {action: 'insert', index: 0, elemId: `2@${actor}`, opId: `2@${actor}`, value: {type: 'value', value: now.getTime(), datatype: 'timestamp'}} ] }}}} }) }) it('should handle updates to an object that has been deleted', () => { const actor1 = uuid(), actor2 = uuid() const change1 = {actor: actor1, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeMap', obj: '_root', key: 'birds', pred: []}, {action: 'set', obj: `1@${actor1}`, key: 'blackbirds', value: 2, pred: []} ]} const change2 = {actor: actor2, seq: 1, startOp: 3, time: 0, deps: [hash(change1)], ops: [ {action: 'del', obj: '_root', key: 'birds', pred: [`1@${actor1}`]} ]} const change3 = {actor: actor1, seq: 2, startOp: 3, time: 0, deps: [hash(change1)], ops: [ {action: 'set', obj: `1@${actor1}`, key: 'blackbirds', value: 2, pred: []} ]} const s0 = Backend.init() const [s1, patch1] = Backend.applyChanges(s0, [encodeChange(change1)]) const [s2, patch2] = Backend.applyChanges(s1, [encodeChange(change2)]) const [s3, patch3] = Backend.applyChanges(s2, [encodeChange(change3)]) assert.deepStrictEqual(patch3, { clock: {[actor1]: 2, [actor2]: 1}, maxOp: 3, pendingChanges: 0, deps: [hash(change2), hash(change3)].sort(), diffs: {objectId: '_root', type: 'map', props: {}} }) }) it('should handle updates to a deleted list element', () => { const actor1 = uuid(), actor2 = uuid() const change1 = {actor: actor1, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeList', obj: '_root', key: 'todos', pred: []}, {action: 'makeMap', obj: `1@${actor1}`, elemId: '_head', insert: true, pred: []}, {action: 'set', obj: `2@${actor1}`, key: 'title', value: 'buy milk', pred: []}, {action: 'set', obj: `2@${actor1}`, key: 'done', value: false, pred: []} ]} const change2 = {actor: actor2, seq: 1, startOp: 5, time: 0, deps: [hash(change1)], ops: [ {action: 'del', obj: `1@${actor1}`, elemId: `2@${actor1}`, pred: [`2@${actor1}`]} ]} const change3 = {actor: actor1, seq: 2, startOp: 5, time: 0, deps: [hash(change1)], ops: [ {action: 'set', obj: `2@${actor1}`, key: 'done', value: true, pred: [`4@${actor1}`]} ]} const s0 = Backend.init() const [s1, patch1] = Backend.applyChanges(s0, [change1, change2].map(encodeChange)) const [s2, patch2] = Backend.applyChanges(s1, [encodeChange(change3)]) assert.deepStrictEqual(patch1, { clock: {[actor1]: 1, [actor2]: 1}, deps: [hash(change2)], maxOp: 5, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {todos: {[`1@${actor1}`]: { objectId: `1@${actor1}`, type: 'list', edits: [ {action: 'insert', index: 0, elemId: `2@${actor1}`, opId: `2@${actor1}`, value: { objectId: `2@${actor1}`, type: 'map', props: { title: {[`3@${actor1}`]: {type: 'value', value: 'buy milk'}}, done: {[`4@${actor1}`]: {type: 'value', value: false}} } }}, {action: 'remove', index: 0, count: 1} ] }}}} }) assert.deepStrictEqual(patch2, { clock: {[actor1]: 2, [actor2]: 1}, deps: [hash(change2), hash(change3)].sort(), maxOp: 5, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {}} }) }) it('should handle nested maps in lists', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeList', obj: '_root', key: 'todos', pred: []}, {action: 'set', obj: `1@${actor}`, insert: true, elemId: '_head', pred: [], value: 'first'}, {action: 'makeMap', obj: `1@${actor}`, elemId: `2@${actor}`, insert: true, pred: []}, {action: 'set', obj: `3@${actor}`, key: 'title', value: 'water plants', pred: []}, {action: 'set', obj: `3@${actor}`, key: 'done', value: false, pred: []} ]} const s0 = Backend.init() const [s1, patch1] = Backend.applyChanges(s0, [encodeChange(change1)]) assert.deepStrictEqual(patch1, { clock: {[actor]: 1}, deps: [hash(change1)], maxOp: 5, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {todos: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'list', edits: [ {action: 'insert', index: 0, elemId: `2@${actor}`, opId: `2@${actor}`, value: {type: 'value', value: 'first'}}, {action: 'insert', index: 1, elemId: `3@${actor}`, opId: `3@${actor}`, value: { type: 'map', objectId: `3@${actor}`, props: { title: {[`4@${actor}`]: {type: 'value', value: 'water plants'}}, done: {[`5@${actor}`]: {type: 'value', value: false}} } }} ] }}}} }) }) it('should support inserting multiple elements in one op (int)', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeList', obj: '_root', key: 'todos', pred: []}, {action: 'set', obj: `1@${actor}`, insert: true, elemId: '_head', pred: [], datatype: 'int', values: [1, 2, 3, 4, 5]}, ]} const s0 = Backend.init() const [s1, patch1] = Backend.applyChanges(s0, [encodeChange(change1)]) assert.deepStrictEqual(patch1, { clock: {[actor]: 1}, deps: [hash(change1)], maxOp: 6, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {todos: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'list', edits: [ {action: 'multi-insert', index: 0, elemId: `2@${actor}`, datatype: 'int', values: [1, 2, 3, 4, 5]} ] }}}} }) }) it('should support inserting multiple elements in one op (bool)', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeList', obj: '_root', key: 'todos', pred: []}, {action: 'set', obj: `1@${actor}`, insert: true, elemId: '_head', pred: [], values: [true, true, false, true, false]}, ]} const s0 = Backend.init() const [s1, patch1] = Backend.applyChanges(s0, [encodeChange(change1)]) assert.deepStrictEqual(patch1, { clock: {[actor]: 1}, deps: [hash(change1)], maxOp: 6, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {todos: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'list', edits: [ {action: 'multi-insert', index: 0, elemId: `2@${actor}`, values: [true, true, false, true, false]} ] }}}} }) }) it('should support inserting multiple elements in one op (null)', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeList', obj: '_root', key: 'todos', pred: []}, {action: 'set', obj: `1@${actor}`, insert: true, elemId: '_head', pred: [], values: [null, null, null]}, ]} const s0 = Backend.init() const [s1, patch1] = Backend.applyChanges(s0, [encodeChange(change1)]) assert.deepStrictEqual(patch1, { clock: {[actor]: 1}, deps: [hash(change1)], maxOp: 4, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {todos: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'list', edits: [ {action: 'multi-insert', index: 0, elemId: `2@${actor}`, values: [null, null, null]} ] }}}} }) }) it('should support inserting multiple elements in one op (uint)', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeList', obj: '_root', key: 'todos', pred: []}, {action: 'set', obj: `1@${actor}`, insert: true, elemId: '_head', pred: [], datatype: 'uint', values: [1, 2, 3, 4, 5]}, ]} const s0 = Backend.init() const [s1, patch1] = Backend.applyChanges(s0, [encodeChange(change1)]) assert.deepStrictEqual(patch1, { clock: {[actor]: 1}, deps: [hash(change1)], maxOp: 6, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {todos: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'list', edits: [ {action: 'multi-insert', index: 0, elemId: `2@${actor}`, datatype: 'uint', values: [1, 2, 3, 4, 5]} ] }}}} }) }) it('should support inserting multiple elements in one op (float64)', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeList', obj: '_root', key: 'todos', pred: []}, {action: 'set', obj: `1@${actor}`, insert: true, elemId: '_head', pred: [], datatype: 'float64', values: [1, 2, 3, 4, 5]}, ]} const s0 = Backend.init() const [s1, patch1] = Backend.applyChanges(s0, [encodeChange(change1)]) assert.deepStrictEqual(patch1, { clock: {[actor]: 1}, deps: [hash(change1)], maxOp: 6, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {todos: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'list', edits: [ {action: 'multi-insert', index: 0, elemId: `2@${actor}`, datatype: 'float64', values: [1, 2, 3, 4, 5]} ] }}}} }) }) it('should support inserting multiple elements in one op (timestamp)', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeList', obj: '_root', key: 'todos', pred: []}, {action: 'set', obj: `1@${actor}`, insert: true, elemId: '_head', pred: [], datatype: 'timestamp', values: [1, 2, 3, 4, 5]}, ]} const s0 = Backend.init() const [s1, patch1] = Backend.applyChanges(s0, [encodeChange(change1)]) assert.deepStrictEqual(patch1, { clock: {[actor]: 1}, deps: [hash(change1)], maxOp: 6, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {todos: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'list', edits: [ {action: 'multi-insert', index: 0, elemId: `2@${actor}`, datatype: 'timestamp', values: [1, 2, 3, 4, 5]} ] }}}} }) }) it('should support inserting multiple elements in one op (counter)', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeList', obj: '_root', key: 'todos', pred: []}, {action: 'set', obj: `1@${actor}`, insert: true, elemId: '_head', pred: [], datatype: 'counter', values: [1, 2, 3, 4, 5]}, ]} const s0 = Backend.init() const [s1, patch1] = Backend.applyChanges(s0, [encodeChange(change1)]) assert.deepStrictEqual(patch1, { clock: {[actor]: 1}, deps: [hash(change1)], maxOp: 6, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {todos: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'list', edits: [ {action: 'multi-insert', index: 0, elemId: `2@${actor}`, datatype: 'counter', values: [1, 2, 3, 4, 5]} ] }}}} }) }) it('should throw an error if the datatype does not match the values', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeList', obj: '_root', key: 'todos', pred: []}, {action: 'set', obj: `1@${actor}`, insert: true, elemId: '_head', pred: [], datatype: 'int', values: [1, true, 'hello']}, ]} const s0 = Backend.init() assert.throws(() => { Backend.applyLocalChange(s0, change1) }, /Decode failed/) }) it('should support deleting multiple elements in one op', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeList', obj: '_root', key: 'todos', pred: []}, {action: 'set', obj: `1@${actor}`, insert: true, elemId: '_head', pred: [], datatype: 'int', values: [1, 2, 3, 4, 5]}, ]} const change2 = {actor, seq: 2, startOp: 7, time: 0, deps: [hash(change1)], ops: [ {action: 'del', obj: `1@${actor}`, elemId: `3@${actor}`, multiOp: 3, pred: [`3@${actor}`]} ]} const s0 = Backend.init() const [s1, patch1] = Backend.applyChanges(s0, [encodeChange(change1)]) const [s2, patch2] = Backend.applyChanges(s1, [encodeChange(change2)]) assert.deepStrictEqual(patch2, { clock: {[actor]: 2}, deps: [hash(change2)], maxOp: 9, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {todos: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'list', edits: [ {action: 'remove', index: 1, count: 3} ] }}}} }) }) }) describe('applyLocalChange()', () => { it('should apply change requests', () => { const change1 = {actor: '111111', seq: 1, time: 0, startOp: 1, deps: [], ops: [ {action: 'set', obj: '_root', key: 'bird', value: 'magpie', pred: []} ]} const s0 = Backend.init() const [s1, patch1] = Backend.applyLocalChange(s0, change1) const changes01 = Backend.getAllChanges(s1).map(decodeChange) assert.deepStrictEqual(patch1, { actor: '111111', seq: 1, clock: {'111111': 1}, deps: [], maxOp: 1, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: { bird: {'1@111111': {type: 'value', value: 'magpie'}} }} }) assert.deepStrictEqual(changes01, [{ hash: '2c2845859ce4336936f56410f9161a09ba269f48aee5826782f1c389ec01d054', actor: '111111', seq: 1, startOp: 1, time: 0, message: '', deps: [], ops: [ {action: 'set', obj: '_root', key: 'bird', insert: false, value: 'magpie', pred: []} ] }]) }) it('should throw an exception on duplicate requests', () => { const actor = uuid() const change1 = {actor, seq: 1, time: 0, startOp: 1, deps: [], ops: [ {action: 'set', obj: '_root', key: 'bird', value: 'magpie', pred: []} ]} const change2 = {actor, seq: 2, time: 0, startOp: 2, deps: [], ops: [ {action: 'set', obj: '_root', key: 'bird', value: 'jay', pred: []} ]} const s0 = Backend.init() const [s1, patch1] = Backend.applyLocalChange(s0, change1) const [s2, patch2] = Backend.applyLocalChange(s1, change2) assert.throws(() => Backend.applyLocalChange(s2, change1), /Change request has already been applied/) assert.throws(() => Backend.applyLocalChange(s2, change2), /Change request has already been applied/) }) it('should handle frontend and backend changes happening concurrently', () => { const local1 = {actor: '111111', seq: 1, time: 0, startOp: 1, deps: [], ops: [ {action: 'set', obj: '_root', key: 'bird', value: 'magpie', pred: []} ]} const local2 = {actor: '111111', seq: 2, time: 0, startOp: 2, deps: [], ops: [ {action: 'set', obj: '_root', key: 'bird', value: 'jay', pred: ['1@111111']} ]} const remote1 = {actor: '222222', seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'set', obj: '_root', key: 'fish', value: 'goldfish', pred: []} ]} const s0 = Backend.init() const [s1, patch1] = Backend.applyLocalChange(s0, local1) const [s2, patch2] = Backend.applyChanges(s1, [encodeChange(remote1)]) const [s3, patch3] = Backend.applyLocalChange(s2, local2) const changes = Backend.getAllChanges(s3).map(decodeChange) assert.deepStrictEqual(changes, [ {hash: '2c2845859ce4336936f56410f9161a09ba269f48aee5826782f1c389ec01d054', actor: '111111', seq: 1, startOp: 1, time: 0, message: '', deps: [], ops: [ {action: 'set', obj: '_root', key: 'bird', insert: false, value: 'magpie', pred: []} ]}, {hash: 'efc7e9b1b809364fb1b7029d2838dd3c7cf539eea595b22f9ae665505187f6c4', actor: '222222', seq: 1, startOp: 1, time: 0, message: '', deps: [], ops: [ {action: 'set', obj: '_root', key: 'fish', insert: false, value: 'goldfish', pred: []} ]}, {hash: 'e7ed7a790432aba39fe7ad75fa9e02a9fc8d8e9ee4ec8c81dcc93da15a561f8a', actor: '111111', seq: 2, startOp: 2, time: 0, message: '', deps: [changes[0].hash], ops: [ {action: 'set', obj: '_root', key: 'bird', insert: false, value: 'jay', pred: ['1@111111']} ]} ]) }) it('should detect conflicts based on the frontend version', () => { const local1 = {requestType: 'change', actor: '111111', seq: 1, time: 0, startOp: 1, deps: [], ops: [ {action: 'set', obj: '_root', key: 'bird', value: 'goldfinch', pred: []} ]} // remote1 depends on local1; the deps field is filled in below when we've computed the hash const remote1 = {actor: '222222', seq: 1, startOp: 2, time: 0, deps: [], ops: [ {action: 'set', obj: '_root', key: 'bird', value: 'magpie', pred: ['1@111111']} ]} // local2 is concurrent with remote1 (because version < 2) const local2 = {requestType: 'change', actor: '111111', seq: 2, time: 0, startOp: 2, deps: [], ops: [ {action: 'set', obj: '_root', key: 'bird', value: 'jay', pred: ['1@111111']} ]} const s0 = Backend.init() const [s1, patch1] = Backend.applyLocalChange(s0, local1) remote1.deps.push(Backend.getAllChanges(s1).map(decodeChange)[0].hash) const [s2, patch2] = Backend.applyChanges(s1, [encodeChange(remote1)]) const [s3, patch3] = Backend.applyLocalChange(s2, local2) const changes = Backend.getAllChanges(s3).map(decodeChange) assert.deepStrictEqual(patch3, { actor: '111111', seq: 2, clock: {'111111': 2, '222222': 1}, deps: [hash(remote1)], maxOp: 2, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: { bird: {'2@222222': {type: 'value', value: 'magpie'}, '2@111111': {type: 'value', value: 'jay'}} }} }) assert.deepStrictEqual(changes[2], { hash: '7a00e28d7fbf179708a1b0045c7f9bad93366c0e69f9af15e830dae9970a9d19', actor: '111111', seq: 2, startOp: 2, time: 0, message: '', deps: [changes[0].hash], ops: [ {action: 'set', obj: '_root', key: 'bird', insert: false, value: 'jay', pred: ['1@111111']} ] }) }) it('should transform list indexes into element IDs', () => { const remote1 = {actor: '222222', seq: 1, startOp: 1, time: 0, deps: [], ops: [ {obj: '_root', action: 'makeList', key: 'birds', pred: []} ]} const remote2 = {actor: '222222', seq: 2, startOp: 2, time: 0, deps: [hash(remote1)], ops: [ {obj: '1@222222', action: 'set', elemId: '_head', insert: true, value: 'magpie', pred: []} ]} const local1 = {actor: '111111', seq: 1, startOp: 2, time: 0, deps: [hash(remote1)], ops: [ {obj: '1@222222', action: 'set', elemId: '_head', insert: true, value: 'goldfinch', pred: []} ]} const local2 = {actor: '111111', seq: 2, startOp: 3, time: 0, deps: [], ops: [ {obj: '1@222222', action: 'set', elemId: '2@111111', insert: true, value: 'wagtail', pred: []} ]} const local3 = {actor: '111111', seq: 3, startOp: 4, time: 0, deps: [hash(remote2)], ops: [ {obj: '1@222222', action: 'set', elemId: '2@222222', value: 'Magpie', pred: ['2@222222']}, {obj: '1@222222', action: 'set', elemId: '2@111111', value: 'Goldfinch', pred: ['2@111111']} ]} const s0 = Backend.init() const [s1, patch1] = Backend.applyChanges(s0, [encodeChange(remote1)]) const [s2, patch2] = Backend.applyLocalChange(s1, local1) const [s3, patch3] = Backend.applyChanges(s2, [encodeChange(remote2)]) const [s4, patch4] = Backend.applyLocalChange(s3, local2) const [s5, patch5] = Backend.applyLocalChange(s4, local3) const changes = Backend.getAllChanges(s5).map(decodeChange) assert.deepStrictEqual(changes[1], { hash: '06392148c4a0dfff8b346ad58a3261cc15187cbf8a58779f78d54251126d4ccc', actor: '111111', seq: 1, startOp: 2, time: 0, message: '', deps: [hash(remote1)], ops: [ {obj: '1@222222', action: 'set', elemId: '_head', insert: true, value: 'goldfinch', pred: []} ] }) assert.deepStrictEqual(changes[3], { hash: '2801c386ec2a140376f3bef285a6e6d294a2d8fb7a180da4fbb6e2bc4f550dd9', actor: '111111', seq: 2, startOp: 3, time: 0, message: '', deps: [changes[1].hash], ops: [ {obj: '1@222222', action: 'set', elemId: '2@111111', insert: true, value: 'wagtail', pred: []} ] }) assert.deepStrictEqual(changes[4], { hash: '734f1dad5fb2f10970bae2baa6ce100c3b85b43072b3799d8f2e15bcd21297fc', actor: '111111', seq: 3, startOp: 4, time: 0, message: '', deps: [hash(remote2), changes[3].hash].sort(), ops: [ {obj: '1@222222', action: 'set', elemId: '2@222222', insert: false, value: 'Magpie', pred: ['2@222222']}, {obj: '1@222222', action: 'set', elemId: '2@111111', insert: false, value: 'Goldfinch', pred: ['2@111111']} ] }) }) it('should handle list element insertion and deletion in the same change', () => { const local1 = {requestType: 'change', actor: '111111', seq: 1, startOp: 1, deps: [], time: 0, ops: [ {obj: '_root', action: 'makeList', key: 'birds', pred: []} ]} const local2 = {requestType: 'change', actor: '111111', seq: 2, startOp: 2, deps: [], time: 0, ops: [ {obj: '1@111111', action: 'set', elemId: '_head', insert: true, value: 'magpie', pred: []}, {obj: '1@111111', action: 'del', elemId: '2@111111', pred: ['2@111111']} ]} const s0 = Backend.init() const [s1, patch1] = Backend.applyLocalChange(s0, local1) const [s2, patch2] = Backend.applyLocalChange(s1, local2) const changes = Backend.getAllChanges(s2).map(decodeChange) assert.deepStrictEqual(patch2, { actor: '111111', seq: 2, clock: {'111111': 2}, deps: [], maxOp: 3, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: { birds: {'1@111111': {objectId: '1@111111', type: 'list', edits: [ {action: 'insert', index: 0, elemId: '2@111111', opId: '2@111111', value: {type: 'value', value: 'magpie'}}, {action: 'remove', index: 0, count: 1} ]}} }} }) assert.deepStrictEqual(changes, [{ hash: changes[0].hash, actor: '111111', seq: 1, startOp: 1, time: 0, message: '', deps: [], ops: [ {obj: '_root', action: 'makeList', key: 'birds', insert: false, pred: []} ] }, { hash: 'deef4c9b9ca378844144c4bbc5d82a52f30c95a8624f13f243fe8f1214e8e833', actor: '111111', seq: 2, startOp: 2, time: 0, message: '', deps: [changes[0].hash], ops: [ {obj: '1@111111', action: 'set', elemId: '_head', insert: true, value: 'magpie', pred: []}, {obj: '1@111111', action: 'del', elemId: '2@111111', insert: false, pred: ['2@111111']} ] }]) }) it('should compress changes with DEFLATE', () => { let longString = '' for (let i = 0; i < 1024; i++) longString += 'a' const change1 = {actor: '111111', seq: 1, time: 0, startOp: 1, deps: [], ops: [ {action: 'set', obj: '_root', key: 'longString', value: longString, pred: []} ]} const [s1, patch1] = Backend.applyLocalChange(Backend.init(), change1) const changes = Backend.getAllChanges(s1) const [s2, patch2] = Backend.applyChanges(Backend.init(), changes) assert.ok(changes[0].byteLength < 100) assert.deepStrictEqual(patch2, { clock: {'111111': 1}, deps: [hash(change1)], maxOp: 1, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: { longString: {'1@111111': {type: 'value', value: longString}} }} }) }) it('should support inserting multiple elements in one change (int)', () => { const actor = uuid() const localChange = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeList', obj: '_root', key: 'todos', pred: []}, {action: 'set', obj: `1@${actor}`, insert: true, elemId: '_head', pred: [], datatype: 'int', values: [1, 2, 3, 4, 5]}, ]} const s0 = Backend.init() const [s1, patch1] = Backend.applyLocalChange(s0, localChange) const changes = Backend.getChanges(s1, []).map(decodeChange) assert.deepStrictEqual(patch1, { clock: {[actor]: 1}, deps: [], maxOp: 6, actor, seq: 1, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {todos: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'list', edits: [ {action: 'multi-insert', index: 0, elemId: `2@${actor}`, datatype: 'int', values: [1, 2, 3, 4, 5]} ] }}}} }) }) it('should support inserting multiple elements in one change (float64)', () => { const actor = uuid() const localChange = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeList', obj: '_root', key: 'todos', pred: []}, {action: 'set', obj: `1@${actor}`, insert: true, elemId: '_head', pred: [], datatype: 'float64', values: [1, 2, 3.3, 4, 5]}, ]} const s0 = Backend.init() const [s1, patch1] = Backend.applyLocalChange(s0, localChange) const changes = Backend.getChanges(s1, []).map(decodeChange) assert.deepStrictEqual(patch1, { clock: {[actor]: 1}, deps: [], maxOp: 6, actor, seq: 1, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {todos: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'list', edits: [ {action: 'multi-insert', index: 0, elemId: `2@${actor}`, datatype: 'float64', values: [1, 2, 3.3, 4, 5]} ] }}}} }) }) it('should support deleting multiple elements in one op', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeList', obj: '_root', key: 'todos', pred: []}, {action: 'set', obj: `1@${actor}`, insert: true, elemId: '_head', pred: [], datatype: 'int', values: [1, 2, 3, 4, 5]} ]} const change2 = {actor, seq: 2, startOp: 7, time: 0, deps: [hash(change1)], ops: [ {action: 'del', obj: `1@${actor}`, elemId: `3@${actor}`, multiOp: 3, pred: [`3@${actor}`]} ]} const s0 = Backend.init() const [s1, patch1] = Backend.applyLocalChange(s0, change1) const [s2, patch2] = Backend.applyLocalChange(s1, change2) assert.deepStrictEqual(patch2, { clock: {[actor]: 2}, deps: [], maxOp: 9, actor, seq: 2, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {todos: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'list', edits: [ {action: 'remove', index: 1, count: 3} ] }}}} }) }) it('should allow a conflict to be resolved', () => { const change1 = {actor: '111111', seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'set', obj: '_root', key: 'bird', value: 'magpie', pred: []} ]} const change2 = {actor: '222222', seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'set', obj: '_root', key: 'bird', value: 'blackbird', pred: []} ]} const change3 = {actor: '333333', seq: 1, startOp: 2, time: 0, deps: [hash(change1), hash(change2)], ops: [ {action: 'set', obj: '_root', key: 'bird', value: 'robin', pred: ['1@111111', '1@222222']} ]} const s0 = Backend.init() const [s1, patch1] = Backend.applyChanges(s0, [encodeChange(change1), encodeChange(change2)]) const [s2, patch2] = Backend.applyLocalChange(s1, change3) assert.deepStrictEqual(patch2, { clock: {111111: 1, 222222: 1, 333333: 1}, deps: [], actor: '333333', seq: 1, maxOp: 2, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: { bird: {'2@333333': {type: 'value', value: 'robin'}} }} }) // Check that we can change the order of `pred` without affecting the outcome change3.ops[0].pred.reverse() const s3 = Backend.init() const [s4, patch4] = Backend.applyChanges(s3, [encodeChange(change1), encodeChange(change2)]) const [s5, patch5] = Backend.applyLocalChange(s4, change3) assert.deepStrictEqual(Backend.getHeads(s2), Backend.getHeads(s5)) }) }) describe('save() and load()', () => { it('should reconstruct changes that resolve conflicts', () => { const actor1 = '8765', actor2 = '1234' const change1 = {actor: actor1, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'set', obj: '_root', key: 'bird', value: 'magpie', pred: []} ]} const change2 = {actor: actor2, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'set', obj: '_root', key: 'bird', value: 'blackbird', pred: []} ]} const change3 = {actor: actor1, seq: 2, startOp: 2, time: 0, deps: [hash(change1), hash(change2)], ops: [ {action: 'set', obj: '_root', key: 'bird', value: 'robin', pred: [`1@${actor1}`, `1@${actor2}`]} ]} const s1 = Backend.loadChanges(Backend.init(), [change1, change2, change3].map(encodeChange)) const s2 = Backend.load(Backend.save(s1)) assert.deepStrictEqual(Backend.getHeads(s2), [hash(change3)]) }) it('should compress columns with DEFLATE', () => { let longString = '' for (let i = 0; i < 1024; i++) longString += 'a' const change1 = {actor: '111111', seq: 1, time: 0, startOp: 1, deps: [], ops: [ {action: 'set', obj: '_root', key: 'longString', value: longString, pred: []} ]} const doc = Backend.save(Backend.loadChanges(Backend.init(), [encodeChange(change1)])) const patch = Backend.getPatch(Backend.load(doc)) assert.ok(doc.byteLength < 200) assert.deepStrictEqual(patch, { clock: {'111111': 1}, deps: [hash(change1)], maxOp: 1, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: { longString: {'1@111111': {type: 'value', value: longString}} }} }) }) it('should load floats correctly', () => { // This was generated from saving a document in the Rust backend // Rust code: // ``` // let initial_state_json: serde_json::Value = serde_json::from_str(r#"{ "birds": 3.0 }"#).unwrap(); // let value = Value::from_json(&initial_state_json); // let (mut frontend, change) = Frontend::new_with_initial_state(value).unwrap(); // let mut backend = Backend::init(); // backend.apply_local_change(change).unwrap(); // let bytes = backend.save().unwrap(); // ``` const bytes = Uint8Array.from([133, 111, 74, 131, 233, 181, 157, 86, 0, 144, 1, 1, 16, 228, 91, 238, 197, 233, 52, 66, 187, 138, 75, 115, 104, 190, 195, 159, 200, 1, 221, 158, 172, 238, 121, 38, 160, 123, 25, 33, 97, 124, 142, 27, 86, 224, 238, 83, 14, 157, 207, 233, 8, 110, 91, 151, 172, 38, 120, 221, 38, 162, 7, 1, 2, 3, 2, 19, 2, 35, 7, 53, 16, 64, 2, 86, 2, 8, 21, 7, 33, 2, 35, 2, 52, 1, 66, 2, 86, 3, 87, 8, 128, 1, 2, 127, 0, 127, 1, 127, 1, 127, 243, 145, 234, 194, 149, 47, 127, 14, 73, 110, 105, 116, 105, 97, 108, 105, 122, 97, 116, 105, 111, 110, 127, 0, 127, 7, 127, 5, 98, 105, 114, 100, 115, 127, 0, 127, 1, 1, 127, 1, 127, 133, 1, 0, 0, 0, 0, 0, 0, 8, 64, 127, 0]) const doc = Automerge.load(bytes) assert.deepStrictEqual(doc, { birds: 3.0 }) }); }) describe('getPatch()', () => { it('should include the most recent value for a key', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'set', obj: '_root', key: 'bird', value: 'magpie', pred: []} ]} const change2 = {actor, seq: 2, startOp: 2, time: 0, deps: [hash(change1)], ops: [ {action: 'set', obj: '_root', key: 'bird', value: 'blackbird', pred: [`1@${actor}`]} ]} const s1 = Backend.loadChanges(Backend.init(), [change1, change2].map(encodeChange)) assert.deepStrictEqual(Backend.getPatch(s1), { clock: {[actor]: 2}, deps: [hash(change2)], maxOp: 2, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: { bird: {[`2@${actor}`]: {type: 'value', value: 'blackbird'}} }} }) }) it('should include conflicting values for a key', () => { const change1 = {actor: '111111', seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'set', obj: '_root', key: 'bird', value: 'magpie', pred: []} ]} const change2 = {actor: '222222', seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'set', obj: '_root', key: 'bird', value: 'blackbird', pred: []} ]} const s1 = Backend.loadChanges(Backend.init(), [change1, change2].map(encodeChange)) assert.deepStrictEqual(Backend.getPatch(s1), { clock: {111111: 1, 222222: 1}, deps: [hash(change1), hash(change2)].sort(), maxOp: 1, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: { bird: {'1@111111': {type: 'value', value: 'magpie'}, '1@222222': {type: 'value', value: 'blackbird'}} }} }) }) it('should handle counter increments at a key in a map', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'set', obj: '_root', key: 'counter', value: 1, datatype: 'counter', pred: []} ]} const change2 = {actor, seq: 2, startOp: 2, time: 0, deps: [hash(change1)], ops: [ {action: 'inc', obj: '_root', key: 'counter', value: 2, pred: [`1@${actor}`]} ]} const s1 = Backend.loadChanges(Backend.init(), [change1, change2].map(encodeChange)) assert.deepStrictEqual(Backend.getPatch(s1), { clock: {[actor]: 2}, deps: [hash(change2)], maxOp: 2, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: { counter: {[`1@${actor}`]: {type: 'value', value: 3, datatype: 'counter'}} }} }) }) it('should handle deletion of a counter', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'set', obj: '_root', key: 'counter', value: 1, datatype: 'counter', pred: []} ]} const change2 = {actor, seq: 2, startOp: 2, time: 0, deps: [hash(change1)], ops: [ {action: 'inc', obj: '_root', key: 'counter', value: 2, pred: [`1@${actor}`]} ]} const change3 = {actor, seq: 3, startOp: 3, time: 0, deps: [hash(change2)], ops: [ {action: 'del', obj: '_root', key: 'counter', pred: [`1@${actor}`]} ]} const s1 = Backend.loadChanges(Backend.init(), [change1, change2, change3].map(encodeChange)) assert.deepStrictEqual(Backend.getPatch(s1), { clock: {[actor]: 3}, deps: [hash(change3)], maxOp: 3, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {}} }) }) it('should create nested maps', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeMap', obj: '_root', key: 'birds', pred: []}, {action: 'set', obj: `1@${actor}`, key: 'wrens', value: 3, pred: []} ]} const change2 = {actor, seq: 2, startOp: 3, time: 0, deps: [hash(change1)], ops: [ {action: 'del', obj: `1@${actor}`, key: 'wrens', pred: [`2@${actor}`]}, {action: 'set', obj: `1@${actor}`, key: 'sparrows', value: 15, pred: []} ]} const s1 = Backend.loadChanges(Backend.init(), [change1, change2].map(encodeChange)) assert.deepStrictEqual(Backend.getPatch(s1), { clock: {[actor]: 2}, deps: [hash(change2)], maxOp: 4, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {birds: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'map', props: {sparrows: {[`4@${actor}`]: {type: 'value', value: 15, datatype: 'int'}}} }}}} }) }) it('should create lists', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeList', obj: '_root', key: 'birds', pred: []}, {action: 'set', obj: `1@${actor}`, elemId: '_head', insert: true, value: 'chaffinch', pred: []} ]} const s1 = Backend.loadChanges(Backend.init(), [encodeChange(change1)]) assert.deepStrictEqual(Backend.getPatch(s1), { clock: {[actor]: 1}, deps: [hash(change1)], maxOp: 2, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {birds: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'list', edits: [ {action: 'insert', index: 0, elemId: `2@${actor}`, opId: `2@${actor}`, value: {type: 'value', value: 'chaffinch'}} ] }}}} }) }) it('should include the latest state of a list', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeList', obj: '_root', key: 'birds', pred: []}, {action: 'set', obj: `1@${actor}`, elemId: '_head', insert: true, value: 'chaffinch', pred: []}, {action: 'set', obj: `1@${actor}`, elemId: `2@${actor}`, insert: true, value: 'goldfinch', pred: []} ]} const change2 = {actor, seq: 2, startOp: 4, time: 0, deps: [hash(change1)], ops: [ {action: 'del', obj: `1@${actor}`, elemId: `2@${actor}`, pred: [`2@${actor}`]}, {action: 'set', obj: `1@${actor}`, elemId: `2@${actor}`, insert: true, value: 'greenfinch', pred: []}, {action: 'set', obj: `1@${actor}`, elemId: `3@${actor}`, value: 'goldfinches!!', pred: [`3@${actor}`]} ]} const s1 = Backend.loadChanges(Backend.init(), [change1, change2].map(encodeChange)) assert.deepStrictEqual(Backend.getPatch(s1), { clock: {[actor]: 2}, deps: [hash(change2)], maxOp: 6, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {birds: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'list', edits: [ {action: 'insert', index: 0, elemId: `5@${actor}`, opId: `5@${actor}`, value: {type: 'value', value: 'greenfinch'}}, {action: 'insert', index: 1, elemId: `3@${actor}`, opId: `6@${actor}`, value: {type: 'value', value: 'goldfinches!!'}} ] }}}} }) }) it('should handle conflicts on list elements', () => { const actor1 = '01234567', actor2 = '89abcdef' const change1 = {actor: actor1, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeList', obj: '_root', key: 'birds', pred: []}, {action: 'set', obj: `1@${actor1}`, elemId: '_head', insert: true, value: 'chaffinch', pred: []}, {action: 'set', obj: `1@${actor1}`, elemId: `2@${actor1}`, insert: true, value: 'magpie', pred: []} ]} const change2 = {actor: actor1, seq: 2, startOp: 4, time: 0, deps: [hash(change1)], ops: [ {action: 'set', obj: `1@${actor1}`, elemId: `2@${actor1}`, value: 'greenfinch', pred: [`2@${actor1}`]} ]} const change3 = {actor: actor2, seq: 1, startOp: 4, time: 0, deps: [hash(change1)], ops: [ {action: 'set', obj: `1@${actor1}`, elemId: `2@${actor1}`, value: 'goldfinch', pred: [`2@${actor1}`]} ]} const s1 = Backend.loadChanges(Backend.init(), [change1, change2, change3].map(encodeChange)) assert.deepStrictEqual(Backend.getPatch(s1), { clock: {[actor1]: 2, [actor2]: 1}, deps: [hash(change2), hash(change3)].sort(), maxOp: 4, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {birds: {[`1@${actor1}`]: { objectId: `1@${actor1}`, type: 'list', edits: [ {action: 'insert', index: 0, elemId: `2@${actor1}`, opId: `4@${actor1}`, value: {type: 'value', value: 'greenfinch'}}, {action: 'update', index: 0, opId: `4@${actor2}`, value: {type: 'value', value: 'goldfinch'}}, {action: 'insert', index: 1, elemId: `3@${actor1}`, opId: `3@${actor1}`, value: {type: 'value', value: 'magpie'}} ] }}}} }) }) it('should handle nested maps in lists', () => { const actor = uuid() const change = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeList', obj: '_root', key: 'todos', pred: []}, {action: 'makeMap', obj: `1@${actor}`, elemId: '_head', insert: true, pred: []}, {action: 'set', obj: `2@${actor}`, key: 'title', value: 'water plants', pred: []}, {action: 'set', obj: `2@${actor}`, key: 'done', value: false, pred: []} ]} const s1 = Backend.loadChanges(Backend.init(), [encodeChange(change)]) assert.deepStrictEqual(Backend.getPatch(s1), { clock: {[actor]: 1}, deps: [hash(change)], maxOp: 4, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {todos: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'list', edits: [ {action: 'insert', index: 0, elemId: `2@${actor}`, opId: `2@${actor}`, value: { type: 'map', objectId: `2@${actor}`, props: { title: {[`3@${actor}`]: {type: 'value', value: 'water plants'}}, done: {[`4@${actor}`]: {type: 'value', value: false}} } }} ] }}}} }) }) it('should include Date objects at the root', () => { const now = new Date() const actor = uuid(), change = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'set', obj: '_root', key: 'now', value: now.getTime(), datatype: 'timestamp', pred: []} ]} const s1 = Backend.loadChanges(Backend.init(), [encodeChange(change)]) assert.deepStrictEqual(Backend.getPatch(s1), { clock: {[actor]: 1}, deps: [hash(change)], maxOp: 1, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: { now: {[`1@${actor}`]: {type: 'value', value: now.getTime(), datatype: 'timestamp'}} }} }) }) it('should include Date objects in a list', () => { const now = new Date(), actor = uuid() const change = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeList', obj: '_root', key: 'list', pred: []}, {action: 'set', obj: `1@${actor}`, elemId: '_head', insert: true, value: now.getTime(), datatype: 'timestamp', pred: []} ]} const s1 = Backend.loadChanges(Backend.init(), [encodeChange(change)]) assert.deepStrictEqual(Backend.getPatch(s1), { clock: {[actor]: 1}, deps: [hash(change)], maxOp: 2, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {list: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'list', edits: [ {action: 'insert', index: 0, elemId: `2@${actor}`, opId: `2@${actor}`, value: {type: 'value', value: now.getTime(), datatype: 'timestamp'}} ] }}}} }) }) it('should condense multiple inserts into a single edit', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeList', obj: '_root', key: 'birds', pred: []}, {action: 'set', obj: `1@${actor}`, elemId: '_head', insert: true, value: 'chaffinch', pred: []}, {action: 'set', obj: `1@${actor}`, elemId: `2@${actor}`, insert: true, value: 'goldfinch', pred: []}, {action: 'set', obj: `1@${actor}`, elemId: `3@${actor}`, insert: true, values: ['bullfinch', 'greenfinch'], pred: []} ]} const s1 = Backend.loadChanges(Backend.init(), [change1].map(encodeChange)) assert.deepStrictEqual(Backend.getPatch(s1), { clock: {[actor]: 1}, deps: [hash(change1)], maxOp: 5, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {birds: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'list', edits: [ {action: 'multi-insert', index: 0, elemId: `2@${actor}`, values: [ 'chaffinch', 'goldfinch', 'bullfinch', 'greenfinch', ]} ] }}}} }) }) it('should use a multi-insert only for consecutive elemIds', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeList', obj: '_root', key: 'birds', pred: []}, {action: 'set', obj: `1@${actor}`, elemId: '_head', insert: true, value: 'chaffinch', pred: []}, {action: 'set', obj: `1@${actor}`, elemId: `2@${actor}`, insert: true, value: 'goldfinch', pred: []}, {action: 'set', obj: `1@${actor}`, elemId: '_head', insert: true, values: ['bullfinch', 'greenfinch'], pred: []} ]} const s1 = Backend.loadChanges(Backend.init(), [change1].map(encodeChange)) assert.deepStrictEqual(Backend.getPatch(s1), { clock: {[actor]: 1}, deps: [hash(change1)], maxOp: 5, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {birds: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'list', edits: [ {action: 'multi-insert', index: 0, elemId: `4@${actor}`, values: ['bullfinch', 'greenfinch']}, {action: 'multi-insert', index: 2, elemId: `2@${actor}`, values: ['chaffinch', 'goldfinch']} ] }}}} }) }) }) }) ================================================ FILE: test/columnar_test.js ================================================ const assert = require('assert') const { checkEncoded } = require('./helpers') const Automerge = process.env.TEST_DIST === '1' ? require('../dist/automerge') : require('../src/automerge') const { encodeChange, decodeChange } = require('../backend/columnar') describe('change encoding', () => { it('should encode text edits', () => { const change1 = {actor: 'aaaa', seq: 1, startOp: 1, time: 9, message: '', deps: [], ops: [ {action: 'makeText', obj: '_root', key: 'text', insert: false, pred: []}, {action: 'set', obj: '1@aaaa', elemId: '_head', insert: true, value: 'h', pred: []}, {action: 'del', obj: '1@aaaa', elemId: '2@aaaa', insert: false, pred: ['2@aaaa']}, {action: 'set', obj: '1@aaaa', elemId: '_head', insert: true, value: 'H', pred: []}, {action: 'set', obj: '1@aaaa', elemId: '4@aaaa', insert: true, value: 'i', pred: []} ]} checkEncoded(encodeChange(change1), [ 0x85, 0x6f, 0x4a, 0x83, // magic bytes 0xe2, 0xbd, 0xfb, 0xf5, // checksum 1, 94, 0, 2, 0xaa, 0xaa, // chunkType: change, length, deps, actor 'aaaa' 1, 1, 9, 0, 0, // seq, startOp, time, message, actor list 12, 0x01, 4, 0x02, 4, // column count, objActor, objCtr 0x11, 8, 0x13, 7, 0x15, 8, // keyActor, keyCtr, keyStr 0x34, 4, 0x42, 6, // insert, action 0x56, 6, 0x57, 3, // valLen, valRaw 0x70, 6, 0x71, 2, 0x73, 2, // predNum, predActor, predCtr 0, 1, 4, 0, // objActor column: null, 0, 0, 0, 0 0, 1, 4, 1, // objCtr column: null, 1, 1, 1, 1 0, 2, 0x7f, 0, 0, 1, 0x7f, 0, // keyActor column: null, null, 0, null, 0 0, 1, 0x7c, 0, 2, 0x7e, 4, // keyCtr column: null, 0, 2, 0, 4 0x7f, 4, 0x74, 0x65, 0x78, 0x74, 0, 4, // keyStr column: 'text', null, null, null, null 1, 1, 1, 2, // insert column: false, true, false, true, true 0x7d, 4, 1, 3, 2, 1, // action column: makeText, set, del, set, set 0x7d, 0, 0x16, 0, 2, 0x16, // valLen column: 0, 0x16, 0, 0x16, 0x16 0x68, 0x48, 0x69, // valRaw column: 'h', 'H', 'i' 2, 0, 0x7f, 1, 2, 0, // predNum column: 0, 0, 1, 0, 0 0x7f, 0, // predActor column: 0 0x7f, 2 // predCtr column: 2 ]) const decoded = decodeChange(encodeChange(change1)) assert.deepStrictEqual(decoded, Object.assign({hash: decoded.hash}, change1)) }) it('should require strict ordering of preds', () => { const change = new Uint8Array([ 133, 111, 74, 131, 31, 229, 112, 44, 1, 105, 1, 58, 30, 190, 100, 253, 180, 180, 66, 49, 126, 81, 142, 10, 3, 35, 140, 189, 231, 34, 145, 57, 66, 23, 224, 149, 64, 97, 88, 140, 168, 194, 229, 4, 244, 209, 58, 138, 67, 140, 1, 152, 236, 250, 2, 0, 1, 4, 55, 234, 66, 242, 8, 21, 11, 52, 1, 66, 2, 86, 3, 87, 10, 112, 2, 113, 3, 115, 4, 127, 9, 99, 111, 109, 109, 111, 110, 86, 97, 114, 1, 127, 1, 127, 166, 1, 52, 48, 57, 49, 52, 57, 52, 53, 56, 50, 127, 2, 126, 0, 1, 126, 139, 1, 0 ]) assert.throws(() => { decodeChange(change) }, /operation IDs are not in ascending order/) }) describe('with trailing bytes', () => { let change = new Uint8Array([ 0x85, 0x6f, 0x4a, 0x83, // magic bytes 0xb2, 0x98, 0x9e, 0xa9, // checksum 1, 61, 0, 2, 0x12, 0x34, // chunkType: change, length, deps, actor '1234' 1, 1, 252, 250, 220, 255, 5, // seq, startOp, time 14, 73, 110, 105, 116, 105, 97, 108, 105, 122, 97, 116, 105, 111, 110, // message: 'Initialization' 0, 6, // actor list, column count 0x15, 3, 0x34, 1, 0x42, 2, // keyStr, insert, action 0x56, 2, 0x57, 1, 0x70, 2, // valLen, valRaw, predNum 0x7f, 1, 0x78, // keyStr: 'x' 1, // insert: false 0x7f, 1, // action: set 0x7f, 19, // valLen: 1 byte of type uint 1, // valRaw: 1 0x7f, 0, // predNum: 0 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 // 10 trailing bytes ]) it('should allow decoding and re-encoding', () => { // NOTE: This calls the JavaScript encoding and decoding functions, even when the WebAssembly // backend is loaded. Should the wasm backend export its own functions for testing? checkEncoded(change, encodeChange(decodeChange(change))) }) it('should be preserved in document encoding', () => { const [doc] = Automerge.applyChanges(Automerge.init(), [change]) const [reconstructed] = Automerge.getAllChanges(Automerge.load(Automerge.save(doc))) checkEncoded(change, reconstructed) }) }) }) ================================================ FILE: test/context_test.js ================================================ const assert = require('assert') const sinon = require('sinon') const { Context } = require('../frontend/context') const { CACHE, OBJECT_ID, CONFLICTS, STATE, ELEM_IDS } = require('../frontend/constants') const { Counter } = require('../frontend/counter') const { Table, instantiateTable } = require('../frontend/table') const { Text } = require('../frontend/text') const uuid = require('../src/uuid') describe('Proxying context', () => { let context, applyPatch beforeEach(() => { applyPatch = sinon.spy() context = new Context({[STATE]: { maxOp: 0 }, [CACHE]: {_root: {}}}, uuid(), applyPatch) }) describe('.setMapKey', () => { it('should assign a primitive value to a map key', () => { context.setMapKey([], 'sparrows', 5) assert(applyPatch.calledOnce) assert.deepStrictEqual(applyPatch.firstCall.args[0], {objectId: '_root', type: 'map', props: { sparrows: {[`1@${context.actorId}`]: {value: 5, datatype: 'int', type: 'value'}} }}) assert.deepStrictEqual(context.ops, [ {obj: '_root', action: 'set', key: 'sparrows', insert: false, datatype: 'int', value: 5, pred: []} ]) }) it('should do nothing if the value was not changed', () => { context.cache._root = {[OBJECT_ID]: '_root', goldfinches: 3, [CONFLICTS]: {goldfinches: {'1@actor1': 3}}} context.setMapKey([], 'goldfinches', 3) assert(applyPatch.notCalled) assert.deepStrictEqual(context.ops, []) }) it('should allow a conflict to be resolved', () => { context.cache._root = {[OBJECT_ID]: '_root', goldfinches: 5, [CONFLICTS]: {goldfinches: {'1@actor1': 3, '2@actor2': 5}}} context.setMapKey([], 'goldfinches', 3) assert(applyPatch.calledOnce) assert.deepStrictEqual(applyPatch.firstCall.args[0], {objectId: '_root', type: 'map', props: { goldfinches: {[`1@${context.actorId}`]: {value: 3, datatype: 'int', type: 'value'}} }}) assert.deepStrictEqual(context.ops, [ {obj: '_root', action: 'set', key: 'goldfinches', insert: false, datatype: 'int', value: 3, pred: ['1@actor1', '2@actor2']} ]) }) it('should create nested maps', () => { context.setMapKey([], 'birds', {goldfinches: 3}) assert(applyPatch.calledOnce) const objectId = applyPatch.firstCall.args[0].props.birds[`1@${context.actorId}`].objectId assert.deepStrictEqual(applyPatch.firstCall.args[0], {objectId: '_root', type: 'map', props: { birds: {[`1@${context.actorId}`]: {objectId, type: 'map', props: { goldfinches: {[`2@${context.actorId}`]: {value: 3, datatype: 'int', type: 'value'}} }}} }}) assert.deepStrictEqual(context.ops, [ {obj: '_root', action: 'makeMap', key: 'birds', insert: false, pred: []}, {obj: objectId, action: 'set', key: 'goldfinches', insert: false, datatype: 'int', value: 3, pred: []} ]) }) it('should perform assignment inside nested maps', () => { const objectId = uuid(), child = {[OBJECT_ID]: objectId} context.cache[objectId] = child context.cache._root = {[OBJECT_ID]: '_root', [CONFLICTS]: {birds: {'1@actor1': child}}, birds: child} context.setMapKey([{key: 'birds', objectId}], 'goldfinches', 3) assert(applyPatch.calledOnce) assert.deepStrictEqual(applyPatch.firstCall.args[0], {objectId: '_root', type: 'map', props: { birds: {'1@actor1': {objectId, type: 'map', props: { goldfinches: {[`1@${context.actorId}`]: {value: 3, datatype: 'int', type: 'value'}} }}} }}) assert.deepStrictEqual(context.ops, [ {obj: objectId, action: 'set', key: 'goldfinches', insert: false, datatype: 'int', value: 3, pred: []} ]) }) it('should perform assignment inside conflicted maps', () => { const objectId1 = uuid(), child1 = {[OBJECT_ID]: objectId1} const objectId2 = uuid(), child2 = {[OBJECT_ID]: objectId2} context.cache[objectId1] = child1 context.cache[objectId2] = child2 context.cache._root = {[OBJECT_ID]: '_root', birds: child2, [CONFLICTS]: {birds: {'1@actor1': child1, '1@actor2': child2}}} context.setMapKey([{key: 'birds', objectId: objectId2}], 'goldfinches', 3) assert(applyPatch.calledOnce) assert.deepStrictEqual(applyPatch.firstCall.args[0], {objectId: '_root', type: 'map', props: {birds: { '1@actor1': {objectId: objectId1, type: 'map', props: {}}, '1@actor2': {objectId: objectId2, type: 'map', props: { goldfinches: {[`1@${context.actorId}`]: {value: 3, datatype: 'int', type: 'value'}} }} }}}) assert.deepStrictEqual(context.ops, [ {obj: objectId2, action: 'set', key: 'goldfinches', insert: false, datatype: 'int', value: 3, pred: []} ]) }) it('should handle conflict values of various types', () => { const objectId = uuid(), child = {[OBJECT_ID]: objectId}, dateValue = new Date() context.cache[objectId] = child context.cache._root = {[OBJECT_ID]: '_root', values: child, [CONFLICTS]: {values: { '1@actor1': dateValue, '1@actor2': new Counter(), '1@actor3': 42, '1@actor4': null, '1@actor5': child }}} context.setMapKey([{key: 'values', objectId}], 'goldfinches', 3) assert(applyPatch.calledOnce) assert.deepStrictEqual(applyPatch.firstCall.args[0], {objectId: '_root', type: 'map', props: {values: { '1@actor1': {value: dateValue.getTime(), datatype: 'timestamp', type: 'value'}, '1@actor2': {value: 0, datatype: 'counter', type: 'value'}, '1@actor3': {value: 42, datatype: 'int', type: 'value'}, '1@actor4': {value: null, type: 'value'}, '1@actor5': {objectId, type: 'map', props: {goldfinches: {[`1@${context.actorId}`]: {value: 3, type: 'value', datatype: 'int' }}}} }}}) assert.deepStrictEqual(context.ops, [ {obj: objectId, action: 'set', key: 'goldfinches', insert: false, datatype: 'int', value: 3, pred: []} ]) }) it('should create nested lists', () => { context.setMapKey([], 'birds', ['sparrow', 'goldfinch']) assert(applyPatch.calledOnce) const objectId = applyPatch.firstCall.args[0].props.birds[`1@${context.actorId}`].objectId assert.deepStrictEqual(applyPatch.firstCall.args[0], {objectId: '_root', type: 'map', props: { birds: {[`1@${context.actorId}`]: {objectId, type: 'list', edits: [ {action: 'multi-insert', index: 0, elemId: `2@${context.actorId}`, values: ['sparrow', 'goldfinch']} ]}} }}) assert.deepStrictEqual(context.ops, [ {obj: '_root', action: 'makeList', key: 'birds', insert: false, pred: []}, {obj: objectId, action: 'set', elemId: '_head', insert: true, values: ['sparrow', 'goldfinch'], pred: []} ]) }) it('should create nested Text objects', () => { context.setMapKey([], 'text', new Text('hi')) const objectId = applyPatch.firstCall.args[0].props.text[`1@${context.actorId}`].objectId assert(applyPatch.calledOnce) assert.deepStrictEqual(applyPatch.firstCall.args[0], {objectId: '_root', type: 'map', props: { text: {[`1@${context.actorId}`]: {objectId, type: 'text', edits: [ {action: 'multi-insert', index: 0, elemId: `2@${context.actorId}`, values: ['h', 'i']} ]}} }}) assert.deepStrictEqual(context.ops, [ {obj: '_root', action: 'makeText', key: 'text', insert: false, pred: []}, {obj: objectId, action: 'set', elemId: '_head', insert: true, values: ['h', 'i'], pred: []} ]) }) it('should create nested Table objects', () => { context.setMapKey([], 'books', new Table()) assert(applyPatch.calledOnce) const objectId = applyPatch.firstCall.args[0].props.books[`1@${context.actorId}`].objectId assert.deepStrictEqual(applyPatch.firstCall.args[0], {objectId: '_root', type: 'map', props: { books: {[`1@${context.actorId}`]: {objectId, type: 'table', props: {}}} }}) assert.deepStrictEqual(context.ops, [ {obj: '_root', action: 'makeTable', key: 'books', insert: false, pred: []} ]) }) it('should allow assignment of Date values', () => { const now = new Date() context.setMapKey([], 'now', now) assert(applyPatch.calledOnce) assert.deepStrictEqual(applyPatch.firstCall.args[0], {objectId: '_root', type: 'map', props: { now: {[`1@${context.actorId}`]: {value: now.getTime(), datatype: 'timestamp', type: 'value'}} }}) assert.deepStrictEqual(context.ops, [ {obj: '_root', action: 'set', key: 'now', insert: false, value: now.getTime(), datatype: 'timestamp', pred: []} ]) }) it('should allow assignment of Counter values', () => { const counter = new Counter(3) context.setMapKey([], 'counter', counter) assert(applyPatch.calledOnce) assert.deepStrictEqual(applyPatch.firstCall.args[0], {objectId: '_root', type: 'map', props: { counter: {[`1@${context.actorId}`]: {value: 3, datatype: 'counter', type: 'value'}} }}) assert.deepStrictEqual(context.ops, [ {obj: '_root', action: 'set', key: 'counter', insert: false, value: 3, datatype: 'counter', pred: []} ]) }) }) describe('.deleteMapKey', () => { it('should remove an existing key', () => { context.cache._root = {[OBJECT_ID]: '_root', goldfinches: 3, [CONFLICTS]: {goldfinches: {'1@actor1': 3}}} context.deleteMapKey([], 'goldfinches') assert(applyPatch.calledOnce) assert.deepStrictEqual(applyPatch.firstCall.args[0], {objectId: '_root', type: 'map', props: {goldfinches: {}}}) assert.deepStrictEqual(context.ops, [ {obj: '_root', action: 'del', key: 'goldfinches', insert: false, pred: ['1@actor1']} ]) }) it('should do nothing if the key does not exist', () => { context.cache._root = {[OBJECT_ID]: '_root', goldfinches: 3, [CONFLICTS]: {goldfinches: {'1@actor1': 3}}} context.deleteMapKey([], 'sparrows') assert(applyPatch.notCalled) assert.deepStrictEqual(context.ops, []) }) it('should update a nested object', () => { const objectId = uuid(), child = {[OBJECT_ID]: objectId, [CONFLICTS]: {goldfinches: {'5@actor1': 3}}, goldfinches: 3} context.cache[objectId] = child context.cache._root = {[OBJECT_ID]: '_root', [CONFLICTS]: {birds: {'1@actor1': child}}, birds: child} context.deleteMapKey([{key: 'birds', objectId}], 'goldfinches') assert(applyPatch.calledOnce) assert.deepStrictEqual(applyPatch.firstCall.args[0], {objectId: '_root', type: 'map', props: { birds: {'1@actor1': {objectId, type: 'map', props: {goldfinches: {}}}} }}) assert.deepStrictEqual(context.ops, [ {obj: objectId, action: 'del', key: 'goldfinches', insert: false, pred: ['5@actor1']} ]) }) }) describe('list manipulation', () => { let listId, list beforeEach(() => { listId = uuid() list = ['swallow', 'magpie'] Object.defineProperty(list, OBJECT_ID, {value: listId}) Object.defineProperty(list, CONFLICTS, {value: [{'1@xxx': 'swallow'}, {'2@xxx': 'magpie'}]}) Object.defineProperty(list, ELEM_IDS, {value: ['1@xxx', '2@xxx']}) context.cache[listId] = list context.cache._root = {[OBJECT_ID]: '_root', birds: list, [CONFLICTS]: {birds: {'1@actor1': list}}} }) it('should overwrite an existing list element', () => { context.setListIndex([{key: 'birds', objectId: listId}], 0, 'starling') assert(applyPatch.calledOnce) assert.deepStrictEqual(applyPatch.firstCall.args[0], {objectId: '_root', type: 'map', props: { birds: {'1@actor1': {objectId: listId, type: 'list', edits: [ {action: 'update', index: 0, opId: `1@${context.actorId}`, value: {value: 'starling', type: 'value'}} ]}} }}) assert.deepStrictEqual(context.ops, [ {obj: listId, action: 'set', elemId: '1@xxx', insert: false, value: 'starling', pred: ['1@xxx']} ]) }) it('should create nested objects on assignment', () => { context.setListIndex([{key: 'birds', objectId: listId}], 1, {english: 'goldfinch', latin: 'carduelis'}) assert(applyPatch.calledOnce) const nestedId = applyPatch.firstCall.args[0].props.birds['1@actor1'].edits[0].value.objectId assert.deepStrictEqual(applyPatch.firstCall.args[0], {objectId: '_root', type: 'map', props: { birds: {'1@actor1': {objectId: listId, type: 'list', edits: [{ action: 'update', index: 1, opId: `1@${context.actorId}`, value: { objectId: nestedId, type: 'map', props: { english: {[`2@${context.actorId}`]: {value: 'goldfinch', type: 'value'}}, latin: {[`3@${context.actorId}`]: {value: 'carduelis', type: 'value'}} } } }]}} }}) assert.deepStrictEqual(context.ops, [ {obj: listId, action: 'makeMap', elemId: '2@xxx', insert: false, pred: ['2@xxx']}, {obj: nestedId, action: 'set', key: 'english', insert: false, value: 'goldfinch', pred: []}, {obj: nestedId, action: 'set', key: 'latin', insert: false, value: 'carduelis', pred: []} ]) }) it('should create nested objects on insertion', () => { context.splice([{key: 'birds', objectId: listId}], 2, 0, [{english: 'goldfinch', latin: 'carduelis'}]) assert(applyPatch.calledOnce) const nestedId = applyPatch.firstCall.args[0].props.birds['1@actor1'].edits[0].value.objectId assert.deepStrictEqual(applyPatch.firstCall.args[0], {objectId: '_root', type: 'map', props: { birds: {'1@actor1': {objectId: listId, type: 'list', edits: [ {action: 'insert', index: 2, elemId: `1@${context.actorId}`, opId: `1@${context.actorId}`, value: { objectId: nestedId, type: 'map', props: { english: {[`2@${context.actorId}`]: {value: 'goldfinch', type: 'value'}}, latin: {[`3@${context.actorId}`]: {value: 'carduelis', type: 'value'}} } }} ]}} }}) assert.deepStrictEqual(context.ops, [ {obj: listId, action: 'makeMap', elemId: '2@xxx', insert: true, pred: []}, {obj: nestedId, action: 'set', key: 'english', insert: false, value: 'goldfinch', pred: []}, {obj: nestedId, action: 'set', key: 'latin', insert: false, value: 'carduelis', pred: []} ]) }) it('should generate multi-inserts when splicing arrays of primitives', () => { context.splice([{key: 'birds', objectId: listId}], 2, 0, ['goldfinch', 'greenfinch']) assert(applyPatch.calledOnce) assert.deepStrictEqual(applyPatch.firstCall.args[0], {objectId: '_root', type: 'map', props: { birds: {'1@actor1': {objectId: listId, type: 'list', edits: [ {action: 'multi-insert', index: 2, elemId: `1@${context.actorId}`, values: ['goldfinch', 'greenfinch']} ]}} }}) assert.deepStrictEqual(context.ops, [ {obj: listId, action: 'set', elemId: '2@xxx', insert: true, values: ['goldfinch', 'greenfinch'], pred: []} ]) }) it('should support deleting list elements', () => { context.splice([{key: 'birds', objectId: listId}], 0, 1, []) assert(applyPatch.calledOnce) assert.deepStrictEqual(applyPatch.firstCall.args[0], {objectId: '_root', type: 'map', props: { birds: {'1@actor1': {objectId: listId, type: 'list', edits: [ {action: 'remove', index: 0, count: 1} ]}} }}) assert.deepStrictEqual(context.ops, [ {obj: listId, action: 'del', elemId: '1@xxx', insert: false, pred: ['1@xxx']} ]) }) it('should support deleting multiple list elements as a multiOp', () => { context.splice([{key: 'birds', objectId: listId}], 0, 2, []) assert(applyPatch.calledOnce) assert.deepStrictEqual(applyPatch.firstCall.args[0], {objectId: '_root', type: 'map', props: { birds: {'1@actor1': {objectId: listId, type: 'list', edits: [ {action: 'remove', index: 0, count: 2} ]}} }}) assert.deepStrictEqual(context.ops, [ {obj: listId, action: 'del', elemId: '1@xxx', multiOp: 2, insert: false, pred: ['1@xxx']} ]) }) it('should use multiOps for consecutive runs of elemIds', () => { list.unshift('sparrow') list[ELEM_IDS].unshift('3@xxx') list[CONFLICTS].unshift({'3@xxx': 'sparrow'}) context.splice([{key: 'birds', objectId: listId}], 0, 3, []) assert(applyPatch.calledOnce) assert.deepStrictEqual(applyPatch.firstCall.args[0], {objectId: '_root', type: 'map', props: { birds: {'1@actor1': {objectId: listId, type: 'list', edits: [ {action: 'remove', index: 0, count: 3} ]}} }}) assert.deepStrictEqual(context.ops, [ {obj: listId, action: 'del', elemId: '3@xxx', insert: false, pred: ['3@xxx']}, {obj: listId, action: 'del', elemId: '1@xxx', multiOp: 2, insert: false, pred: ['1@xxx']} ]) }) it('should use multiOps for consecutive runs of preds', () => { list[1] = 'sparrow' list[CONFLICTS][1] = {'3@xxx': 'sparrow'} context.splice([{key: 'birds', objectId: listId}], 0, 2, []) assert(applyPatch.calledOnce) assert.deepStrictEqual(applyPatch.firstCall.args[0], {objectId: '_root', type: 'map', props: { birds: {'1@actor1': {objectId: listId, type: 'list', edits: [ {action: 'remove', index: 0, count: 2} ]}} }}) assert.deepStrictEqual(context.ops, [ {obj: listId, action: 'del', elemId: '1@xxx', insert: false, pred: ['1@xxx']}, {obj: listId, action: 'del', elemId: '2@xxx', insert: false, pred: ['3@xxx']} ]) }) it('should support list splicing', () => { context.splice([{key: 'birds', objectId: listId}], 0, 1, ['starling', 'goldfinch']) assert(applyPatch.calledOnce) assert.deepStrictEqual(applyPatch.firstCall.args[0], {objectId: '_root', type: 'map', props: { birds: {'1@actor1': {objectId: listId, type: 'list', edits: [ {action: 'remove', index: 0, count: 1}, {action: 'multi-insert', index: 0, elemId: `2@${context.actorId}`, values: ['starling', 'goldfinch']} ]}} }}) assert.deepStrictEqual(context.ops, [ {obj: listId, action: 'del', elemId: '1@xxx', insert: false, pred: ['1@xxx']}, {obj: listId, action: 'set', elemId: '_head', insert: true, values: ['starling', 'goldfinch'], pred: []} ]) }) }) describe('Table manipulation', () => { let tableId, table beforeEach(() => { tableId = uuid() table = instantiateTable(tableId) context.cache[tableId] = table context.cache._root = {[OBJECT_ID]: '_root', books: table, [CONFLICTS]: {books: {'1@actor1': table}}} }) it('should add a table row', () => { const rowId = context.addTableRow([{key: 'books', objectId: tableId}], {author: 'Mary Shelley', title: 'Frankenstein'}) assert(applyPatch.calledOnce) assert.deepStrictEqual(applyPatch.firstCall.args[0], {objectId: '_root', type: 'map', props: { books: {'1@actor1': {objectId: tableId, type: 'table', props: { [rowId]: {[`1@${context.actorId}`]: {objectId: `1@${context.actorId}`, type: 'map', props: { author: {[`2@${context.actorId}`]: {value: 'Mary Shelley', type: 'value'}}, title: {[`3@${context.actorId}`]: {value: 'Frankenstein', type: 'value'}} }}} }}} }}) assert.deepStrictEqual(context.ops, [ {obj: tableId, action: 'makeMap', key: rowId, insert: false, pred: []}, {obj: `1@${context.actorId}`, action: 'set', key: 'author', insert: false, value: 'Mary Shelley', pred: []}, {obj: `1@${context.actorId}`, action: 'set', key: 'title', insert: false, value: 'Frankenstein', pred: []} ]) }) it('should delete a table row', () => { const rowId = uuid() const row = {author: 'Mary Shelley', title: 'Frankenstein'} row[OBJECT_ID] = rowId table.entries[rowId] = row context.deleteTableRow([{key: 'books', objectId: tableId}], rowId, '5@actor1') assert(applyPatch.calledOnce) assert.deepStrictEqual(applyPatch.firstCall.args[0], {objectId: '_root', type: 'map', props: { books: {'1@actor1': {objectId: tableId, type: 'table', props: {[rowId]: {}}}} }}) assert.deepStrictEqual(context.ops, [ {obj: tableId, action: 'del', key: rowId, insert: false, pred: ['5@actor1']} ]) }) }) it('should increment a counter', () => { const counter = new Counter() context.cache._root = {[OBJECT_ID]: '_root', counter, [CONFLICTS]: {counter: {'1@actor1': counter}}} context.increment([], 'counter', 1) assert(applyPatch.calledOnce) assert.deepStrictEqual(applyPatch.firstCall.args[0], {objectId: '_root', type: 'map', props: { counter: {[`1@${context.actorId}`]: {value: 1, datatype: 'counter'}} }}) assert.deepStrictEqual(context.ops, [{obj: '_root', action: 'inc', key: 'counter', insert: false, value: 1, pred: ['1@actor1']}]) }) }) ================================================ FILE: test/encoding_test.js ================================================ const assert = require('assert') const { checkEncoded } = require('./helpers') const { Encoder, Decoder, RLEEncoder, RLEDecoder, DeltaEncoder, DeltaDecoder, BooleanEncoder, BooleanDecoder } = require('../backend/encoding') describe('Binary encoding', () => { describe('Encoder and Decoder', () => { describe('32-bit LEB128 encoding', () => { it('should encode unsigned integers', () => { function encode(value) { const encoder = new Encoder() encoder.appendUint32(value) return encoder } checkEncoded(encode(0), [0]) checkEncoded(encode(1), [1]) checkEncoded(encode(0x42), [0x42]) checkEncoded(encode(0x7f), [0x7f]) checkEncoded(encode(0x80), [0x80, 0x01]) checkEncoded(encode(0xff), [0xff, 0x01]) checkEncoded(encode(0x1234), [0xb4, 0x24]) checkEncoded(encode(0x3fff), [0xff, 0x7f]) checkEncoded(encode(0x4000), [0x80, 0x80, 0x01]) checkEncoded(encode(0x5678), [0xf8, 0xac, 0x01]) checkEncoded(encode(0xfffff), [0xff, 0xff, 0x3f]) checkEncoded(encode(0x1fffff), [0xff, 0xff, 0x7f]) checkEncoded(encode(0x200000), [0x80, 0x80, 0x80, 0x01]) checkEncoded(encode(0xfffffff), [0xff, 0xff, 0xff, 0x7f]) checkEncoded(encode(0x10000000), [0x80, 0x80, 0x80, 0x80, 0x01]) checkEncoded(encode(0x7fffffff), [0xff, 0xff, 0xff, 0xff, 0x07]) checkEncoded(encode(0x87654321), [0xa1, 0x86, 0x95, 0xbb, 0x08]) checkEncoded(encode(0xffffffff), [0xff, 0xff, 0xff, 0xff, 0x0f]) }) it('should round-trip unsigned integers', () => { const examples = [ 0, 1, 0x42, 0x7f, 0x80, 0xff, 0x1234, 0x3fff, 0x4000, 0x5678, 0xfffff, 0x1fffff, 0x200000, 0xfffffff, 0x10000000, 0x7fffffff, 0x87654321, 0xffffffff ] for (let value of examples) { const encoder = new Encoder() encoder.appendUint32(value) const decoder = new Decoder(encoder.buffer) assert.strictEqual(decoder.readUint32(), value) assert.strictEqual(decoder.done, true) } }) it('should encode signed integers', () => { function encode(value) { const encoder = new Encoder() encoder.appendInt32(value) return encoder } checkEncoded(encode(0), [0]) checkEncoded(encode(1), [1]) checkEncoded(encode(-1), [0x7f]) checkEncoded(encode(0x3f), [0x3f]) checkEncoded(encode(0x40), [0xc0, 0x00]) checkEncoded(encode(-0x3f), [0x41]) checkEncoded(encode(-0x40), [0x40]) checkEncoded(encode(-0x41), [0xbf, 0x7f]) checkEncoded(encode(0x1fff), [0xff, 0x3f]) checkEncoded(encode(0x2000), [0x80, 0xc0, 0x00]) checkEncoded(encode(-0x2000), [0x80, 0x40]) checkEncoded(encode(-0x2001), [0xff, 0xbf, 0x7f]) checkEncoded(encode(0xfffff), [0xff, 0xff, 0x3f]) checkEncoded(encode(0x100000), [0x80, 0x80, 0xc0, 0x00]) checkEncoded(encode(-0x100000), [0x80, 0x80, 0x40]) checkEncoded(encode(-0x100001), [0xff, 0xff, 0xbf, 0x7f]) checkEncoded(encode(0x7ffffff), [0xff, 0xff, 0xff, 0x3f]) checkEncoded(encode(0x8000000), [0x80, 0x80, 0x80, 0xc0, 0x00]) checkEncoded(encode(-0x8000000), [0x80, 0x80, 0x80, 0x40]) checkEncoded(encode(-0x8000001), [0xff, 0xff, 0xff, 0xbf, 0x7f]) checkEncoded(encode(0x76543210), [0x90, 0xe4, 0xd0, 0xb2, 0x07]) checkEncoded(encode(-0x76543210), [0xf0, 0x9b, 0xaf, 0xcd, 0x78]) checkEncoded(encode(0x7fffffff), [0xff, 0xff, 0xff, 0xff, 0x07]) checkEncoded(encode(-0x80000000), [0x80, 0x80, 0x80, 0x80, 0x78]) }) it('should round-trip signed integers', () => { const examples = [ 0, 1, -1, 0x3f, 0x40, -0x3f, -0x40, -0x41, 0x1fff, 0x2000, -0x2000, -0x2001, 0xfffff, 0x100000, -0x100000, -0x100001, 0x7ffffff, 0x8000000, -0x8000000, -0x8000001, 0x76543210, -0x76543210, 0x7fffffff, -0x80000000 ] for (let value of examples) { const encoder = new Encoder() encoder.appendInt32(value) const decoder = new Decoder(encoder.buffer) assert.strictEqual(decoder.readInt32(), value) assert.strictEqual(decoder.done, true) } }) it('should not encode values that are out of range', () => { assert.throws(() => { new Encoder().appendUint32(0x100000000) }, /out of range/) assert.throws(() => { new Encoder().appendUint32(Number.MAX_SAFE_INTEGER) }, /out of range/) assert.throws(() => { new Encoder().appendUint32(-1) }, /out of range/) assert.throws(() => { new Encoder().appendUint32(-0x80000000) }, /out of range/) assert.throws(() => { new Encoder().appendUint32(Number.NEGATIVE_INFINITY) }, /not an integer/) assert.throws(() => { new Encoder().appendUint32(Number.NaN) }, /not an integer/) assert.throws(() => { new Encoder().appendUint32(Math.PI) }, /not an integer/) assert.throws(() => { new Encoder().appendInt32(0x80000000) }, /out of range/) assert.throws(() => { new Encoder().appendInt32(Number.MAX_SAFE_INTEGER) }, /out of range/) assert.throws(() => { new Encoder().appendInt32(-0x80000001) }, /out of range/) assert.throws(() => { new Encoder().appendInt32(Number.NEGATIVE_INFINITY) }, /not an integer/) assert.throws(() => { new Encoder().appendInt32(Number.NaN) }, /not an integer/) assert.throws(() => { new Encoder().appendInt32(Math.PI) }, /not an integer/) }) it('should not decode values that are out of range', () => { assert.throws(() => { new Decoder(new Uint8Array([0x80, 0x80, 0x80, 0x80, 0x80, 0x00])).readUint32() }, /out of range/) assert.throws(() => { new Decoder(new Uint8Array([0x80, 0x80, 0x80, 0x80, 0x80, 0x00])).readInt32() }, /out of range/) assert.throws(() => { new Decoder(new Uint8Array([0x80, 0x80, 0x80, 0x80, 0x10])).readUint32() }, /out of range/) assert.throws(() => { new Decoder(new Uint8Array([0x80, 0x80, 0x80, 0x80, 0x08])).readInt32() }, /out of range/) assert.throws(() => { new Decoder(new Uint8Array([0xff, 0xff, 0xff, 0xff, 0x77])).readInt32() }, /out of range/) assert.throws(() => { new Decoder(new Uint8Array([0x80, 0x80])).readUint32() }, /incomplete number/) assert.throws(() => { new Decoder(new Uint8Array([0x80, 0x80])).readInt32() }, /incomplete number/) }) }) describe('53-bit LEB128 encoding', () => { it('should encode unsigned integers', () => { function encode(value) { const encoder = new Encoder() encoder.appendUint53(value) return encoder } checkEncoded(encode(0), [0]) checkEncoded(encode(0x7f), [0x7f]) checkEncoded(encode(0x80), [0x80, 0x01]) checkEncoded(encode(0x3fff), [0xff, 0x7f]) checkEncoded(encode(0x4000), [0x80, 0x80, 0x01]) checkEncoded(encode(0x1fffff), [0xff, 0xff, 0x7f]) checkEncoded(encode(0x200000), [0x80, 0x80, 0x80, 0x01]) checkEncoded(encode(0xfffffff), [0xff, 0xff, 0xff, 0x7f]) checkEncoded(encode(0x10000000), [0x80, 0x80, 0x80, 0x80, 0x01]) checkEncoded(encode(0xffffffff), [0xff, 0xff, 0xff, 0xff, 0x0f]) checkEncoded(encode(0x100000000), [0x80, 0x80, 0x80, 0x80, 0x10]) checkEncoded(encode(0x7ffffffff), [0xff, 0xff, 0xff, 0xff, 0x7f]) checkEncoded(encode(0x800000000), [0x80, 0x80, 0x80, 0x80, 0x80, 0x01]) checkEncoded(encode(0x3ffffffffff), [0xff, 0xff, 0xff, 0xff, 0xff, 0x7f]) checkEncoded(encode(0x40000000000), [0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x01]) checkEncoded(encode(0x2000000000000), [0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x01]) checkEncoded(encode(0x123456789abcde), [0xde, 0xf9, 0xea, 0xc4, 0xe7, 0x8a, 0x8d, 0x09]) checkEncoded(encode(Number.MAX_SAFE_INTEGER), [0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x0f]) }) it('should round-trip unsigned integers', () => { const examples = [ 0, 0x7f, 0x80, 0x3fff, 0x4000, 0x1fffff, 0x200000, 0xfffffff, 0x10000000, 0xffffffff, 0x100000000, 0x7ffffffff, 0x800000000, 0x3ffffffffff, 0x40000000000, 0x2000000000000, 0x123456789abcde, Number.MAX_SAFE_INTEGER ] for (let value of examples) { const encoder = new Encoder() encoder.appendUint53(value) const decoder = new Decoder(encoder.buffer) assert.strictEqual(decoder.readUint53(), value) assert.strictEqual(decoder.done, true) } }) it('should encode signed integers', () => { function encode(value) { const encoder = new Encoder() encoder.appendInt53(value) return encoder } checkEncoded(encode(0), [0]) checkEncoded(encode(1), [1]) checkEncoded(encode(-1), [0x7f]) checkEncoded(encode(0x3f), [0x3f]) checkEncoded(encode(-0x40), [0x40]) checkEncoded(encode(0x40), [0xc0, 0x00]) checkEncoded(encode(-0x41), [0xbf, 0x7f]) checkEncoded(encode(0x1fff), [0xff, 0x3f]) checkEncoded(encode(-0x2000), [0x80, 0x40]) checkEncoded(encode(0x2000), [0x80, 0xc0, 0x00]) checkEncoded(encode(-0x2001), [0xff, 0xbf, 0x7f]) checkEncoded(encode(0xfffff), [0xff, 0xff, 0x3f]) checkEncoded(encode(-0x100000), [0x80, 0x80, 0x40]) checkEncoded(encode(0x100000), [0x80, 0x80, 0xc0, 0x00]) checkEncoded(encode(-0x100001), [0xff, 0xff, 0xbf, 0x7f]) checkEncoded(encode(0x7ffffff), [0xff, 0xff, 0xff, 0x3f]) checkEncoded(encode(-0x8000000), [0x80, 0x80, 0x80, 0x40]) checkEncoded(encode(0x8000000), [0x80, 0x80, 0x80, 0xc0, 0x00]) checkEncoded(encode(-0x8000001), [0xff, 0xff, 0xff, 0xbf, 0x7f]) checkEncoded(encode(0x7fffffff), [0xff, 0xff, 0xff, 0xff, 0x07]) checkEncoded(encode(0x80000000), [0x80, 0x80, 0x80, 0x80, 0x08]) checkEncoded(encode(-0x80000000), [0x80, 0x80, 0x80, 0x80, 0x78]) checkEncoded(encode(-0x80000001), [0xff, 0xff, 0xff, 0xff, 0x77]) checkEncoded(encode(0x3ffffffff), [0xff, 0xff, 0xff, 0xff, 0x3f]) checkEncoded(encode(-0x400000000), [0x80, 0x80, 0x80, 0x80, 0x40]) checkEncoded(encode(0x400000000), [0x80, 0x80, 0x80, 0x80, 0xc0, 0x00]) checkEncoded(encode(-0x400000001), [0xff, 0xff, 0xff, 0xff, 0xbf, 0x7f]) checkEncoded(encode(0x1ffffffffff), [0xff, 0xff, 0xff, 0xff, 0xff, 0x3f]) checkEncoded(encode(-0x20000000000), [0x80, 0x80, 0x80, 0x80, 0x80, 0x40]) checkEncoded(encode(0x20000000000), [0x80, 0x80, 0x80, 0x80, 0x80, 0xc0, 0x00]) checkEncoded(encode(-0x20000000001), [0xff, 0xff, 0xff, 0xff, 0xff, 0xbf, 0x7f]) checkEncoded(encode(0xffffffffffff), [0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f]) checkEncoded(encode(-0x1000000000000), [0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x40]) checkEncoded(encode(0x1000000000000), [0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0xc0, 0x00]) checkEncoded(encode(-0x1000000000001), [0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xbf, 0x7f]) checkEncoded(encode(0x123456789abcde), [0xde, 0xf9, 0xea, 0xc4, 0xe7, 0x8a, 0x8d, 0x09]) checkEncoded(encode(Number.MAX_SAFE_INTEGER), [0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x0f]) checkEncoded(encode(Number.MIN_SAFE_INTEGER), [0x81, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x70]) }) it('should round-trip signed integers', () => { const examples = [ 0, 1, -1, 0x3f, -0x40, 0x40, -0x41, 0x1fff, -0x2000, 0x2000, -0x2001, 0xfffff, -0x100000, 0x100000, -0x100001, 0x7ffffff, -0x8000000, 0x8000000, -0x8000001, 0x7fffffff, 0x80000000, -0x80000000, -0x80000001, 0x3ffffffff, -0x400000000, 0x400000000, -0x400000001, 0x1ffffffffff, -0x20000000000, 0x20000000000, -0x20000000001, 0xffffffffffff, -0x1000000000000, 0x1000000000000, -0x1000000000001, Number.MAX_SAFE_INTEGER, Number.MIN_SAFE_INTEGER, 0x123, -0x123, 0x1234, -0x1234, 0x12345, -0x12345, 0x123456, -0x123456, 0x1234567, -0x1234567, 0x12345678, -0x12345678, 0x123456789, -0x123456789, 0x123456789a, -0x123456789a, 0x123456789ab, -0x123456789ab, 0x123456789abc, -0x123456789abc, 0x123456789abcd, -0x123456789abcd, 0x123456789abcde, -0x123456789abcde ] for (let value of examples) { const encoder = new Encoder() encoder.appendInt53(value) const decoder = new Decoder(encoder.buffer) assert.strictEqual(decoder.readInt53(), value) assert.strictEqual(decoder.done, true) } }) it('should not encode values that are out of range', () => { assert.throws(() => { new Encoder().appendUint53(Number.MAX_SAFE_INTEGER + 1) }, /out of range/) assert.throws(() => { new Encoder().appendUint53(-1) }, /out of range/) assert.throws(() => { new Encoder().appendUint53(-0x80000000) }, /out of range/) assert.throws(() => { new Encoder().appendUint53(Number.MIN_SAFE_INTEGER) }, /out of range/) assert.throws(() => { new Encoder().appendUint53(Number.NEGATIVE_INFINITY) }, /not an integer/) assert.throws(() => { new Encoder().appendUint53(Number.NaN) }, /not an integer/) assert.throws(() => { new Encoder().appendUint53(Math.PI) }, /not an integer/) assert.throws(() => { new Encoder().appendInt53(Number.MAX_SAFE_INTEGER + 1) }, /out of range/) assert.throws(() => { new Encoder().appendInt53(Number.MIN_SAFE_INTEGER - 1) }, /out of range/) assert.throws(() => { new Encoder().appendInt53(Number.NEGATIVE_INFINITY) }, /not an integer/) assert.throws(() => { new Encoder().appendInt53(Number.NaN) }, /not an integer/) assert.throws(() => { new Encoder().appendInt53(Math.PI) }, /not an integer/) }) it('should not decode values that are out of range', () => { assert.throws(() => { new Decoder(new Uint8Array([0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x10])).readUint53() }, /out of range/) assert.throws(() => { new Decoder(new Uint8Array([0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x10])).readInt53() }, /out of range/) assert.throws(() => { new Decoder(new Uint8Array([0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x70])).readInt53() }, /out of range/) assert.throws(() => { new Decoder(new Uint8Array([0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x6f])).readInt53() }, /out of range/) assert.throws(() => { new Decoder(new Uint8Array([0x80, 0x80])).readUint53() }, /incomplete number/) assert.throws(() => { new Decoder(new Uint8Array([0x80, 0x80])).readInt53() }, /incomplete number/) }) }) describe('64-bit LEB128 encoding', () => { it('should encode unsigned integers', () => { function encode(high32, low32) { const encoder = new Encoder() encoder.appendUint64(high32, low32) return encoder } checkEncoded(encode(0, 0), [0]) checkEncoded(encode(0, 0x7f), [0x7f]) checkEncoded(encode(0, 0x80), [0x80, 0x01]) checkEncoded(encode(0, 0x3fff), [0xff, 0x7f]) checkEncoded(encode(0, 0x4000), [0x80, 0x80, 0x01]) checkEncoded(encode(0, 0x1fffff), [0xff, 0xff, 0x7f]) checkEncoded(encode(0, 0x200000), [0x80, 0x80, 0x80, 0x01]) checkEncoded(encode(0, 0xfffffff), [0xff, 0xff, 0xff, 0x7f]) checkEncoded(encode(0, 0x10000000), [0x80, 0x80, 0x80, 0x80, 0x01]) checkEncoded(encode(0, 0xffffffff), [0xff, 0xff, 0xff, 0xff, 0x0f]) checkEncoded(encode(0x1, 0x00000000), [0x80, 0x80, 0x80, 0x80, 0x10]) checkEncoded(encode(0x7, 0xffffffff), [0xff, 0xff, 0xff, 0xff, 0x7f]) checkEncoded(encode(0x8, 0x00000000), [0x80, 0x80, 0x80, 0x80, 0x80, 0x01]) checkEncoded(encode(0x3ff, 0xffffffff), [0xff, 0xff, 0xff, 0xff, 0xff, 0x7f]) checkEncoded(encode(0x400, 0x00000000), [0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x01]) checkEncoded(encode(0x1ffff, 0xffffffff), [0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f]) checkEncoded(encode(0x20000, 0x00000000), [0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x01]) checkEncoded(encode(0xffffff, 0xffffffff), [0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f]) checkEncoded(encode(0x1000000, 0x00000000), [0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x01]) checkEncoded(encode(0xffffffff, 0xffffffff), [0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01]) }) it('should round-trip unsigned integers', () => { const examples = [ {high32: 0, low32: 0}, {high32: 0, low32: 0x7f}, {high32: 0, low32: 0x80}, {high32: 0, low32: 0x3fff}, {high32: 0, low32: 0x4000}, {high32: 0, low32: 0x1fffff}, {high32: 0, low32: 0x200000}, {high32: 0, low32: 0xfffffff}, {high32: 0, low32: 0x10000000}, {high32: 0, low32: 0xffffffff}, {high32: 0x1, low32: 0x00000000}, {high32: 0x7, low32: 0xffffffff}, {high32: 0x8, low32: 0x00000000}, {high32: 0x3ff, low32: 0xffffffff}, {high32: 0x400, low32: 0x00000000}, {high32: 0x1ffff, low32: 0xffffffff}, {high32: 0x20000, low32: 0x00000000}, {high32: 0xffffff, low32: 0xffffffff}, {high32: 0x1000000, low32: 0x00000000}, {high32: 0xffffffff, low32: 0xffffffff}, {high32: 0, low32: 0x123}, {high32: 0, low32: 0x1234}, {high32: 0, low32: 0x12345}, {high32: 0, low32: 0x123456}, {high32: 0, low32: 0x1234567}, {high32: 0, low32: 0x12345678}, {high32: 0x9, low32: 0x12345678}, {high32: 0x98, low32: 0x12345678}, {high32: 0x987, low32: 0x12345678}, {high32: 0x9876, low32: 0x12345678}, {high32: 0x98765, low32: 0x12345678}, {high32: 0x987654, low32: 0x12345678}, {high32: 0x9876543, low32: 0x12345678}, {high32: 0x98765432, low32: 0x12345678} ] for (let value of examples) { const encoder = new Encoder() encoder.appendUint64(value.high32, value.low32) const decoder = new Decoder(encoder.buffer) assert.deepStrictEqual(decoder.readUint64(), value) assert.strictEqual(decoder.done, true) } }) it('should encode signed integers', () => { function encode(high32, low32) { const encoder = new Encoder() encoder.appendInt64(high32, low32) return encoder } checkEncoded(encode(0, 0), [0]) checkEncoded(encode(0, 1), [1]) checkEncoded(encode(-1, -1), [0x7f]) checkEncoded(encode(0, 0x3f), [0x3f]) checkEncoded(encode(-1, -0x40), [0x40]) checkEncoded(encode(0, 0x40), [0xc0, 0x00]) checkEncoded(encode(-1, -0x41), [0xbf, 0x7f]) checkEncoded(encode(0, 0x1fff), [0xff, 0x3f]) checkEncoded(encode(-1, -0x2000), [0x80, 0x40]) checkEncoded(encode(0, 0x2000), [0x80, 0xc0, 0x00]) checkEncoded(encode(-1, -0x2001), [0xff, 0xbf, 0x7f]) checkEncoded(encode(0, 0xfffff), [0xff, 0xff, 0x3f]) checkEncoded(encode(-1, -0x100000), [0x80, 0x80, 0x40]) checkEncoded(encode(0, 0x100000), [0x80, 0x80, 0xc0, 0x00]) checkEncoded(encode(-1, -0x100001), [0xff, 0xff, 0xbf, 0x7f]) checkEncoded(encode(0, 0x7ffffff), [0xff, 0xff, 0xff, 0x3f]) checkEncoded(encode(-1, -0x8000000), [0x80, 0x80, 0x80, 0x40]) checkEncoded(encode(0, 0x8000000), [0x80, 0x80, 0x80, 0xc0, 0x00]) checkEncoded(encode(-1, -0x8000001), [0xff, 0xff, 0xff, 0xbf, 0x7f]) checkEncoded(encode(0, 0x7fffffff), [0xff, 0xff, 0xff, 0xff, 0x07]) checkEncoded(encode(0, 0x80000000), [0x80, 0x80, 0x80, 0x80, 0x08]) checkEncoded(encode(0, 0xffffffff), [0xff, 0xff, 0xff, 0xff, 0x0f]) checkEncoded(encode(-1, -0x80000000), [0x80, 0x80, 0x80, 0x80, 0x78]) checkEncoded(encode(-1, 0x7fffffff), [0xff, 0xff, 0xff, 0xff, 0x77]) checkEncoded(encode(-1, 1), [0x81, 0x80, 0x80, 0x80, 0x70]) checkEncoded(encode(-1, 0), [0x80, 0x80, 0x80, 0x80, 0x70]) checkEncoded(encode(-2, -1), [0xff, 0xff, 0xff, 0xff, 0x6f]) checkEncoded(encode(3, 0xffffffff), [0xff, 0xff, 0xff, 0xff, 0x3f]) checkEncoded(encode(-4, 0), [0x80, 0x80, 0x80, 0x80, 0x40]) checkEncoded(encode(4, 0), [0x80, 0x80, 0x80, 0x80, 0xc0, 0x00]) checkEncoded(encode(-5, -1), [0xff, 0xff, 0xff, 0xff, 0xbf, 0x7f]) checkEncoded(encode(0x1ff, 0xffffffff), [0xff, 0xff, 0xff, 0xff, 0xff, 0x3f]) checkEncoded(encode(-0x200, 0), [0x80, 0x80, 0x80, 0x80, 0x80, 0x40]) checkEncoded(encode(0x200, 0), [0x80, 0x80, 0x80, 0x80, 0x80, 0xc0, 0x00]) checkEncoded(encode(-0x201, -1), [0xff, 0xff, 0xff, 0xff, 0xff, 0xbf, 0x7f]) checkEncoded(encode(0xffff, 0xffffffff), [0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f]) checkEncoded(encode(-0x10000, 0), [0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x40]) checkEncoded(encode(0x10000, 0), [0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0xc0, 0x00]) checkEncoded(encode(-0x10001, -1), [0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xbf, 0x7f]) checkEncoded(encode(0x7fffff, 0xffffffff), [0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f]) checkEncoded(encode(-0x800000, 0), [0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x40]) checkEncoded(encode(0x800000, 0), [0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0xc0, 0x00]) checkEncoded(encode(-0x800001, -1), [0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xbf, 0x7f]) checkEncoded(encode(0x3fffffff, 0xffffffff), [0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f]) checkEncoded(encode(-0x40000000, 0), [0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x40]) checkEncoded(encode(0x40000000, 0), [0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0xc0, 0x00]) checkEncoded(encode(-0x40000001, -1), [0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xbf, 0x7f]) checkEncoded(encode(0x7fffffff, 0xffffffff), [0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00]) checkEncoded(encode(-0x80000000, 0), [0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x7f]) }) it('should round-trip signed integers', () => { const examples = [ {high32: 0, low32: 0}, {high32: 0, low32: 1}, {high32: -1, low32: -1 >>> 0}, {high32: 0, low32: 0x3f}, {high32: -1, low32: -0x40 >>> 0}, {high32: 0, low32: 0x40}, {high32: -1, low32: -0x41 >>> 0}, {high32: 0, low32: 0x1fff}, {high32: -1, low32: -0x2000 >>> 0}, {high32: 0, low32: 0x2000}, {high32: -1, low32: -0x2001 >>> 0}, {high32: 0, low32: 0xfffff}, {high32: -1, low32: -0x100000 >>> 0}, {high32: 0, low32: 0x100000}, {high32: -1, low32: -0x100001 >>> 0}, {high32: 0, low32: 0x7ffffff}, {high32: -1, low32: -0x8000000 >>> 0}, {high32: 0, low32: 0x8000000}, {high32: -1, low32: -0x8000001 >>> 0}, {high32: 0, low32: 0x7fffffff}, {high32: 0, low32: 0x80000000}, {high32: 0, low32: 0xffffffff}, {high32: -1, low32: -0x80000000 >>> 0}, {high32: -1, low32: 0x7fffffff}, {high32: -1, low32: 1}, {high32: -1, low32: 0}, {high32: -2, low32: -1 >>> 0}, {high32: 3, low32: 0xffffffff}, {high32: -4, low32: 0}, {high32: 4, low32: 0}, {high32: -5, low32: -1 >>> 0}, {high32: 0x1ff, low32: 0xffffffff}, {high32: -0x200, low32: 0}, {high32: 0x200, low32: 0}, {high32: -0x201, low32: -1 >>> 0}, {high32: 0xffff, low32: 0xffffffff}, {high32: -0x10000, low32: 0}, {high32: 0x10000, low32: 0}, {high32: -0x10001, low32: -1 >>> 0}, {high32: 0x7fffff, low32: 0xffffffff}, {high32: -0x800000, low32: 0}, {high32: 0x800000, low32: 0}, {high32: -0x800001, low32: -1 >>> 0}, {high32: 0x3fffffff, low32: 0xffffffff}, {high32: -0x40000000, low32: 0}, {high32: 0x40000000, low32: 0}, {high32: -0x40000001, low32: -1 >>> 0}, {high32: 0x7fffffff, low32: 0xffffffff}, {high32: -0x80000000, low32: 0}, {high32: 0, low32: 0x123}, {high32: -1, low32: -0x123 >>> 0}, {high32: 0, low32: 0x1234}, {high32: -1, low32: -0x1234 >>> 0}, {high32: 0, low32: 0x12345}, {high32: -1, low32: -0x12345 >>> 0}, {high32: 0, low32: 0x123456}, {high32: -1, low32: -0x123456 >>> 0}, {high32: 0, low32: 0x1234567}, {high32: -1, low32: -0x1234567 >>> 0}, {high32: 0, low32: 0x12345678}, {high32: -1, low32: -0x12345678 >>> 0}, {high32: 0x9, low32: 0x12345678}, {high32: -0x9, low32: -0x12345678 >>> 0}, {high32: 0x98, low32: 0x12345678}, {high32: -0x98, low32: -0x12345678 >>> 0}, {high32: 0x987, low32: 0x12345678}, {high32: -0x987, low32: -0x12345678 >>> 0}, {high32: 0x9876, low32: 0x12345678}, {high32: -0x9876, low32: -0x12345678 >>> 0}, {high32: 0x98765, low32: 0x12345678}, {high32: -0x98765, low32: -0x12345678 >>> 0}, {high32: 0x987654, low32: 0x12345678}, {high32: -0x987654, low32: -0x12345678 >>> 0}, {high32: 0x9876543, low32: 0x12345678}, {high32: -0x9876543, low32: -0x12345678 >>> 0}, {high32: 0x78765432, low32: 0x12345678}, {high32: -0x78765432, low32: -0x12345678 >>> 0} ] for (let value of examples) { const encoder = new Encoder() encoder.appendInt64(value.high32, value.low32) const decoder = new Decoder(encoder.buffer) assert.deepStrictEqual(decoder.readInt64(), value) assert.strictEqual(decoder.done, true) } }) it('should not encode values that are out of range', () => { assert.throws(() => { new Encoder().appendUint64(0, 0x100000000) }, /out of range/) assert.throws(() => { new Encoder().appendUint64(0x100000000, 0) }, /out of range/) assert.throws(() => { new Encoder().appendUint64(0, -1) }, /out of range/) assert.throws(() => { new Encoder().appendUint64(-1, 0) }, /out of range/) assert.throws(() => { new Encoder().appendUint64(123, Number.NaN) }, /not an integer/) assert.throws(() => { new Encoder().appendUint64(123, Math.PI) }, /not an integer/) assert.throws(() => { new Encoder().appendInt64(0, 0x100000000) }, /out of range/) assert.throws(() => { new Encoder().appendInt64(0x80000000, 0) }, /out of range/) assert.throws(() => { new Encoder().appendInt64(0, -0x80000001) }, /out of range/) assert.throws(() => { new Encoder().appendInt64(-0x80000001, 0) }, /out of range/) assert.throws(() => { new Encoder().appendInt64(123, Number.NaN) }, /not an integer/) assert.throws(() => { new Encoder().appendInt64(123, Math.PI) }, /not an integer/) }) it('should not decode values that are out of range', () => { assert.throws(() => { new Decoder(new Uint8Array([0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x00])).readUint64() }, /out of range/) assert.throws(() => { new Decoder(new Uint8Array([0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x00])).readInt64() }, /out of range/) assert.throws(() => { new Decoder(new Uint8Array([0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x02])).readUint64() }, /out of range/) assert.throws(() => { new Decoder(new Uint8Array([0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01])).readInt64() }, /out of range/) assert.throws(() => { new Decoder(new Uint8Array([0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x7e])).readInt64() }, /out of range/) assert.throws(() => { new Decoder(new Uint8Array([0x80, 0x80])).readUint64() }, /incomplete number/) assert.throws(() => { new Decoder(new Uint8Array([0x80, 0x80])).readInt64() }, /incomplete number/) }) }) describe('UTF-8 encoding', () => { it('should encode strings', () => { checkEncoded(new Encoder().appendPrefixedString(''), [0]) checkEncoded(new Encoder().appendPrefixedString('a'), [1, 0x61]) checkEncoded(new Encoder().appendPrefixedString('Oh là là'), [10, 79, 104, 32, 108, 195, 160, 32, 108, 195, 160]) checkEncoded(new Encoder().appendPrefixedString('😄'), [4, 0xf0, 0x9f, 0x98, 0x84]) }) it('should round-trip strings', () => { assert.strictEqual(new Decoder(new Encoder().appendPrefixedString('').buffer).readPrefixedString(), '') assert.strictEqual(new Decoder(new Encoder().appendPrefixedString('a').buffer).readPrefixedString(), 'a') assert.strictEqual(new Decoder(new Encoder().appendPrefixedString('Oh là là').buffer).readPrefixedString(), 'Oh là là') assert.strictEqual(new Decoder(new Encoder().appendPrefixedString('😄').buffer).readPrefixedString(), '😄') }) it('should encode multiple strings', () => { const encoder = new Encoder() encoder.appendPrefixedString('one') encoder.appendPrefixedString('two') encoder.appendPrefixedString('three') const decoder = new Decoder(encoder.buffer) assert.strictEqual(decoder.readPrefixedString(), 'one') assert.strictEqual(decoder.readPrefixedString(), 'two') assert.strictEqual(decoder.readPrefixedString(), 'three') }) }) describe('hex encoding', () => { it('should encode hex strings', () => { checkEncoded(new Encoder().appendHexString(''), [0]) checkEncoded(new Encoder().appendHexString('00'), [1, 0]) checkEncoded(new Encoder().appendHexString('0123'), [2, 1, 0x23]) checkEncoded(new Encoder().appendHexString('fedcba9876543210'), [8, 0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10]) }) it('should round-trip strings', () => { assert.strictEqual(new Decoder(new Encoder().appendHexString('').buffer).readHexString(), '') assert.strictEqual(new Decoder(new Encoder().appendHexString('00').buffer).readHexString(), '00') assert.strictEqual(new Decoder(new Encoder().appendHexString('0123').buffer).readHexString(), '0123') assert.strictEqual(new Decoder(new Encoder().appendHexString('fedcba9876543210').buffer).readHexString(), 'fedcba9876543210') }) it('should not allow malformed hex strings', () => { assert.throws(() => { new Encoder().appendHexString(0x1234) }, /value is not a string/) assert.throws(() => { new Encoder().appendHexString('abcd-ef') }, /value is not hexadecimal/) assert.throws(() => { new Encoder().appendHexString('0') }, /value is not hexadecimal/) assert.throws(() => { new Encoder().appendHexString('ABCD') }, /value is not hexadecimal/) assert.throws(() => { new Encoder().appendHexString('zz') }, /value is not hexadecimal/) }) }) }) describe('RLEEncoder and RLEDecoder', () => { function encodeRLE(type, values) { const encoder = new RLEEncoder(type) for (let value of values) encoder.appendValue(value) return encoder.buffer } function decodeRLE(type, buffer) { if (Array.isArray(buffer)) buffer = new Uint8Array(buffer) const decoder = new RLEDecoder(type, buffer), values = [] while (!decoder.done) values.push(decoder.readValue()) return values } it('should encode sequences without nulls', () => { checkEncoded(encodeRLE('uint', []), []) checkEncoded(encodeRLE('uint', [1, 2, 3]), [0x7d, 1, 2, 3]) checkEncoded(encodeRLE('uint', [0, 1, 2, 2, 3]), [0x7e, 0, 1, 2, 2, 0x7f, 3]) checkEncoded(encodeRLE('uint', [1, 1, 1, 1, 1, 1]), [6, 1]) checkEncoded(encodeRLE('uint', [1, 1, 1, 4, 4, 4]), [3, 1, 3, 4]) checkEncoded(encodeRLE('uint', [0xff]), [0x7f, 0xff, 0x01]) checkEncoded(encodeRLE('int', [-0x40]), [0x7f, 0x40]) }) it('should encode sequences containing nulls', () => { checkEncoded(encodeRLE('uint', [null, 1]), [0, 1, 0x7f, 1]) checkEncoded(encodeRLE('uint', [1, null]), [0x7f, 1, 0, 1]) checkEncoded(encodeRLE('uint', [1, 1, 1, null]), [3, 1, 0, 1]) checkEncoded(encodeRLE('uint', [null, null, null, 3, 4, 5, null]), [0, 3, 0x7d, 3, 4, 5, 0, 1]) checkEncoded(encodeRLE('uint', [null, null, null, 9, 9, 9]), [0, 3, 3, 9]) checkEncoded(encodeRLE('uint', [1, 1, 1, 1, 1, null, null, null, 1]), [5, 1, 0, 3, 0x7f, 1]) }) it('should round-trip sequences without nulls', () => { assert.deepStrictEqual(decodeRLE('uint', encodeRLE('uint', [])), []) assert.deepStrictEqual(decodeRLE('uint', encodeRLE('uint', [1, 2, 3])), [1, 2, 3]) assert.deepStrictEqual(decodeRLE('uint', encodeRLE('uint', [0, 1, 2, 2, 3])), [0, 1, 2, 2, 3]) assert.deepStrictEqual(decodeRLE('uint', encodeRLE('uint', [1, 1, 1, 1, 1, 1])), [1, 1, 1, 1, 1, 1]) assert.deepStrictEqual(decodeRLE('uint', encodeRLE('uint', [1, 1, 1, 4, 4, 4])), [1, 1, 1, 4, 4, 4]) assert.deepStrictEqual(decodeRLE('uint', encodeRLE('uint', [0xff])), [0xff]) assert.deepStrictEqual(decodeRLE('int', encodeRLE('int', [-0x40])), [-0x40]) }) it('should round-trip sequences containing nulls', () => { assert.deepStrictEqual(decodeRLE('uint', encodeRLE('uint', [null, 1])), [null, 1]) assert.deepStrictEqual(decodeRLE('uint', encodeRLE('uint', [1, null])), [1, null]) assert.deepStrictEqual(decodeRLE('uint', encodeRLE('uint', [1, 1, 1, null])), [1, 1, 1, null]) assert.deepStrictEqual(decodeRLE('uint', encodeRLE('uint', [null, null, null, 3, 4, 5, null])), [null, null, null, 3, 4, 5, null]) assert.deepStrictEqual(decodeRLE('uint', encodeRLE('uint', [null, null, null, 9, 9, 9])), [null, null, null, 9, 9, 9]) assert.deepStrictEqual(decodeRLE('uint', encodeRLE('uint', [1, 1, 1, 1, 1, null, null, null, 1])), [1, 1, 1, 1, 1, null, null, null, 1]) }) it('should support encoding string values', () => { checkEncoded(encodeRLE('utf8', ['a']), [0x7f, 1, 0x61]) checkEncoded(encodeRLE('utf8', ['a', 'b', 'c', 'd']), [0x7c, 1, 0x61, 1, 0x62, 1, 0x63, 1, 0x64]) checkEncoded(encodeRLE('utf8', ['a', 'a', 'a', 'a']), [4, 1, 0x61]) checkEncoded(encodeRLE('utf8', ['a', 'a', null, null, 'a', 'a']), [2, 1, 0x61, 0, 2, 2, 1, 0x61]) checkEncoded(encodeRLE('utf8', [null, null, null, null, 'abc']), [0, 4, 0x7f, 3, 0x61, 0x62, 0x63]) }) it('should round-trip sequences of string values', () => { assert.deepStrictEqual(decodeRLE('utf8', encodeRLE('utf8', ['a'])), ['a']) assert.deepStrictEqual(decodeRLE('utf8', encodeRLE('utf8', ['a', 'b', 'c', 'd'])), ['a', 'b', 'c', 'd']) assert.deepStrictEqual(decodeRLE('utf8', encodeRLE('utf8', ['a', 'a', 'a', 'a'])), ['a', 'a', 'a', 'a']) assert.deepStrictEqual(decodeRLE('utf8', encodeRLE('utf8', ['a', 'a', null, null, 'a', 'a'])), ['a', 'a', null, null, 'a', 'a']) assert.deepStrictEqual(decodeRLE('utf8', encodeRLE('utf8', [null, null, null, null, 'abc'])), [null, null, null, null, 'abc']) }) it('should allow repetition counts to be specified', () => { let e e = new RLEEncoder('uint'); e.appendValue(3, 0); checkEncoded(e, []) e = new RLEEncoder('uint'); e.appendValue(3, 10); checkEncoded(e, [10, 3]) e = new RLEEncoder('uint'); e.appendValue(3, 10); e.appendValue(3, 10); checkEncoded(e, [20, 3]) e = new RLEEncoder('uint'); e.appendValue(3, 10); e.appendValue(4, 10); checkEncoded(e, [10, 3, 10, 4]) e = new RLEEncoder('uint'); e.appendValue(3, 10); e.appendValue(null, 10); checkEncoded(e, [10, 3, 0, 10]) e = new RLEEncoder('uint'); e.appendValue(1); e.appendValue(1, 2); checkEncoded(e, [3, 1]) e = new RLEEncoder('uint'); e.appendValue(1); e.appendValue(2, 3); checkEncoded(e, [0x7f, 1, 3, 2]) e = new RLEEncoder('uint'); e.appendValue(1); e.appendValue(2); e.appendValue(3, 3); checkEncoded(e, [0x7e, 1, 2, 3, 3]) e = new RLEEncoder('uint'); e.appendValue(null); e.appendValue(3, 3); checkEncoded(e, [0, 1, 3, 3]) e = new RLEEncoder('uint'); e.appendValue(null); e.appendValue(null, 3); e.appendValue(1); checkEncoded(e, [0, 4, 0x7f, 1]) }) it('should return an empty buffer if the values are only nulls', () => { assert.strictEqual(encodeRLE('uint', []).byteLength, 0) assert.strictEqual(encodeRLE('uint', [null]).byteLength, 0) assert.strictEqual(encodeRLE('uint', [null, null, null, null]).byteLength, 0) }) it('should strictly enforce canonical encoded form', () => { assert.throws(() => { decodeRLE('int', [1, 1]) }, /Repetition count of 1 is not allowed/) assert.throws(() => { decodeRLE('int', [2, 1, 2, 1]) }, /Successive repetitions with the same value/) assert.throws(() => { decodeRLE('int', [0, 1, 0, 2]) }, /Successive null runs are not allowed/) assert.throws(() => { decodeRLE('int', [0, 0]) }, /Zero-length null runs are not allowed/) assert.throws(() => { decodeRLE('int', [0x7f, 1, 0x7f, 2]) }, /Successive literals are not allowed/) assert.throws(() => { decodeRLE('int', [0x7d, 1, 2, 2]) }, /Repetition of values is not allowed/) assert.throws(() => { decodeRLE('int', [2, 0, 0x7e, 0, 1]) }, /Repetition of values is not allowed/) assert.throws(() => { decodeRLE('int', [0x7e, 1, 2, 2, 2]) }, /Successive repetitions with the same value/) }) it('should allow skipping string values', () => { const example = [null, null, null, 'a', 'a', 'a', 'b', 'c', 'd', 'e'] const encoded = encodeRLE('utf8', example) for (let skipNum = 0; skipNum < example.length; skipNum++) { const decoder = new RLEDecoder('utf8', encoded), values = [] decoder.skipValues(skipNum) while (!decoder.done) values.push(decoder.readValue()) assert.deepStrictEqual(values, example.slice(skipNum), `skipping ${skipNum} values failed`) } }) it('should allow skipping integer values', () => { const example = [null, null, null, 1, 1, 1, 2, 3, 4, 5] const encoded = encodeRLE('uint', example) for (let skipNum = 0; skipNum < example.length; skipNum++) { const decoder = new RLEDecoder('uint', encoded), values = [] decoder.skipValues(skipNum) while (!decoder.done) values.push(decoder.readValue()) assert.deepStrictEqual(values, example.slice(skipNum), `skipping ${skipNum} values failed`) } }) describe('copying from a decoder', () => { function doCopy(input1, input2, options = {}) { let encoder1 = input1 if (Array.isArray(input1)) { encoder1 = new RLEEncoder('uint') for (let value of input1) encoder1.appendValue(value) } const encoder2 = new RLEEncoder('uint') for (let value of input2) encoder2.appendValue(value) const decoder2 = new RLEDecoder('uint', encoder2.buffer) if (options.skip) decoder2.skipValues(options.skip) encoder1.copyFrom(decoder2, options) return encoder1 } it('should copy a sequence', () => { checkEncoded(doCopy([], [0, 1, 2]), [0x7d, 0, 1, 2]) checkEncoded(doCopy([0, 1, 2], []), [0x7d, 0, 1, 2]) checkEncoded(doCopy([0, 1, 2], [3, 4, 5, 6]), [0x79, 0, 1, 2, 3, 4, 5, 6]) checkEncoded(doCopy([0, 1], [2, 3, 4, 4, 4]), [0x7c, 0, 1, 2, 3, 3, 4]) checkEncoded(doCopy([0, 1, 2], [3, 4, 4, 4]), [0x7c, 0, 1, 2, 3, 3, 4]) checkEncoded(doCopy([0, 1, 2], [3, 3, 3, 4, 4, 4]), [0x7d, 0, 1, 2, 3, 3, 3, 4]) checkEncoded(doCopy([0, 1, 2], [null, null, 4, 4, 4]), [0x7d, 0, 1, 2, 0, 2, 3, 4]) checkEncoded(doCopy([0, 1, 2], [3, 4, 4, null, null]), [0x7c, 0, 1, 2, 3, 2, 4, 0, 2]) checkEncoded(doCopy([0, 1, 2], [3, 4, 4, 5, 6, 6]), [0x7c, 0, 1, 2, 3, 2, 4, 0x7f, 5, 2, 6]) checkEncoded(doCopy([0, 1, 2], [2, 2, 3, 3, 4, 5, 6]), [0x7e, 0, 1, 3, 2, 2, 3, 0x7d, 4, 5, 6]) checkEncoded(doCopy([0, 0, 0], [0, 0, 0]), [6, 0]) checkEncoded(doCopy([0, 0, 0], [0, 1, 1]), [4, 0, 2, 1]) checkEncoded(doCopy([0, 0, 0], [1, 2, 2]), [3, 0, 0x7f, 1, 2, 2]) checkEncoded(doCopy([0, 0, 0], [1, 2, 3]), [3, 0, 0x7d, 1, 2, 3]) checkEncoded(doCopy([0, 0, 0], [null, null, 2, 2]), [3, 0, 0, 2, 2, 2]) checkEncoded(doCopy([0, 0, 0], [null, 0, 0, 0]), [3, 0, 0, 1, 3, 0]) checkEncoded(doCopy([0, 0, null], [null, 0, 0]), [2, 0, 0, 2, 2, 0]) checkEncoded(doCopy([0, 0, null], [0, 0, 0]), [2, 0, 0, 1, 3, 0]) checkEncoded(doCopy([0, 0, null], [1, 2, 3]), [2, 0, 0, 1, 0x7d, 1, 2, 3]) }) it('should copy multiple sequences', () => { checkEncoded(doCopy(doCopy([0, 0, 1], [1, 2]), [2, 3]), [2, 0, 2, 1, 2, 2, 0x7f, 3]) checkEncoded(doCopy(doCopy([0], [0, 0, 1, 1, 2]), [2, 3, 3, 4]), [3, 0, 2, 1, 2, 2, 2, 3, 0x7f, 4]) checkEncoded(doCopy(doCopy([0, 1, 2], [3, 4]), [5, 6]), [0x79, 0, 1, 2, 3, 4, 5, 6]) checkEncoded(doCopy(doCopy([0, 0, 0], [0, 0, 1, 1]), [1, 1]), [5, 0, 4, 1]) checkEncoded(doCopy(doCopy([0, null], [null, 1, null]), [null, 2]), [0x7f, 0, 0, 2, 0x7f, 1, 0, 2, 0x7f, 2]) }) it('should copy a sub-sequence', () => { checkEncoded(doCopy([0, 1, 2], [3, 4, 5, 6], {skip: 0, count: 0}), [0x7d, 0, 1, 2]) checkEncoded(doCopy([0, 1, 2], [3, 4, 5, 6], {skip: 0, count: 1}), [0x7c, 0, 1, 2, 3]) checkEncoded(doCopy([0, 1, 2], [3, 4, 5, 6], {skip: 0, count: 2}), [0x7b, 0, 1, 2, 3, 4]) checkEncoded(doCopy([0, 1, 2], [3, 4, 5, 6], {skip: 0, count: 4}), [0x79, 0, 1, 2, 3, 4, 5, 6]) checkEncoded(doCopy([0, 1, 2], [3, 4, 5, 6], {skip: 1, count: 1}), [0x7c, 0, 1, 2, 4]) checkEncoded(doCopy([0, 1, 2], [3, 4, 5, 6], {skip: 1, count: 2}), [0x7b, 0, 1, 2, 4, 5]) checkEncoded(doCopy([0, 1, 2], [3, 3, 3, 3], {skip: 0, count: 2}), [0x7d, 0, 1, 2, 2, 3]) checkEncoded(doCopy([0, 0, 0], [0, 0, 0, 0], {skip: 0, count: 2}), [5, 0]) checkEncoded(doCopy([0, 0], [0, 0, 1, 1, 1], {skip: 0, count: 4}), [4, 0, 2, 1]) checkEncoded(doCopy([0, 0], [0, 0, 1, 1, 2, 2], {skip: 1, count: 4}), [3, 0, 2, 1, 0x7f, 2]) checkEncoded(doCopy([0, 0], [1, 1, 2, 3, 4, 5], {skip: 0, count: 3}), [2, 0, 2, 1, 0x7f, 2]) checkEncoded(doCopy([null], [null, 1, 1, null], {skip: 0, count: 2}), [0, 2, 0x7f, 1]) checkEncoded(doCopy([null], [null, 1, 1, null], {skip: 1, count: 3}), [0, 1, 2, 1, 0, 1]) checkEncoded(doCopy([], [null, null, null, 0, 0], {skip: 0, count: 5}), [0, 3, 2, 0]) }) it('should allow insertion into a sequence', () => { const decoder1 = new RLEDecoder('uint', encodeRLE('uint', [0, 1, 2, 3, 4, 5, 6])) const decoder2 = new RLEDecoder('uint', encodeRLE('uint', [3, 3, 3])) const encoder = new RLEEncoder('uint') encoder.copyFrom(decoder1, {count: 4}) encoder.copyFrom(decoder2) encoder.copyFrom(decoder1) checkEncoded(encoder, [0x7d, 0, 1, 2, 4, 3, 0x7d, 4, 5, 6]) }) it('should allow insertion into repetition run', () => { const decoder1 = new RLEDecoder('uint', encodeRLE('uint', [1, 2, 3, 3, 4])) const decoder2 = new RLEDecoder('uint', encodeRLE('uint', [5])) const encoder = new RLEEncoder('uint') encoder.copyFrom(decoder1, {count: 3}) encoder.copyFrom(decoder2) encoder.copyFrom(decoder1) checkEncoded(encoder, [0x7a, 1, 2, 3, 5, 3, 4]) }) it('should allow copying from a decoder starting with nulls', () => { const decoder = new RLEDecoder('uint', new Uint8Array([0, 2, 0x7f, 0])) // null, null, 0 new RLEEncoder('uint').copyFrom(decoder, {count: 1}) assert.strictEqual(decoder.readValue(), null) assert.strictEqual(decoder.readValue(), 0) decoder.reset() new RLEEncoder('uint').copyFrom(decoder, {count: 2}) assert.strictEqual(decoder.readValue(), 0) }) it('should compute the sum of values copied', () => { const encoder1 = new RLEEncoder('uint'), encoder2 = new RLEEncoder('uint') for (let v of [1, 2, 3, 10, 10, 10]) encoder2.appendValue(v) assert.deepStrictEqual( encoder1.copyFrom(new RLEDecoder('uint', encoder2.buffer), {sumValues: true}), {nonNullValues: 6, sum: 36}) assert.deepStrictEqual( encoder1.copyFrom(new RLEDecoder('uint', encoder2.buffer), {sumValues: true, sumShift: 2}), {nonNullValues: 6, sum: 6}) }) it('should throw an exception if the decoder has too few values', () => { assert.throws(() => { doCopy([0, 1, 2], [], {count: 1}) }, /cannot copy 1 values/) assert.throws(() => { doCopy([0, 1, 2], [3], {count: 2}) }, /cannot copy 2 values/) assert.throws(() => { doCopy([0, 1, 2], [3, 4, 5, 6], {count: 5}) }, /cannot copy 5 values/) assert.throws(() => { doCopy([0, 1, 2], [3], {count: 2}) }, /cannot copy 2 values/) assert.throws(() => { doCopy([0, 1, 2], [3, 3, 3], {count: 4}) }, /cannot copy 4 values/) assert.throws(() => { doCopy([0, 1, 2], [3, 3, 4, 4, 5, 5], {count: 7}) }, /cannot copy 7 values/) assert.throws(() => { new RLEEncoder('uint').copyFrom(new RLEDecoder('uint', new Uint8Array([0x7e, 1]))) }, /incomplete literal/) assert.throws(() => { new RLEEncoder('uint').copyFrom(new RLEDecoder('uint', new Uint8Array([2, 1, 0x7f, 1]))) }, /Repetition of values/) }) it('should check the type of the decoder', () => { const encoder1 = new RLEEncoder('uint') assert.throws(() => { encoder1.copyFrom(new Decoder(new Uint8Array(0))) }, /incompatible type of decoder/) assert.throws(() => { encoder1.copyFrom(new RLEDecoder('int', new Uint8Array(0))) }, /incompatible type of decoder/) }) }) }) describe('DeltaEncoder and DeltaDecoder', () => { function encodeDelta(values) { const encoder = new DeltaEncoder() for (let value of values) encoder.appendValue(value) return encoder.buffer } function decodeDelta(buffer) { const decoder = new DeltaDecoder(buffer), values = [] while (!decoder.done) values.push(decoder.readValue()) return values } it('should encode sequences', () => { checkEncoded(encodeDelta([]), []) checkEncoded(encodeDelta([18, 2, 9, 15, 16, 19, 25]), [0x79, 18, 0x70, 7, 6, 1, 3, 6]) checkEncoded(encodeDelta([1, 2, 3, 4, 5, 6, 7, 8]), [8, 1]) checkEncoded(encodeDelta([10, 11, 12, 13, 14, 15]), [0x7f, 10, 5, 1]) checkEncoded(encodeDelta([10, 11, 12, 13, 0, 1, 2, 3]), [0x7f, 10, 3, 1, 0x7f, 0x73, 3, 1]) checkEncoded(encodeDelta([0, 1, 2, 3, null, null, null, 4, 5, 6]), [0x7f, 0, 3, 1, 0, 3, 3, 1]) checkEncoded(encodeDelta([-64, -60, -56, -52, -48, -44, -40, -36]), [0x7f, 0x40, 7, 4]) }) it('should encode-decode round-trip sequences', () => { assert.deepStrictEqual(decodeDelta(encodeDelta([])), []) assert.deepStrictEqual(decodeDelta(encodeDelta([18, 2, 9, 15, 16, 19, 25])), [18, 2, 9, 15, 16, 19, 25]) assert.deepStrictEqual(decodeDelta(encodeDelta([1, 2, 3, 4, 5, 6, 7, 8])), [1, 2, 3, 4, 5, 6, 7, 8]) assert.deepStrictEqual(decodeDelta(encodeDelta([10, 11, 12, 13, 14, 15])), [10, 11, 12, 13, 14, 15]) assert.deepStrictEqual(decodeDelta(encodeDelta([10, 11, 12, 13, 0, 1, 2, 3])), [10, 11, 12, 13, 0, 1, 2, 3]) assert.deepStrictEqual(decodeDelta(encodeDelta([0, 1, 2, 3, null, null, null, 4, 5, 6])), [0, 1, 2, 3, null, null, null, 4, 5, 6]) assert.deepStrictEqual(decodeDelta(encodeDelta([-64, -60, -56, -52, -48, -44, -40, -36])), [-64, -60, -56, -52, -48, -44, -40, -36]) }) it('should allow repetition counts to be specified', () => { let e e = new DeltaEncoder(); e.appendValue(3, 0); checkEncoded(e, []) e = new DeltaEncoder(); e.appendValue(3, 10); checkEncoded(e, [0x7f, 3, 9, 0]) e = new DeltaEncoder(); e.appendValue(1, 3); e.appendValue(1, 3); checkEncoded(e, [0x7f, 1, 5, 0]) }) it('should allow skipping values', () => { const example = [null, null, null, 10, 11, 12, 13, 14, 15, 16, 1, 2, 3, 40, 11, 13, 21, 103] const encoded = encodeDelta(example) for (let skipNum = 0; skipNum < example.length; skipNum++) { const decoder = new DeltaDecoder(encoded), values = [] decoder.skipValues(skipNum) while (!decoder.done) values.push(decoder.readValue()) assert.deepStrictEqual(values, example.slice(skipNum), `skipping ${skipNum} values failed`) } }) describe('copying from a decoder', () => { function doCopy(input1, input2, options = {}) { let encoder1 = input1 if (Array.isArray(input1)) { encoder1 = new DeltaEncoder() for (let value of input1) encoder1.appendValue(value) } const encoder2 = new DeltaEncoder() for (let value of input2) encoder2.appendValue(value) const decoder2 = new DeltaDecoder(encoder2.buffer) if (options.skip) decoder2.skipValues(options.skip) encoder1.copyFrom(decoder2, options) return encoder1 } it('should copy a sequence', () => { checkEncoded(doCopy([], [0, 0, 0]), [3, 0]) checkEncoded(doCopy([0, 0, 0], []), [3, 0]) checkEncoded(doCopy([0, 0, 0], [0, 0, 0]), [6, 0]) checkEncoded(doCopy([1, 2, 3], [4, 5, 6]), [6, 1]) checkEncoded(doCopy([1, 2, 3], [4, 10, 20]), [4, 1, 0x7e, 6, 10]) checkEncoded(doCopy([1, 2, 3], [1, 2, 3, 4]), [3, 1, 0x7f, 0x7e, 3, 1]) checkEncoded(doCopy([0, 1, 3], [6, 10, 15]), [0x7a, 0, 1, 2, 3, 4, 5]) checkEncoded(doCopy([0, 1, 3], [5, 9, 14]), [0x7e, 0, 1, 2, 2, 0x7e, 4, 5]) checkEncoded(doCopy([1, 2, 4], [5, 6, 8, 9, 10, 12]), [2, 1, 0x7f, 2, 2, 1, 0x7f, 2, 2, 1, 0x7f, 2]) checkEncoded(doCopy([4, 4, 4], [4, 4, 4, 5, 6, 7]), [0x7f, 4, 5, 0, 3, 1]) checkEncoded(doCopy([0, 1, 4], [9, 6, 2, 5, 3]), [0x78, 0, 1, 3, 5, 0x7d, 0x7c, 3, 0x7e]) checkEncoded(doCopy([1, 2, 3], [null, 4, 5, 6]), [3, 1, 0, 1, 3, 1]) checkEncoded(doCopy([1, 2, 3], [null, 6, 6, 6]), [3, 1, 0, 1, 0x7f, 3, 2, 0]) checkEncoded(doCopy([1, 2, 3], [null, null, 4, 5, 7, 9]), [3, 1, 0, 2, 2, 1, 2, 2]) checkEncoded(doCopy([1, 2, null], [3, 4, 5]), [2, 1, 0, 1, 3, 1]) checkEncoded(doCopy([1, 2, null], [6, 6, 6]), [2, 1, 0, 1, 0x7f, 4, 2, 0]) checkEncoded(doCopy([1, 2, null], [null, 3, 4]), [2, 1, 0, 2, 2, 1]) checkEncoded(doCopy([1, 2, null], [null, 6, 6]), [2, 1, 0, 2, 0x7e, 4, 0]) }) it('should copy a sub-sequence', () => { checkEncoded(doCopy([1, 2, 3], [4, 5, 6, 7], {count: 2}), [5, 1]) checkEncoded(doCopy([1, 2, 3], [null, null, 4], {count: 1}), [3, 1, 0, 1]) checkEncoded(doCopy([1, 2, 3], [null, null, 4], {count: 2}), [3, 1, 0, 2]) }) it('should copy non-ascending sequences', () => { const decoder = new DeltaDecoder(new Uint8Array([2, 1, 0x7e, 2, 0x7f])) // 1, 2, 4, 3 const encoder = new DeltaEncoder() encoder.copyFrom(decoder, {count: 4}) encoder.appendValue(5) checkEncoded(encoder, [2, 1, 0x7d, 2, 0x7f, 2]) // 1, 2, 4, 3, 5 }) it('should be able to pause and resume copying', () => { const numValues = 13 // 1, 3, 4, 2, null, 3, 4, 5, null, null, 4, 2, -1 const bytes = [0x7c, 1, 2, 1, 0x7e, 0, 1, 3, 1, 0, 2, 0x7d, 0x7f, 0x7e, 0x7d] const decoder = new DeltaDecoder(new Uint8Array(bytes)) for (let i = 0; i <= numValues; i++) { const encoder = new DeltaEncoder() encoder.copyFrom(decoder, {count: i}) encoder.copyFrom(decoder, {count: numValues - i}) checkEncoded(encoder, bytes) decoder.reset() } }) it('should handle copying followed by appending', () => { const encoder1 = doCopy([], [1, 2, 3]) encoder1.appendValue(4) checkEncoded(encoder1, [4, 1]) const encoder2 = doCopy([5], [6, null, null, null, 7, 8]) encoder2.appendValue(9) checkEncoded(encoder2, [0x7e, 5, 1, 0, 3, 3, 1]) const encoder3 = doCopy([1], [2]) encoder3.appendValue(3) checkEncoded(encoder3, [3, 1]) }) it('should throw an exception if the decoder has too few values', () => { assert.throws(() => { doCopy([0, 1, 2], [], {count: 1}) }, /cannot copy 1 values/) assert.throws(() => { doCopy([0, 1, 2], [null, 3], {count: 3}) }, /cannot copy 1 values/) assert.throws(() => { new DeltaEncoder().copyFrom(new DeltaDecoder(new Uint8Array([0, 2])), {count: 3}) }, /cannot copy 3 values/) }) it('should check the arguments are valid', () => { const encoder1 = new DeltaEncoder('uint') assert.throws(() => { encoder1.copyFrom(new Decoder(new Uint8Array(0))) }, /incompatible type of decoder/) assert.throws(() => { encoder1.copyFrom(new DeltaDecoder(new Uint8Array(0)), {sumValues: true}) }, /unsupported options/) }) }) }) describe('BooleanEncoder and BooleanDecoder', () => { function encodeBools(values) { const encoder = new BooleanEncoder() for (let value of values) encoder.appendValue(value) return encoder.buffer } function decodeBools(buffer) { if (Array.isArray(buffer)) buffer = new Uint8Array(buffer) const decoder = new BooleanDecoder(buffer), values = [] while (!decoder.done) values.push(decoder.readValue()) return values } it('should encode sequences of booleans', () => { checkEncoded(encodeBools([]), []) checkEncoded(encodeBools([false]), [1]) checkEncoded(encodeBools([true]), [0, 1]) checkEncoded(encodeBools([false, false, false, true, true]), [3, 2]) checkEncoded(encodeBools([true, true, true, false, false]), [0, 3, 2]) checkEncoded(encodeBools([true, false, true, false, true, true, false]), [0, 1, 1, 1, 1, 2, 1]) }) it('should encode-decode round-trip booleans', () => { assert.deepStrictEqual(decodeBools(encodeBools([])), []) assert.deepStrictEqual(decodeBools(encodeBools([false])), [false]) assert.deepStrictEqual(decodeBools(encodeBools([true])), [true]) assert.deepStrictEqual(decodeBools(encodeBools([false, false, false, true, true])), [false, false, false, true, true]) assert.deepStrictEqual(decodeBools(encodeBools([true, true, true, false, false])), [true, true, true, false, false]) assert.deepStrictEqual(decodeBools(encodeBools([true, false, true, false, true, true, false])), [true, false, true, false, true, true, false]) }) it('should not allow non-boolean values', () => { assert.throws(() => { encodeBools([42]) }, /Unsupported value/) assert.throws(() => { encodeBools([null]) }, /Unsupported value/) assert.throws(() => { encodeBools(['false']) }, /Unsupported value/) assert.throws(() => { encodeBools([undefined]) }, /Unsupported value/) }) it('should allow repetition counts to be specified', () => { let e e = new BooleanEncoder(); e.appendValue(false, 0); checkEncoded(e, []) e = new BooleanEncoder(); e.appendValue(false, 2); e.appendValue(false, 2); checkEncoded(e, [4]) e = new BooleanEncoder(); e.appendValue(true, 2); e.appendValue(false, 2); checkEncoded(e, [0, 2, 2]) }) it('should allow skipping values', () => { const example = [false, false, false, true, true, true, false, true, false, true] const encoded = encodeBools(example) for (let skipNum = 0; skipNum < example.length; skipNum++) { const decoder = new BooleanDecoder(encoded), values = [] decoder.skipValues(skipNum) while (!decoder.done) values.push(decoder.readValue()) assert.deepStrictEqual(values, example.slice(skipNum), `skipping ${skipNum} values failed`) } }) it('should strictly enforce canonical encoded form', () => { assert.throws(() => { decodeBools([1, 0]) }, /Zero-length runs are not allowed/) assert.throws(() => { decodeBools([1, 1, 0]) }, /Zero-length runs are not allowed/) const decoder = new BooleanDecoder(new Uint8Array([2, 0, 1])) decoder.skipValues(1) assert.throws(() => { decoder.skipValues(2) }, /Zero-length runs are not allowed/) }) describe('copying from a decoder', () => { function doCopy(input1, input2, options = {}) { let encoder1 = input1 if (Array.isArray(input1)) { encoder1 = new BooleanEncoder() for (let value of input1) encoder1.appendValue(value) } const encoder2 = new BooleanEncoder() for (let value of input2) encoder2.appendValue(value) const decoder2 = new BooleanDecoder(encoder2.buffer) if (options.skip) decoder2.skipValues(options.skip) encoder1.copyFrom(decoder2, options) return encoder1 } it('should copy a sequence', () => { checkEncoded(doCopy([false, false, true], []), [2, 1]) checkEncoded(doCopy([], [false, false, true, true]), [2, 2]) checkEncoded(doCopy([false, false], [false, false, true, true]), [4, 2]) checkEncoded(doCopy([true, true], [false, false, true, true]), [0, 2, 2, 2]) checkEncoded(doCopy([true, true], [true, true]), [0, 4]) }) it('should copy a sub-sequence', () => { checkEncoded(doCopy([false], [false, false, false, true], {count: 2}), [3]) checkEncoded(doCopy([false], [true, true, true, true], {count: 3}), [1, 3]) checkEncoded(doCopy([false], [false, true, true, true], {skip: 1}), [1, 3]) checkEncoded(doCopy([false], [false, true, true, true], {skip: 2}), [1, 2]) }) it('should throw an exception if the decoder has too few values', () => { assert.throws(() => { doCopy([false], [], {count: 1}) }, /cannot copy 1 values/) assert.throws(() => { doCopy([false], [true, false], {count: 3}) }, /cannot copy 3 values/) }) it('should check the arguments are valid', () => { assert.throws(() => { new BooleanEncoder().copyFrom(new Decoder(new Uint8Array(0))) }, /incompatible type of decoder/) assert.throws(() => { new BooleanEncoder().copyFrom(new BooleanDecoder(new Uint8Array([2, 0]))) }, /Zero-length runs/) }) }) }) }) ================================================ FILE: test/frontend_test.js ================================================ const assert = require('assert') const Frontend = require('../frontend') const { decodeChange } = require('../backend/columnar') const { Backend } = require('../src/automerge') const uuid = require('../src/uuid') const { STATE } = require('../frontend/constants') const UUID_PATTERN = /^[0-9a-f]{32}$/ describe('Automerge.Frontend', () => { describe('initializing', () => { it('should be an empty object by default', () => { const doc = Frontend.init() assert.deepStrictEqual(doc, {}) assert(UUID_PATTERN.test(Frontend.getActorId(doc).toString())) }) it('should allow actorId assignment to be deferred', () => { let doc0 = Frontend.init({ deferActorId: true }) assert.strictEqual(Frontend.getActorId(doc0), undefined) assert.throws(() => { Frontend.change(doc0, doc => doc.foo = 'bar') }, /Actor ID must be initialized with setActorId/) const doc1 = Frontend.setActorId(doc0, uuid()) const [doc2] = Frontend.change(doc1, doc => doc.foo = 'bar') assert.deepStrictEqual(doc2, { foo: 'bar' }) }) it('should allow instantiating from an existing object', () => { const initialState = { birds: { wrens: 3, magpies: 4 } } const [doc] = Frontend.from(initialState) assert.deepStrictEqual(doc, initialState) }) it('should accept an empty object as initial state', () => { const [doc] = Frontend.from({}) assert.deepStrictEqual(doc, {}) }) }) describe('performing changes', () => { it('should return the unmodified document if nothing changed', () => { const doc0 = Frontend.init() const [doc1] = Frontend.change(doc0, () => {}) assert.strictEqual(doc1, doc0) }) it('should set root object properties', () => { const actor = uuid() const [doc, change] = Frontend.change(Frontend.init(actor), doc => doc.bird = 'magpie') assert.deepStrictEqual(doc, {bird: 'magpie'}) assert.deepStrictEqual(change, { actor, seq: 1, time: change.time, message: '', startOp: 1, deps: [], ops: [ {obj: '_root', action: 'set', key: 'bird', insert: false, value: 'magpie', pred: []} ] }) }) it('should create nested maps', () => { const [doc, change] = Frontend.change(Frontend.init(), doc => doc.birds = {wrens: 3}) const birds = Frontend.getObjectId(doc.birds), actor = Frontend.getActorId(doc) assert.deepStrictEqual(doc, {birds: {wrens: 3}}) assert.deepStrictEqual(change, { actor, seq: 1, time: change.time, message: '', startOp: 1, deps: [], ops: [ {obj: '_root', action: 'makeMap', key: 'birds', insert: false, pred: []}, {obj: birds, action: 'set', key: 'wrens', insert: false, datatype: 'int', value: 3, pred: []} ] }) }) it('should apply updates inside nested maps', () => { const [doc1] = Frontend.change(Frontend.init(), doc => doc.birds = {wrens: 3}) const [doc2, change2] = Frontend.change(doc1, doc => doc.birds.sparrows = 15) const birds = Frontend.getObjectId(doc2.birds), actor = Frontend.getActorId(doc1) assert.deepStrictEqual(doc1, {birds: {wrens: 3}}) assert.deepStrictEqual(doc2, {birds: {wrens: 3, sparrows: 15}}) assert.deepStrictEqual(change2, { actor, seq: 2, time: change2.time, message: '', startOp: 3, deps: [], ops: [ {obj: birds, action: 'set', key: 'sparrows', insert: false, datatype: 'int', value: 15, pred: []} ] }) }) it('should delete keys in maps', () => { const actor = uuid() const [doc1] = Frontend.change(Frontend.init(actor), doc => { doc.magpies = 2; doc.sparrows = 15 }) const [doc2, change2] = Frontend.change(doc1, doc => delete doc.magpies) assert.deepStrictEqual(doc1, {magpies: 2, sparrows: 15}) assert.deepStrictEqual(doc2, {sparrows: 15}) assert.deepStrictEqual(change2, { actor, seq: 2, time: change2.time, message: '', startOp: 3, deps: [], ops: [ {obj: '_root', action: 'del', key: 'magpies', insert: false, pred: [ `1@${actor}`]} ] }) }) it('should create lists', () => { const [doc, change] = Frontend.change(Frontend.init(), doc => doc.birds = ['chaffinch']) const actor = Frontend.getActorId(doc) assert.deepStrictEqual(doc, {birds: ['chaffinch']}) assert.deepStrictEqual(change, { actor, seq: 1, time: change.time, message: '', startOp: 1, deps: [], ops: [ {obj: '_root', action: 'makeList', key: 'birds', insert: false, pred: []}, {obj: `1@${actor}`, action: 'set', elemId: '_head', insert: true, value: 'chaffinch', pred: []} ] }) }) it('should apply updates inside lists', () => { const [doc1] = Frontend.change(Frontend.init(), doc => doc.birds = ['chaffinch']) const [doc2, change2] = Frontend.change(doc1, doc => doc.birds[0] = 'greenfinch') const birds = Frontend.getObjectId(doc2.birds), actor = Frontend.getActorId(doc2) assert.deepStrictEqual(doc1, {birds: ['chaffinch']}) assert.deepStrictEqual(doc2, {birds: ['greenfinch']}) assert.deepStrictEqual(change2, { actor, seq: 2, time: change2.time, message: '', startOp: 3, deps: [], ops: [ {obj: birds, action: 'set', elemId: `2@${actor}`, insert: false, value: 'greenfinch', pred: [ `2@${actor}` ]} ] }) }) it('should insert nulls when indexing out of upper-bound range', () => { const [doc1] = Frontend.change(Frontend.init(), doc => doc.birds = ['chaffinch']) const [doc2, change2] = Frontend.change(doc1, doc => doc.birds[3] = 'greenfinch') const birds = Frontend.getObjectId(doc2.birds), actor = Frontend.getActorId(doc2) assert.deepStrictEqual(doc1, {birds: ['chaffinch']}) assert.deepStrictEqual(doc2, {birds: ['chaffinch', null, null, 'greenfinch']}) assert.deepStrictEqual(change2, { actor, seq: 2, startOp: 3, deps: [], time: change2.time, message: '', ops: [ {action: 'set', obj: birds, elemId: `2@${actor}`, insert: true, values: [null, null, 'greenfinch'], pred: []} ] }) }) it('should delete list elements', () => { const [doc1] = Frontend.change(Frontend.init(), doc => doc.birds = ['chaffinch', 'goldfinch']) const [doc2, change2] = Frontend.change(doc1, doc => doc.birds.deleteAt(0)) const birds = Frontend.getObjectId(doc2.birds), actor = Frontend.getActorId(doc2) assert.deepStrictEqual(doc1, {birds: ['chaffinch', 'goldfinch']}) assert.deepStrictEqual(doc2, {birds: ['goldfinch']}) assert.deepStrictEqual(change2, { actor, seq: 2, time: change2.time, message: '', startOp: 4, deps: [], ops: [ {obj: birds, action: 'del', elemId: `2@${actor}`, insert: false, pred: [`2@${actor}`]} ] }) }) it('should store Date objects as timestamps', () => { const now = new Date() const [doc, change] = Frontend.change(Frontend.init(), doc => doc.now = now) const actor = Frontend.getActorId(doc) assert.strictEqual(doc.now instanceof Date, true) assert.strictEqual(doc.now.getTime(), now.getTime()) assert.deepStrictEqual(change, { actor, seq: 1, time: change.time, message: '', startOp: 1, deps: [], ops: [ {obj: '_root', action: 'set', key: 'now', insert: false, value: now.getTime(), datatype: 'timestamp', pred: []} ] }) }) describe('counters', () => { it('should handle counters inside maps', () => { const [doc1, change1] = Frontend.change(Frontend.init(), doc => { doc.wrens = new Frontend.Counter() assert.strictEqual(doc.wrens.value, 0) }) const [doc2, change2] = Frontend.change(doc1, doc => { doc.wrens.increment() assert.strictEqual(doc.wrens.value, 1) }) const actor = Frontend.getActorId(doc2) assert.deepStrictEqual(doc1, {wrens: new Frontend.Counter(0)}) assert.deepStrictEqual(doc2, {wrens: new Frontend.Counter(1)}) assert.deepStrictEqual(change1, { actor, seq: 1, time: change1.time, message: '', startOp: 1, deps: [], ops: [ {obj: '_root', action: 'set', key: 'wrens', insert: false, value: 0, datatype: 'counter', pred: []} ] }) assert.deepStrictEqual(change2, { actor, seq: 2, time: change2.time, message: '', startOp: 2, deps: [], ops: [ {obj: '_root', action: 'inc', key: 'wrens', insert: false, value: 1, pred: [`1@${actor}`]} ] }) }) it('should handle counters inside lists', () => { const [doc1, change1] = Frontend.change(Frontend.init(), doc => { doc.counts = [new Frontend.Counter(1)] assert.strictEqual(doc.counts[0].value, 1) }) const [doc2, change2] = Frontend.change(doc1, doc => { doc.counts[0].increment(2) assert.strictEqual(doc.counts[0].value, 3) }) const counts = Frontend.getObjectId(doc2.counts), actor = Frontend.getActorId(doc2) assert.deepStrictEqual(doc1, {counts: [new Frontend.Counter(1)]}) assert.deepStrictEqual(doc2, {counts: [new Frontend.Counter(3)]}) assert.deepStrictEqual(change1, { actor, deps: [], seq: 1, time: change1.time, message: '', startOp: 1, ops: [ {obj: '_root', action: 'makeList', key: 'counts', insert: false, pred: []}, {obj: counts, action: 'set', elemId: '_head', insert: true, value: 1, datatype: 'counter', pred: []} ] }) assert.deepStrictEqual(change2, { actor, deps: [], seq: 2, time: change2.time, message: '', startOp: 3, ops: [ {obj: counts, action: 'inc', elemId: `2@${actor}`, insert: false, value: 2, pred: [`2@${actor}`]} ] }) }) it('should refuse to overwrite a property with a counter value', () => { const [doc1] = Frontend.change(Frontend.init(), doc => { doc.counter = new Frontend.Counter() doc.list = [new Frontend.Counter()] }) assert.throws(() => Frontend.change(doc1, doc => doc.counter++), /Cannot overwrite a Counter object/) assert.throws(() => Frontend.change(doc1, doc => doc.list[0] = 3), /Cannot overwrite a Counter object/) }) it('should make counter objects behave like primitive numbers', () => { const [doc1] = Frontend.change(Frontend.init(), doc => doc.birds = new Frontend.Counter(3)) assert.equal(doc1.birds, 3) // they are equal according to ==, but not strictEqual according to === assert.notStrictEqual(doc1.birds, 3) assert(doc1.birds < 4) assert(doc1.birds >= 0) assert(!(doc1.birds <= 2)) assert.strictEqual(doc1.birds + 10, 13) assert.strictEqual(`I saw ${doc1.birds} birds`, 'I saw 3 birds') assert.strictEqual(['I saw', doc1.birds, 'birds'].join(' '), 'I saw 3 birds') }) it('should allow counters to be serialized to JSON', () => { const [doc1] = Frontend.change(Frontend.init(), doc => doc.birds = new Frontend.Counter()) assert.strictEqual(JSON.stringify(doc1), '{"birds":0}') }) }) }) describe('backend concurrency', () => { function getRequests(doc) { return doc[STATE].requests.map(req => ({actor: req.actor, seq: req.seq})) } it('should use version and sequence number from the backend', () => { const local = uuid(), remote1 = uuid(), remote2 = uuid() const patch1 = { clock: {[local]: 4, [remote1]: 11, [remote2]: 41}, maxOp: 4, deps: [], diffs: {objectId: '_root', type: 'map', props: {blackbirds: {[local]: {type: 'value', value: 24}}}} } let doc1 = Frontend.applyPatch(Frontend.init(local), patch1) let [doc2, change] = Frontend.change(doc1, doc => doc.partridges = 1) assert.deepStrictEqual(change, { actor: local, seq: 5, deps: [], startOp: 5, time: change.time, message: '', ops: [ {obj: '_root', action: 'set', key: 'partridges', insert: false, datatype: 'int', value: 1, pred: []} ] }) assert.deepStrictEqual(getRequests(doc2), [{actor: local, seq: 5}]) }) it('should remove pending requests once handled', () => { const actor = uuid() let [doc1, change1] = Frontend.change(Frontend.init(actor), doc => doc.blackbirds = 24) let [doc2, change2] = Frontend.change(doc1, doc => doc.partridges = 1) assert.deepStrictEqual(change1, { actor, seq: 1, deps: [], startOp: 1, time: change1.time, message: '', ops: [ {obj: '_root', action: 'set', key: 'blackbirds', insert: false, datatype: 'int', value: 24, pred: []} ] }) assert.deepStrictEqual(change2, { actor, seq: 2, deps: [], startOp: 2, time: change2.time, message: '', ops: [ {obj: '_root', action: 'set', key: 'partridges', insert: false, datatype: 'int', value: 1, pred: []} ] }) assert.deepStrictEqual(getRequests(doc2), [{actor, seq: 1}, {actor, seq: 2}]) doc2 = Frontend.applyPatch(doc2, { actor, seq: 1, clock: {[actor]: 1}, diffs: { objectId: '_root', type: 'map', props: {blackbirds: {[actor]: {type: 'value', value: 24}}} } }) assert.deepStrictEqual(getRequests(doc2), [{actor, seq: 2}]) assert.deepStrictEqual(doc2, {blackbirds: 24, partridges: 1}) doc2 = Frontend.applyPatch(doc2, { actor, seq: 2, clock: {[actor]: 2}, diffs: { objectId: '_root', type: 'map', props: {partridges: {[actor]: {type: 'value', value: 1}}} } }) assert.deepStrictEqual(doc2, {blackbirds: 24, partridges: 1}) assert.deepStrictEqual(getRequests(doc2), []) }) it('should leave the request queue unchanged on remote patches', () => { const actor = uuid(), other = uuid() let [doc, req] = Frontend.change(Frontend.init(actor), doc => doc.blackbirds = 24) assert.deepStrictEqual(req, { actor, seq: 1, deps: [], startOp: 1, time: req.time, message: '', ops: [ {obj: '_root', action: 'set', key: 'blackbirds', insert: false, datatype: 'int', value: 24, pred: []} ] }) assert.deepStrictEqual(getRequests(doc), [{actor, seq: 1}]) doc = Frontend.applyPatch(doc, { clock: {[other]: 1}, diffs: { objectId: '_root', type: 'map', props: {pheasants: {[other]: {type: 'value', value: 2}}} } }) assert.deepStrictEqual(doc, {blackbirds: 24}) assert.deepStrictEqual(getRequests(doc), [{actor, seq: 1}]) doc = Frontend.applyPatch(doc, { actor, seq: 1, clock: {[actor]: 1, [other]: 1}, diffs: { objectId: '_root', type: 'map', props: {blackbirds: {[actor]: {type: 'value', value: 24}}} } }) assert.deepStrictEqual(doc, {blackbirds: 24, pheasants: 2}) assert.deepStrictEqual(getRequests(doc), []) }) it('should not allow request patches to be applied out of order', () => { const [doc1] = Frontend.change(Frontend.init(), doc => doc.blackbirds = 24) const [doc2] = Frontend.change(doc1, doc => doc.partridges = 1) const actor = Frontend.getActorId(doc2) const diffs = {objectId: '_root', type: 'map', props: {partridges: {[actor]: {type: 'value', value: 1}}}} assert.throws(() => { Frontend.applyPatch(doc2, {actor, seq: 2, clock: {[actor]: 2}, diffs}) }, /Mismatched sequence number/) }) it('should handle concurrent insertions into lists', () => { let [doc1] = Frontend.change(Frontend.init(), doc => doc.birds = ['goldfinch']) const birds = Frontend.getObjectId(doc1.birds), actor = Frontend.getActorId(doc1) doc1 = Frontend.applyPatch(doc1, { actor, seq: 1, clock: {[actor]: 1}, maxOp: 2, diffs: {objectId: '_root', type: 'map', props: { birds: {[actor]: {objectId: birds, type: 'list', edits: [ {action: 'insert', elemId: `2@${actor}`, opId: `2@${actor}`, index: 0, value: {type: 'value', value: 'goldfinch'}} ]}} }} }) assert.deepStrictEqual(doc1, {birds: ['goldfinch']}) assert.deepStrictEqual(getRequests(doc1), []) const [doc2] = Frontend.change(doc1, doc => { doc.birds.insertAt(0, 'chaffinch') doc.birds.insertAt(2, 'greenfinch') }) assert.deepStrictEqual(doc2, {birds: ['chaffinch', 'goldfinch', 'greenfinch']}) const remoteActor = uuid() const doc3 = Frontend.applyPatch(doc2, { clock: {[actor]: 1, [remoteActor]: 1}, maxOp: 4, diffs: {objectId: '_root', type: 'map', props: { birds: {[actor]: {objectId: birds, type: 'list', edits: [ {action: 'insert', elemId: `1@${remoteActor}`, opId: `1@${remoteActor}`, index: 1, value: {type: 'value', value: 'bullfinch'}} ]}} }} }) // The addition of 'bullfinch' does not take effect yet: it is queued up until the pending // request has made its round-trip through the backend. assert.deepStrictEqual(doc3, {birds: ['chaffinch', 'goldfinch', 'greenfinch']}) const doc4 = Frontend.applyPatch(doc3, { actor, seq: 2, clock: {[actor]: 2, [remoteActor]: 1}, maxOp: 4, diffs: {objectId: '_root', type: 'map', props: { birds: {[actor]: {objectId: birds, type: 'list', edits: [ {action: 'insert', index: 0, elemId: `3@${actor}`, opId: `3@${actor}`, value: {type: 'value', value: 'chaffinch'}}, {action: 'insert', index: 2, elemId: `4@${actor}`, opId: `4@${actor}`, value: {type: 'value', value: 'greenfinch'}} ]}} }} }) assert.deepStrictEqual(doc4, {birds: ['chaffinch', 'goldfinch', 'greenfinch', 'bullfinch']}) assert.deepStrictEqual(getRequests(doc4), []) }) it('should allow interleaving of patches and changes', () => { const actor = uuid() const [doc1, change1] = Frontend.change(Frontend.init(actor), doc => doc.number = 1) const [doc2, change2] = Frontend.change(doc1, doc => doc.number = 2) assert.deepStrictEqual(change1, { actor, deps: [], startOp: 1, seq: 1, time: change1.time, message: '', ops: [ {obj: '_root', action: 'set', key: 'number', insert: false, datatype: 'int', value: 1, pred: []} ] }) assert.deepStrictEqual(change2, { actor, deps: [], startOp: 2, seq: 2, time: change2.time, message: '', ops: [ {obj: '_root', action: 'set', key: 'number', insert: false, datatype: 'int', value: 2, pred: [`1@${actor}`]} ] }) const state0 = Backend.init() const [/* state1 */, patch1, /* binChange1 */] = Backend.applyLocalChange(state0, change1) const doc2a = Frontend.applyPatch(doc2, patch1) const [/* doc3 */, change3] = Frontend.change(doc2a, doc => doc.number = 3) assert.deepStrictEqual(change3, { actor, seq: 3, startOp: 3, time: change3.time, message: '', deps: [], ops: [ {obj: '_root', action: 'set', key: 'number', insert: false, datatype: 'int', value: 3, pred: [`2@${actor}`]} ] }) }) it('deps are filled in if the frontend does not have the latest patch', () => { const actor1 = uuid(), actor2 = uuid() const [/* doc1 */, change1] = Frontend.change(Frontend.init(actor1), doc => doc.number = 1) const [/* state1 */, /* patch1 */, binChange1] = Backend.applyLocalChange(Backend.init(), change1) const [state1a, patch1a] = Backend.applyChanges(Backend.init(), [binChange1]) const doc1a = Frontend.applyPatch(Frontend.init(actor2), patch1a) const [doc2, change2] = Frontend.change(doc1a, doc => doc.number = 2) const [doc3, change3] = Frontend.change(doc2, doc => doc.number = 3) assert.deepStrictEqual(change2, { actor: actor2, seq: 1, startOp: 2, deps: [decodeChange(binChange1).hash], time: change2.time, message: '', ops: [ {obj: '_root', action: 'set', key: 'number', insert: false, datatype: 'int', value: 2, pred: [`1@${actor1}`]} ] }) assert.deepStrictEqual(change3, { actor: actor2, seq: 2, startOp: 3, deps: [], time: change3.time, message: '', ops: [ {obj: '_root', action: 'set', key: 'number', insert: false, datatype: 'int', value: 3, pred: [`2@${actor2}`]} ] }) const [state2, patch2, binChange2] = Backend.applyLocalChange(state1a, change2) const [state3, patch3, binChange3] = Backend.applyLocalChange(state2, change3) assert.deepStrictEqual(decodeChange(binChange2).deps, [decodeChange(binChange1).hash]) assert.deepStrictEqual(decodeChange(binChange3).deps, [decodeChange(binChange2).hash]) assert.deepStrictEqual(patch1a.deps, [decodeChange(binChange1).hash]) assert.deepStrictEqual(patch2.deps, []) const doc2a = Frontend.applyPatch(doc3, patch2) const doc3a = Frontend.applyPatch(doc2a, patch3) const [/* doc4 */, change4] = Frontend.change(doc3a, doc => doc.number = 4) assert.deepStrictEqual(change4, { actor: actor2, seq: 3, startOp: 4, time: change4.time, message: '', deps: [], ops: [ {obj: '_root', action: 'set', key: 'number', insert: false, datatype: 'int', value: 4, pred: [`3@${actor2}`]} ] }) const [/* state4 */, /* patch4 */, binChange4] = Backend.applyLocalChange(state3, change4) assert.deepStrictEqual(decodeChange(binChange4).deps, [decodeChange(binChange3).hash]) }) }) describe('applying patches', () => { it('should set root object properties', () => { const actor = uuid() const patch = { clock: {[actor]: 1}, diffs: {objectId: '_root', type: 'map', props: {bird: {[actor]: {type: 'value', value: 'magpie'}}}} } const doc = Frontend.applyPatch(Frontend.init(), patch) assert.deepStrictEqual(doc, {bird: 'magpie'}) }) it('should reveal conflicts on root object properties', () => { const patch = { clock: {actor1: 1, actor2: 1}, diffs: {objectId: '_root', type: 'map', props: { favoriteBird: {actor1: {type: 'value', value: 'robin'}, actor2: {type: 'value', value: 'wagtail'}} }} } const doc = Frontend.applyPatch(Frontend.init(), patch) assert.deepStrictEqual(doc, {favoriteBird: 'wagtail'}) assert.deepStrictEqual(Frontend.getConflicts(doc, 'favoriteBird'), {actor1: 'robin', actor2: 'wagtail'}) }) it('should create nested maps', () => { const birds = uuid(), actor = uuid() const patch = { clock: {[actor]: 1}, diffs: {objectId: '_root', type: 'map', props: {birds: {[actor]: { objectId: birds, type: 'map', props: {wrens: {[actor]: {value: 3}}} }}}} } const doc = Frontend.applyPatch(Frontend.init(), patch) assert.deepStrictEqual(doc, {birds: {wrens: 3}}) }) it('should apply updates inside nested maps', () => { const birds = uuid(), actor = uuid() const patch1 = { clock: {[actor]: 1}, diffs: {objectId: '_root', type: 'map', props: {birds: {[actor]: { objectId: birds, type: 'map', props: {wrens: {[actor]: {type: 'value', value: 3}}} }}}} } const patch2 = { clock: {[actor]: 2}, diffs: {objectId: '_root', type: 'map', props: {birds: {[actor]: { objectId: birds, type: 'map', props: {sparrows: {[actor]: {type: 'value', value: 15}}} }}}} } const doc1 = Frontend.applyPatch(Frontend.init(), patch1) const doc2 = Frontend.applyPatch(doc1, patch2) assert.deepStrictEqual(doc1, {birds: {wrens: 3}}) assert.deepStrictEqual(doc2, {birds: {wrens: 3, sparrows: 15}}) }) it('should apply updates inside map key conflicts', () => { const birds1 = uuid(), birds2 = uuid() const patch1 = { clock: {[birds1]: 1, [birds2]: 1}, diffs: {objectId: '_root', type: 'map', props: {favoriteBirds: { actor1: {objectId: birds1, type: 'map', props: {blackbirds: {actor1: {type: 'value', value: 1}}}}, actor2: {objectId: birds2, type: 'map', props: {wrens: {actor2: {type: 'value', value: 3}}}} }}} } const patch2 = { clock: {[birds1]: 2, [birds2]: 1}, diffs: {objectId: '_root', type: 'map', props: {favoriteBirds: { actor1: {objectId: birds1, type: 'map', props: {blackbirds: {actor1: {value: 2}}}}, actor2: {objectId: birds2, type: 'map'} }}} } const doc1 = Frontend.applyPatch(Frontend.init(), patch1) const doc2 = Frontend.applyPatch(doc1, patch2) assert.deepStrictEqual(doc1, {favoriteBirds: {wrens: 3}}) assert.deepStrictEqual(doc2, {favoriteBirds: {wrens: 3}}) assert.deepStrictEqual(Frontend.getConflicts(doc1, 'favoriteBirds'), {actor1: {blackbirds: 1}, actor2: {wrens: 3}}) assert.deepStrictEqual(Frontend.getConflicts(doc2, 'favoriteBirds'), {actor1: {blackbirds: 2}, actor2: {wrens: 3}}) }) it('should structure-share unmodified objects', () => { const birds = uuid(), mammals = uuid(), actor = uuid() const patch1 = { clock: {[actor]: 1}, diffs: {objectId: '_root', type: 'map', props: { birds: {[actor]: {objectId: birds, type: 'map', props: {wrens: {[actor]: {value: 3}}}}}, mammals: {[actor]: {objectId: mammals, type: 'map', props: {badgers: {[actor]: {value: 1}}}}} }} } const patch2 = { clock: {[actor]: 2}, diffs: {objectId: '_root', type: 'map', props: { birds: {[actor]: {objectId: birds, type: 'map', props: {sparrows: {[actor]: {value: 15}}}}} }} } const doc1 = Frontend.applyPatch(Frontend.init(), patch1) const doc2 = Frontend.applyPatch(doc1, patch2) assert.deepStrictEqual(doc1, {birds: {wrens: 3}, mammals: {badgers: 1}}) assert.deepStrictEqual(doc2, {birds: {wrens: 3, sparrows: 15}, mammals: {badgers: 1}}) assert.strictEqual(doc1.mammals, doc2.mammals) }) it('should delete keys in maps', () => { const actor = uuid() const patch1 = { clock: {[actor]: 1}, diffs: {objectId: '_root', type: 'map', props: { magpies: {[actor]: {value: 2}}, sparrows: {[actor]: {value: 15}} }} } const patch2 = { clock: {[actor]: 2}, diffs: {objectId: '_root', type: 'map', props: { magpies: {} }} } const doc1 = Frontend.applyPatch(Frontend.init(), patch1) const doc2 = Frontend.applyPatch(doc1, patch2) assert.deepStrictEqual(doc1, {magpies: 2, sparrows: 15}) assert.deepStrictEqual(doc2, {sparrows: 15}) }) it('should create lists', () => { const birds = uuid(), actor = uuid() const patch = { clock: {[actor]: 1}, diffs: {objectId: '_root', type: 'map', props: {birds: {[actor]: { objectId: birds, type: 'list', edits: [ {action: 'insert', index: 0, elemId: `2@${actor}`, opId: `2@${actor}`, value: {value: 'chaffinch'}} ] }}}} } const doc = Frontend.applyPatch(Frontend.init(), patch) assert.deepStrictEqual(doc, {birds: ['chaffinch']}) }) it('should apply updates inside lists', () => { const birds = uuid(), actor = uuid() const patch1 = { clock: {[actor]: 1}, diffs: {objectId: '_root', type: 'map', props: {birds: {[actor]: { objectId: birds, type: 'list', edits: [ {action: 'insert', index: 0, elemId: `2@${actor}`, opId: `2@${actor}`, value: {value: 'chaffinch'}} ] }}}} } const patch2 = { clock: {[actor]: 2}, diffs: {objectId: '_root', type: 'map', props: {birds: {[actor]: { objectId: birds, type: 'list', edits: [{action: 'update', index: 0, opId: `3@${actor}`, value: {value: 'greenfinch'}}] }}}} } const doc1 = Frontend.applyPatch(Frontend.init(), patch1) const doc2 = Frontend.applyPatch(doc1, patch2) assert.deepStrictEqual(doc1, {birds: ['chaffinch']}) assert.deepStrictEqual(doc2, {birds: ['greenfinch']}) }) it('should apply updates inside list element conflicts', () => { const actor1 = '01234567', actor2 = '89abcdef', birds = `1@${actor1}` const patch1 = { clock: {[actor1]: 2, [actor2]: 1}, diffs: {objectId: '_root', type: 'map', props: {birds: {[birds]: { objectId: birds, type: 'list', edits: [ {action: 'insert', index: 0, elemId: `2@${actor1}`, opId: `2@${actor1}`, value: { objectId: `2@${actor1}`, type: 'map', props: { species: {[`3@${actor1}`]: {type: 'value', value: 'woodpecker'}}, numSeen: {[`4@${actor1}`]: {type: 'value', value: 1}} } }}, {action: 'update', index: 0, opId: `2@${actor2}`, value: { objectId: `2@${actor2}`, type: 'map', props: { species: {[`3@${actor2}`]: {type: 'value', value: 'lapwing'}}, numSeen: {[`4@${actor2}`]: {type: 'value', value: 2}} } }} ] }}}} } const patch2 = { clock: {[actor1]: 3, [actor2]: 1}, diffs: {objectId: '_root', type: 'map', props: {birds: {[birds]: { objectId: birds, type: 'list', edits: [ {action: 'update', index: 0, opId: `2@${actor1}`, value: { objectId: `2@${actor1}`, type: 'map', props: { numSeen: {[`5@${actor1}`]: {type: 'value', value: 2}} } }}, {action: 'update', index: 0, opId: `2@${actor2}`, value: { objectId: `2@${actor2}`, type: 'map', props: {} }} ] }}}} } const patch3 = { clock: {[actor1]: 3, [actor2]: 1}, diffs: {objectId: '_root', type: 'map', props: {birds: {[birds]: { objectId: birds, type: 'list', edits: [ {action: 'update', index: 0, opId: `2@${actor1}`, value: { objectId: `2@${actor1}`, type: 'map', props: { numSeen: {[`6@${actor1}`]: {type: 'value', value: 2}} } }} ] }}}} } const doc1 = Frontend.applyPatch(Frontend.init(), patch1) const doc2 = Frontend.applyPatch(doc1, patch2) const doc3 = Frontend.applyPatch(doc2, patch3) assert.deepStrictEqual(doc1, {birds: [{species: 'lapwing', numSeen: 2}]}) assert.deepStrictEqual(doc2, {birds: [{species: 'lapwing', numSeen: 2}]}) assert.deepStrictEqual(doc3, {birds: [{species: 'woodpecker', numSeen: 2}]}) assert.strictEqual(doc1.birds[0], doc2.birds[0]) assert.deepStrictEqual(Frontend.getConflicts(doc1.birds, 0), { [`2@${actor1}`]: {species: 'woodpecker', numSeen: 1}, [`2@${actor2}`]: {species: 'lapwing', numSeen: 2} }) assert.deepStrictEqual(Frontend.getConflicts(doc2.birds, 0), { [`2@${actor1}`]: {species: 'woodpecker', numSeen: 2}, [`2@${actor2}`]: {species: 'lapwing', numSeen: 2} }) assert.deepStrictEqual(Frontend.getConflicts(doc3.birds, 0), undefined) }) it('should apply multiinserts on lists', () => { const actor = uuid() const patch1 = { clock: {[actor]: 1}, diffs: {objectId: '_root', type: 'map', props: {birds: {[`@${actor}`]: { objectId: `1@${actor}`, type: 'list', edits: [ {action: 'multi-insert', index: 0, elemId: `2@${actor}`, values: ["chaffinch", "goldfinch", "wren"]} ] }}}} } const doc = Frontend.applyPatch(Frontend.init(), patch1) assert.deepStrictEqual(doc, {birds: ["chaffinch", "goldfinch", "wren"]}) }) it('should delete list elements', () => { const birds = uuid(), actor = uuid() const patch1 = { clock: {[actor]: 1}, diffs: {objectId: '_root', type: 'map', props: {birds: {[`1@${actor}`]: { objectId: birds, type: 'list', edits: [ {action: 'insert', index: 0, elemId: `2@${actor}`, opId: `2@${actor}`, value: {value: 'chaffinch'}}, {action: 'insert', index: 1, elemId: `3@${actor}`, opId: `3@${actor}`, value: {value: 'goldfinch'}} ] }}}} } const patch2 = { clock: {[actor]: 2}, diffs: {objectId: '_root', type: 'map', props: {birds: {[`1@${actor}`]: { objectId: birds, type: 'list', props: {}, edits: [{action: 'remove', index: 0, count: 1}] }}}} } const doc1 = Frontend.applyPatch(Frontend.init(), patch1) const doc2 = Frontend.applyPatch(doc1, patch2) assert.deepStrictEqual(doc1, {birds: ['chaffinch', 'goldfinch']}) assert.deepStrictEqual(doc2, {birds: ['goldfinch']}) }) it('should delete multiple list elements', () => { const birds = uuid(), actor = uuid() const patch1 = { clock: {[actor]: 1}, diffs: {objectId: '_root', type: 'map', props: {birds: {[`1@${actor}`]: { objectId: birds, type: 'list', edits: [ {action: 'insert', index: 0, elemId: `2@${actor}`, opId: `2@${actor}`, value: {value: 'chaffinch'}}, {action: 'insert', index: 1, elemId: `3@${actor}`, opId: `3@${actor}`, value: {value: 'goldfinch'}} ] }}}} } const patch2 = { clock: {[actor]: 2}, diffs: {objectId: '_root', type: 'map', props: {birds: {[`1@${actor}`]: { objectId: birds, type: 'list', props: {}, edits: [{action: 'remove', index: 0, count: 2}] }}}} } const doc1 = Frontend.applyPatch(Frontend.init(), patch1) const doc2 = Frontend.applyPatch(doc1, patch2) assert.deepStrictEqual(doc1, {birds: ['chaffinch', 'goldfinch']}) assert.deepStrictEqual(doc2, {birds: []}) }) it('should apply updates at different levels of the object tree', () => { const actor = uuid() const patch1 = { clock: {[actor]: 1}, diffs: {objectId: '_root', type: 'map', props: { counts: {[`1@${actor}`]: {objectId: `1@${actor}`, type: 'map', props: { magpies: {[`2@${actor}`]: {value: 2}} }}}, details: {[`3@${actor}`]: {objectId: `3@${actor}`, type: 'list', edits: [{action: 'insert', index: 0, elemId: `4@${actor}`, opId: `4@${actor}`, value: { objectId: `4@${actor}`, type: 'map', props: { species: {[`5@${actor}`]: {type: 'value', value: 'magpie'}}, family: {[`6@${actor}`]: {type: 'value', value: 'corvidae'}} } }}]}} }} } const patch2 = { clock: {[actor]: 2}, diffs: {objectId: '_root', type: 'map', props: { counts: {[`1@${actor}`]: {objectId: `1@${actor}`, type: 'map', props: { magpies: {[`7@${actor}`]: {type: 'value', value: 3}} }}}, details: {[`3@${actor}`]: {objectId: `3@${actor}`, type: 'list', edits: [ {action: 'update', index: 0, opId: `4@${actor}`, value: { objectId: `4@${actor}`, type: 'map', props: { species: {[`8@${actor}`]: {type: 'value', value: 'Eurasian magpie'}} } }} ]}} }} } const doc1 = Frontend.applyPatch(Frontend.init(), patch1) const doc2 = Frontend.applyPatch(doc1, patch2) assert.deepStrictEqual(doc1, {counts: {magpies: 2}, details: [{species: 'magpie', family: 'corvidae'}]}) assert.deepStrictEqual(doc2, {counts: {magpies: 3}, details: [{species: 'Eurasian magpie', family: 'corvidae'}]}) }) }) it('should create text objects', () => { const actor = uuid() const patch1 = { clock: {[actor]: 1}, diffs: {objectId: '_root', type: 'map', props: { text: {[`1@${actor}`]: {objectId: `1@${actor}`, type: 'text', edits: [ {action: 'insert', index: 0, elemId: `2@${actor}`, opId: `2@${actor}`, value: {type: 'value', value: '1'}}, {action: 'multi-insert', index: 1, elemId: `3@${actor}`, values: ['2', '3', '4']} ]}} }} } const doc = Frontend.applyPatch(Frontend.init(), patch1) assert.deepStrictEqual(doc.text.toString(), '1234') }) }) ================================================ FILE: test/fuzz_test.js ================================================ /** * Miniature implementation of a subset of Automerge, which is used below as definition of the * expected behaviour during fuzz testing. Supports the following: * - only map, list, and primitive datatypes (no table, text, counter, or date objects) * - no undo/redo * - no conflicts on concurrent updates to the same field (uses last-writer-wins instead) * - no API for creating new changes (you need to create change objects yourself) * - no buffering of changes that are missing their causal dependencies * - no saving or loading in serialised form * - relies on object mutation (no immutability) */ class Micromerge { constructor() { this.byActor = {} // map from actorId to array of changes this.byObjId = {_root: {}} // objects, keyed by the ID of the operation that created the object this.metadata = {_root: {}} // map from objID to object with CRDT metadata for each object field } get root() { return this.byObjId._root } /** * Updates the document state by applying the change object `change`, in the format documented here: * https://github.com/automerge/automerge/blob/performance/BINARY_FORMAT.md#json-representation-of-changes */ applyChange(change) { // Check that the change's dependencies are met const lastSeq = this.byActor[change.actor] ? this.byActor[change.actor].length : 0 if (change.seq !== lastSeq + 1) { throw new RangeError(`Expected sequence number ${lastSeq + 1}, got ${change.seq}`) } for (let [actor, dep] of Object.entries(change.deps || {})) { if (!this.byActor[actor] || this.byActor[actor].length < dep) { throw new RangeError(`Missing dependency: change ${dep} by actor ${actor}`) } } if (!this.byActor[change.actor]) this.byActor[change.actor] = [] this.byActor[change.actor].push(change) change.ops.forEach((op, index) => { this.applyOp(Object.assign({opId: `${change.startOp + index}@${change.actor}`}, op)) }) } /** * Updates the document state with one of the operations from a change. */ applyOp(op) { if (!this.metadata[op.obj]) throw new RangeError(`Object does not exist: ${op.obj}`) if (op.action === 'makeMap') { this.byObjId[op.opId] = {} this.metadata[op.opId] = {} } else if (op.action === 'makeList') { this.byObjId[op.opId] = [] this.metadata[op.opId] = [] } else if (op.action !== 'set' && op.action !== 'del') { throw new RangeError(`Unsupported operation type: ${op.action}`) } if (Array.isArray(this.metadata[op.obj])) { if (op.insert) this.applyListInsert(op); else this.applyListUpdate(op) } else if (!this.metadata[op.obj][op.key] || this.compareOpIds(this.metadata[op.obj][op.key], op.opId)) { this.metadata[op.obj][op.key] = op.opId if (op.action === 'del') { delete this.byObjId[op.obj][op.key] } else if (op.action.startsWith('make')) { this.byObjId[op.obj][op.key] = this.byObjId[op.opId] } else { this.byObjId[op.obj][op.key] = op.value } } } /** * Applies a list insertion operation. */ applyListInsert(op) { const meta = this.metadata[op.obj] const value = op.action.startsWith('make') ? this.byObjId[op.opId] : op.value let {index, visible} = (op.key === '_head') ? {index: -1, visible: 0} : this.findListElement(op.obj, op.key) if (index >= 0 && !meta[index].deleted) visible++ index++ while (index < meta.length && this.compareOpIds(op.opId, meta[index].elemId)) { if (!meta[index].deleted) visible++ index++ } meta.splice(index, 0, {elemId: op.opId, valueId: op.opId, deleted: false}) this.byObjId[op.obj].splice(visible, 0, value) } /** * Applies a list element update (setting the value of a list element, or deleting a list element). */ applyListUpdate(op) { const {index, visible} = this.findListElement(op.obj, op.key) const meta = this.metadata[op.obj][index] if (op.action === 'del') { if (!meta.deleted) this.byObjId[op.obj].splice(visible, 1) meta.deleted = true } else if (this.compareOpIds(meta.valueId, op.opId)) { if (!meta.deleted) { this.byObjId[op.obj][visible] = op.action.startsWith('make') ? this.byObjId[op.opId] : op.value } meta.valueId = op.opId } } /** * Searches for the list element with ID `elemId` in the object with ID `objId`. Returns an object * `{index, visible}` where `index` is the index of the element in the metadata array, and * `visible` is the number of non-deleted elements that precede the specified element. */ findListElement(objectId, elemId) { let index = 0, visible = 0, meta = this.metadata[objectId] while (index < meta.length && meta[index].elemId !== elemId) { if (!meta[index].deleted) visible++ index++ } if (index === meta.length) throw new RangeError(`List element not found: ${elemId}`) return {index, visible} } /** * Compares two operation IDs in the form `counter@actor`. Returns true if `id1` has a lower counter * than `id2`, or if the counter values are the same and `id1` has an actorId that sorts * lexicographically before the actorId of `id2`. */ compareOpIds(id1, id2) { const regex = /^([0-9]+)@(.*)$/ const match1 = regex.exec(id1), match2 = regex.exec(id2) const counter1 = parseInt(match1[1], 10), counter2 = parseInt(match2[1], 10) return (counter1 < counter2) || (counter1 === counter2 && match1[2] < match2[2]) } } /* TESTS */ const assert = require('assert') const change1 = {actor: '1234', seq: 1, deps: {}, startOp: 1, ops: [ {action: 'set', obj: '_root', key: 'title', insert: false, value: 'Hello'}, {action: 'makeList', obj: '_root', key: 'tags', insert: false}, {action: 'set', obj: '2@1234', key: '_head', insert: true, value: 'foo'} ]} const change2 = {actor: '1234', seq: 2, deps: {}, startOp: 4, ops: [ {action: 'set', obj: '_root', key: 'title', insert: false, value: 'Hello 1'}, {action: 'set', obj: '2@1234', key: '3@1234', insert: true, value: 'bar'}, {action: 'del', obj: '2@1234', key: '3@1234', insert: false} ]} const change3 = {actor: 'abcd', seq: 1, deps: {'1234': 1}, startOp: 4, ops: [ {action: 'set', obj: '_root', key: 'title', insert: false, value: 'Hello 2'}, {action: 'set', obj: '2@1234', key: '3@1234', insert: true, value: 'baz'} ]} let doc1 = new Micromerge(), doc2 = new Micromerge() for (let c of [change1, change2, change3]) doc1.applyChange(c) for (let c of [change1, change3, change2]) doc2.applyChange(c) assert.deepStrictEqual(doc1.root, {title: 'Hello 2', tags: ['baz', 'bar']}) assert.deepStrictEqual(doc2.root, {title: 'Hello 2', tags: ['baz', 'bar']}) const change4 = {actor: '2345', seq: 1, deps: {}, startOp: 1, ops: [ {action: 'makeList', obj: '_root', key: 'todos', insert: false}, {action: 'set', obj: '1@2345', key: '_head', insert: true, value: 'Task 1'}, {action: 'set', obj: '1@2345', key: '2@2345', insert: true, value: 'Task 2'} ]} let doc3 = new Micromerge() doc3.applyChange(change4) assert.deepStrictEqual(doc3.root, {todos: ['Task 1', 'Task 2']}) const change5 = {actor: '2345', seq: 2, deps: {}, startOp: 4, ops: [ {action: 'del', obj: '1@2345', key: '2@2345', insert: false}, {action: 'set', obj: '1@2345', key: '3@2345', insert: true, value: 'Task 3'} ]} doc3.applyChange(change5) assert.deepStrictEqual(doc3.root, {todos: ['Task 2', 'Task 3']}) const change6 = {actor: '2345', seq: 3, deps: {}, startOp: 6, ops: [ {action: 'del', obj: '1@2345', key: '3@2345', insert: false}, {action: 'set', obj: '1@2345', key: '5@2345', insert: false, value: 'Task 3b'}, {action: 'set', obj: '1@2345', key: '5@2345', insert: true, value: 'Task 4'} ]} doc3.applyChange(change6) assert.deepStrictEqual(doc3.root, {todos: ['Task 3b', 'Task 4']}) ================================================ FILE: test/helpers.js ================================================ const assert = require('assert') const { Encoder } = require('../backend/encoding') // Assertion that succeeds if the first argument deepStrictEquals at least one of the // subsequent arguments (but we don't care which one) function assertEqualsOneOf(actual, ...expected) { assert(expected.length > 0) for (let i = 0; i < expected.length; i++) { try { assert.deepStrictEqual(actual, expected[i]) return // if we get here without an exception, that means success } catch (e) { if (!e.name.match(/^AssertionError/) || i === expected.length - 1) throw e } } } /** * Asserts that the byte array maintained by `encoder` contains the same byte * sequence as the array `bytes`. */ function checkEncoded(encoder, bytes, detail) { const encoded = (encoder instanceof Encoder) ? encoder.buffer : encoder const expected = new Uint8Array(bytes) const message = (detail ? `${detail}: ` : '') + `${encoded} expected to equal ${expected}` assert(encoded.byteLength === expected.byteLength, message) for (let i = 0; i < encoded.byteLength; i++) { assert(encoded[i] === expected[i], message) } } module.exports = { assertEqualsOneOf, checkEncoded } ================================================ FILE: test/new_backend_test.js ================================================ const assert = require('assert') const { checkEncoded } = require('./helpers') const { DOC_OPS_COLUMNS, encodeChange, decodeChange } = require('../backend/columnar') const { MAX_BLOCK_SIZE, BackendDoc, bloomFilterContains } = require('../backend/new') const uuid = require('../src/uuid') function checkColumns(block, expectedCols) { for (let actual of block.columns) { const {columnName} = DOC_OPS_COLUMNS.find(({columnId}) => columnId === actual.columnId) || {columnName: actual.columnId.toString()} if (expectedCols[columnName]) { checkEncoded(actual.decoder.buf, expectedCols[columnName], `${columnName} column`) } else if (columnName !== 'chldActor' && columnName !== 'chldCtr') { throw new Error(`Unexpected column ${columnName}`) } } for (let expectedName of Object.keys(expectedCols)) { const {columnId} = DOC_OPS_COLUMNS.find(({columnName}) => columnName === expectedName) || {columnId: parseInt(expectedName, 10)} if (!block.columns.find(actual => actual.columnId === columnId)) { throw new Error(`Missing column ${expectedName}`) } } } function hash(change) { return decodeChange(encodeChange(change)).hash } describe('BackendDoc applying changes', () => { it('should overwrite root object properties (1)', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'set', obj: '_root', key: 'x', datatype: 'uint', value: 3, pred: []}, {action: 'set', obj: '_root', key: 'y', datatype: 'uint', value: 4, pred: []} ]} const change2 = {actor, seq: 2, startOp: 3, time: 0, deps: [hash(change1)], ops: [ {action: 'set', obj: '_root', key: 'x', datatype: 'uint', value: 5, pred: [`1@${actor}`]} ]} const backend = new BackendDoc() assert.deepStrictEqual(backend.applyChanges([encodeChange(change1)]), { maxOp: 2, clock: {[actor]: 1}, deps: [hash(change1)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: { x: {[`1@${actor}`]: {type: 'value', value: 3, datatype: 'uint'}}, y: {[`2@${actor}`]: {type: 'value', value: 4, datatype: 'uint'}} }} }) assert.deepStrictEqual(backend.applyChanges([encodeChange(change2)]), { maxOp: 3, clock: {[actor]: 2}, deps: [hash(change2)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: { x: {[`3@${actor}`]: {type: 'value', value: 5, datatype: 'uint'}} }} }) checkColumns(backend.blocks[0], { objActor: [], objCtr: [], keyActor: [], keyCtr: [], keyStr: [2, 1, 0x78, 0x7f, 1, 0x79], // 'x', 'x', 'y' idActor: [3, 0], idCtr: [0x7d, 1, 2, 0x7f], // 1, 3, 2 insert: [3], action: [3, 1], valLen: [3, 0x13], valRaw: [3, 5, 4], succNum: [0x7f, 1, 2, 0], succActor: [0x7f, 0], succCtr: [0x7f, 3] }) assert.strictEqual(backend.blocks[0].lastKey, 'y') assert.strictEqual(backend.blocks[0].numOps, 3) assert.strictEqual(backend.blocks[0].lastObjectActor, null) assert.strictEqual(backend.blocks[0].lastObjectCtr, null) }) it('should overwrite root object properties (2)', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'set', obj: '_root', key: 'x', datatype: 'uint', value: 3, pred: []}, {action: 'set', obj: '_root', key: 'y', datatype: 'uint', value: 4, pred: []} ]} const change2 = {actor, seq: 2, startOp: 3, time: 0, deps: [hash(change1)], ops: [ {action: 'set', obj: '_root', key: 'y', datatype: 'uint', value: 5, pred: [`2@${actor}`]}, {action: 'set', obj: '_root', key: 'z', datatype: 'uint', value: 6, pred: []} ]} const backend = new BackendDoc() assert.deepStrictEqual(backend.applyChanges([encodeChange(change1)]), { maxOp: 2, clock: {[actor]: 1}, deps: [hash(change1)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: { x: {[`1@${actor}`]: {type: 'value', value: 3, datatype: 'uint'}}, y: {[`2@${actor}`]: {type: 'value', value: 4, datatype: 'uint'}} }} }) assert.deepStrictEqual(backend.applyChanges([encodeChange(change2)]), { maxOp: 4, clock: {[actor]: 2}, deps: [hash(change2)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: { y: {[`3@${actor}`]: {type: 'value', value: 5, datatype: 'uint'}}, z: {[`4@${actor}`]: {type: 'value', value: 6, datatype: 'uint'}} }} }) checkColumns(backend.blocks[0], { objActor: [], objCtr: [], keyActor: [], keyCtr: [], keyStr: [0x7f, 1, 0x78, 2, 1, 0x79, 0x7f, 1, 0x7a], // 'x', 'y', 'y', 'z' idActor: [4, 0], idCtr: [4, 1], insert: [4], action: [4, 1], valLen: [4, 0x13], valRaw: [3, 4, 5, 6], succNum: [0x7e, 0, 1, 2, 0], succActor: [0x7f, 0], succCtr: [0x7f, 3] }) assert.strictEqual(backend.blocks[0].lastKey, 'z') assert.strictEqual(backend.blocks[0].numOps, 4) assert.strictEqual(backend.blocks[0].lastObjectActor, null) assert.strictEqual(backend.blocks[0].lastObjectCtr, null) }) it('should allow concurrent overwrites of the same value', () => { const actor1 = '01234567', actor2 = '89abcdef', actor3 = 'fedcba98' const change1 = {actor: actor1, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'set', obj: '_root', key: 'x', datatype: 'uint', value: 1, pred: []} ]} const change2 = {actor: actor1, seq: 2, startOp: 2, time: 0, deps: [hash(change1)], ops: [ {action: 'set', obj: '_root', key: 'x', datatype: 'uint', value: 2, pred: [`1@${actor1}`]} ]} const change3 = {actor: actor2, seq: 1, startOp: 2, time: 0, deps: [hash(change1)], ops: [ {action: 'set', obj: '_root', key: 'x', datatype: 'uint', value: 3, pred: [`1@${actor1}`]} ]} const change4 = {actor: actor3, seq: 1, startOp: 2, time: 0, deps: [hash(change1)], ops: [ {action: 'set', obj: '_root', key: 'x', datatype: 'uint', value: 4, pred: [`1@${actor1}`]} ]} const backend1 = new BackendDoc(), backend2 = new BackendDoc() backend1.applyChanges([encodeChange(change1)]) assert.deepStrictEqual(backend1.applyChanges([encodeChange(change2)]), { maxOp: 2, clock: {[actor1]: 2}, deps: [hash(change2)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {x: { [`2@${actor1}`]: {type: 'value', value: 2, datatype: 'uint'} }}} }) assert.deepStrictEqual(backend1.applyChanges([encodeChange(change3)]), { maxOp: 2, clock: {[actor1]: 2, [actor2]: 1}, deps: [hash(change2), hash(change3)].sort(), pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: { x: { [`2@${actor1}`]: {type: 'value', value: 2, datatype: 'uint'}, [`2@${actor2}`]: {type: 'value', value: 3, datatype: 'uint'} }}} }) assert.deepStrictEqual(backend1.applyChanges([encodeChange(change4)]), { maxOp: 2, clock: {[actor1]: 2, [actor2]: 1, [actor3]: 1}, pendingChanges: 0, deps: [hash(change2), hash(change3), hash(change4)].sort(), diffs: {objectId: '_root', type: 'map', props: {x: { [`2@${actor1}`]: {type: 'value', value: 2, datatype: 'uint'}, [`2@${actor2}`]: {type: 'value', value: 3, datatype: 'uint'}, [`2@${actor3}`]: {type: 'value', value: 4, datatype: 'uint'} }}} }) backend2.applyChanges([encodeChange(change1)]) assert.deepStrictEqual(backend2.applyChanges([encodeChange(change4)]), { maxOp: 2, clock: {[actor1]: 1, [actor3]: 1}, deps: [hash(change4)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {x: { [`2@${actor3}`]: {type: 'value', value: 4, datatype: 'uint'} }}} }) assert.deepStrictEqual(backend2.applyChanges([encodeChange(change3)]), { maxOp: 2, clock: {[actor1]: 1, [actor2]: 1, [actor3]: 1}, pendingChanges: 0, deps: [hash(change3), hash(change4)].sort(), diffs: {objectId: '_root', type: 'map', props: {x: { [`2@${actor2}`]: {type: 'value', value: 3, datatype: 'uint'}, [`2@${actor3}`]: {type: 'value', value: 4, datatype: 'uint'} }}} }) assert.deepStrictEqual(backend2.applyChanges([encodeChange(change2)]), { maxOp: 2, clock: {[actor1]: 2, [actor2]: 1, [actor3]: 1}, pendingChanges: 0, deps: [hash(change2), hash(change3), hash(change4)].sort(), diffs: {objectId: '_root', type: 'map', props: {x: { [`2@${actor1}`]: {type: 'value', value: 2, datatype: 'uint'}, [`2@${actor2}`]: {type: 'value', value: 3, datatype: 'uint'}, [`2@${actor3}`]: {type: 'value', value: 4, datatype: 'uint'} }}} }) checkColumns(backend1.blocks[0], { objActor: [], objCtr: [], keyActor: [], keyCtr: [], keyStr: [4, 1, 0x78], // 4x 'x' idActor: [2, 0, 0x7e, 1, 2], // 0, 0, 1, 2 idCtr: [2, 1, 2, 0], // 1, 2, 2, 2 insert: [4], action: [4, 1], valLen: [4, 0x13], valRaw: [1, 2, 3, 4], succNum: [0x7f, 3, 3, 0], // 3, 0, 0, 0 succActor: [0x7d, 0, 1, 2], // 0, 1, 2 succCtr: [0x7f, 2, 2, 0] // 2, 2, 2 }) assert.strictEqual(backend1.blocks[0].lastKey, 'x') assert.strictEqual(backend1.blocks[0].numOps, 4) assert.strictEqual(backend1.blocks[0].lastObjectActor, null) assert.strictEqual(backend1.blocks[0].lastObjectCtr, null) // The two backends are not identical because actors appear in a different order checkColumns(backend2.blocks[0], { objActor: [], objCtr: [], keyActor: [], keyCtr: [], keyStr: [4, 1, 0x78], // 4x 'x' idActor: [2, 0, 0x7e, 2, 1], // 0, 0, 2, 1 <-- different from backend1 idCtr: [2, 1, 2, 0], // 1, 2, 2, 2 insert: [4], action: [4, 1], valLen: [4, 0x13], valRaw: [1, 2, 3, 4], succNum: [0x7f, 3, 3, 0], // 3, 0, 0, 0 succActor: [0x7d, 0, 2, 1], // 0, 2, 1 <-- different from backend1 succCtr: [0x7f, 2, 2, 0] // 2, 2, 2 }) assert.strictEqual(backend2.blocks[0].lastKey, 'x') assert.strictEqual(backend2.blocks[0].numOps, 4) }) it('should allow a conflict to be resolved', () => { const actor1 = '01234567', actor2 = '89abcdef' const change1 = {actor: actor1, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'set', obj: '_root', key: 'x', datatype: 'uint', value: 1, pred: []} ]} const change2 = {actor: actor2, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'set', obj: '_root', key: 'x', datatype: 'uint', value: 2, pred: []} ]} const change3 = {actor: actor1, seq: 2, startOp: 2, time: 0, deps: [hash(change1), hash(change2)], ops: [ {action: 'set', obj: '_root', key: 'x', datatype: 'uint', value: 3, pred: [`1@${actor1}`, `1@${actor2}`]} ]} const backend = new BackendDoc() assert.deepStrictEqual(backend.applyChanges([encodeChange(change1)]), { maxOp: 1, clock: {[actor1]: 1}, deps: [hash(change1)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {x: { [`1@${actor1}`]: {type: 'value', value: 1, datatype: 'uint'} }}} }) assert.deepStrictEqual(backend.applyChanges([encodeChange(change2)]), { maxOp: 1, clock: {[actor1]: 1, [actor2]: 1}, deps: [hash(change1), hash(change2)].sort(), pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {x: { [`1@${actor1}`]: {type: 'value', value: 1, datatype: 'uint'}, [`1@${actor2}`]: {type: 'value', value: 2, datatype: 'uint'} }}} }) assert.deepStrictEqual(backend.applyChanges([encodeChange(change3)]), { maxOp: 2, clock: {[actor1]: 2, [actor2]: 1}, deps: [hash(change3)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {x: { [`2@${actor1}`]: {type: 'value', value: 3, datatype: 'uint'} }}} }) checkColumns(backend.blocks[0], { objActor: [], objCtr: [], keyActor: [], keyCtr: [], keyStr: [3, 1, 0x78], // 3x 'x' idActor: [0x7d, 0, 1, 0], // 0, 1, 0 idCtr: [0x7d, 1, 0, 1], // 1, 1, 2 insert: [3], action: [3, 1], valLen: [3, 0x13], valRaw: [1, 2, 3], succNum: [2, 1, 0x7f, 0], // 1, 1, 0 succActor: [2, 0], succCtr: [0x7e, 2, 0] // 2, 2 }) assert.strictEqual(backend.blocks[0].lastKey, 'x') assert.strictEqual(backend.blocks[0].numOps, 3) }) it('should throw an error if the predecessor operation does not exist (1)', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'set', obj: '_root', key: 'x', datatype: 'uint', value: 1, pred: []}, {action: 'set', obj: '_root', key: 'y', datatype: 'uint', value: 2, pred: []} ]} const change2 = {actor, seq: 2, startOp: 3, time: 0, deps: [hash(change1)], ops: [ {action: 'set', obj: '_root', key: 'x', datatype: 'uint', value: 3, pred: [`2@${actor}`]} ]} const backend = new BackendDoc() backend.applyChanges([encodeChange(change1)]) assert.throws(() => { backend.applyChanges([encodeChange(change2)]) }, /no matching operation for pred/) }) it('should throw an error if the predecessor operation does not exist (2)', () => { const actor1 = '01234567', actor2 = '89abcdef' const change1 = {actor: actor1, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'set', obj: '_root', key: 'x', datatype: 'uint', value: 1, pred: []} ]} const change2 = {actor: actor2, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'set', obj: '_root', key: 'w', datatype: 'uint', value: 2, pred: []}, {action: 'set', obj: '_root', key: 'x', datatype: 'uint', value: 2, pred: []} ]} const change3 = {actor: actor1, seq: 2, startOp: 2, time: 0, deps: [hash(change1), hash(change2)], ops: [ {action: 'set', obj: '_root', key: 'x', datatype: 'uint', value: 3, pred: [`1@${actor2}`]} ]} const backend = new BackendDoc() backend.applyChanges([encodeChange(change1)]) backend.applyChanges([encodeChange(change2)]) assert.throws(() => { backend.applyChanges([encodeChange(change3)]) }, /no matching operation for pred/) }) it('should create and update nested maps', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeMap', obj: '_root', key: 'map', pred: []}, {action: 'set', obj: `1@${actor}`, key: 'x', value: 'a', pred: []}, {action: 'set', obj: `1@${actor}`, key: 'y', value: 'b', pred: []}, {action: 'set', obj: `1@${actor}`, key: 'z', value: 'c', pred: []} ]} const change2 = {actor, seq: 2, startOp: 5, time: 0, deps: [hash(change1)], ops: [ {action: 'set', obj: `1@${actor}`, key: 'y', value: 'B', pred: [`3@${actor}`]} ]} const backend = new BackendDoc() assert.deepStrictEqual(backend.applyChanges([encodeChange(change1)]), { maxOp: 4, clock: {[actor]: 1}, deps: [hash(change1)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {map: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'map', props: { x: {[`2@${actor}`]: {type: 'value', value: 'a'}}, y: {[`3@${actor}`]: {type: 'value', value: 'b'}}, z: {[`4@${actor}`]: {type: 'value', value: 'c'}} } }}}} }) assert.deepStrictEqual(backend.applyChanges([encodeChange(change2)]), { maxOp: 5, clock: {[actor]: 2}, deps: [hash(change2)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {map: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'map', props: {y: {[`5@${actor}`]: {type: 'value', value: 'B'}}} }}}} }) checkColumns(backend.blocks[0], { objActor: [0, 1, 4, 0], objCtr: [0, 1, 4, 1], keyActor: [], keyCtr: [], keyStr: [0x7e, 3, 0x6d, 0x61, 0x70, 1, 0x78, 2, 1, 0x79, 0x7f, 1, 0x7a], // 'map', 'x', 'y', 'y', 'z' idActor: [5, 0], idCtr: [3, 1, 0x7e, 2, 0x7f], // 1, 2, 3, 5, 4 insert: [5], action: [0x7f, 0, 4, 1], // makeMap, 4x set valLen: [0x7f, 0, 4, 0x16], // null, 4x 1-byte string valRaw: [0x61, 0x62, 0x42, 0x63], // 'a', 'b', 'B', 'c' succNum: [2, 0, 0x7f, 1, 2, 0], succActor: [0x7f, 0], succCtr: [0x7f, 5] }) assert.strictEqual(backend.blocks[0].lastKey, 'z') assert.strictEqual(backend.blocks[0].numOps, 5) assert.strictEqual(backend.blocks[0].lastObjectActor, 0) assert.strictEqual(backend.blocks[0].lastObjectCtr, 1) }) it('should create nested maps several levels deep', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeMap', obj: '_root', key: 'a', pred: []}, {action: 'makeMap', obj: `1@${actor}`, key: 'b', pred: []}, {action: 'makeMap', obj: `2@${actor}`, key: 'c', pred: []}, {action: 'set', obj: `3@${actor}`, key: 'd', datatype: 'uint', value: 1, pred: []} ]} const change2 = {actor, seq: 2, startOp: 5, time: 0, deps: [hash(change1)], ops: [ {action: 'set', obj: `3@${actor}`, key: 'd', datatype: 'uint', value: 2, pred: [`4@${actor}`]} ]} const backend = new BackendDoc() assert.deepStrictEqual(backend.applyChanges([encodeChange(change1)]), { maxOp: 4, clock: {[actor]: 1}, deps: [hash(change1)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {a: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'map', props: {b: {[`2@${actor}`]: { objectId: `2@${actor}`, type: 'map', props: {c: {[`3@${actor}`]: { objectId: `3@${actor}`, type: 'map', props: {d: {[`4@${actor}`]: { type: 'value', value: 1, datatype: 'uint' }}} }}} }}} }}}} }) assert.deepStrictEqual(backend.applyChanges([encodeChange(change2)]), { maxOp: 5, clock: {[actor]: 2}, deps: [hash(change2)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {a: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'map', props: {b: {[`2@${actor}`]: { objectId: `2@${actor}`, type: 'map', props: {c: {[`3@${actor}`]: { objectId: `3@${actor}`, type: 'map', props: {d: {[`5@${actor}`]: { type: 'value', value: 2, datatype: 'uint' }}} }}} }}} }}}} }) checkColumns(backend.blocks[0], { objActor: [0, 1, 4, 0], objCtr: [0, 1, 0x7e, 1, 2, 2, 3], // null, 1, 2, 3, 3 keyActor: [], keyCtr: [], keyStr: [0x7d, 1, 0x61, 1, 0x62, 1, 0x63, 2, 1, 0x64], // 'a', 'b', 'c', 'd', 'd' idActor: [5, 0], idCtr: [5, 1], // 1, 2, 3, 4, 5 insert: [5], action: [3, 0, 2, 1], // 3x makeMap, 2x set valLen: [3, 0, 2, 0x13], // 3x null, 2x uint valRaw: [1, 2], succNum: [3, 0, 0x7e, 1, 0], // 0, 0, 0, 1, 0 succActor: [0x7f, 0], succCtr: [0x7f, 5] }) assert.strictEqual(backend.blocks[0].lastKey, 'd') assert.strictEqual(backend.blocks[0].numOps, 5) assert.strictEqual(backend.blocks[0].lastObjectActor, 0) assert.strictEqual(backend.blocks[0].lastObjectCtr, 3) }) it('should create a text object', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeText', obj: '_root', key: 'text', insert: false, pred: []}, {action: 'set', obj: `1@${actor}`, elemId: '_head', insert: true, value: 'a', pred: []} ]} const backend = new BackendDoc() assert.deepStrictEqual(backend.applyChanges([encodeChange(change1)]), { maxOp: 2, clock: {[actor]: 1}, deps: [hash(change1)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {text: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'text', edits: [ {action: 'insert', index: 0, elemId: `2@${actor}`, opId: `2@${actor}`, value: {type: 'value', value: 'a'}} ] }}}} }) checkColumns(backend.blocks[0], { objActor: [0, 1, 0x7f, 0], objCtr: [0, 1, 0x7f, 1], keyActor: [], keyCtr: [0, 1, 0x7f, 0], keyStr: [0x7f, 4, 0x74, 0x65, 0x78, 0x74, 0, 1], // 'text', null idActor: [2, 0], idCtr: [2, 1], insert: [1, 1], action: [0x7e, 4, 1], valLen: [0x7e, 0, 0x16], valRaw: [0x61], succNum: [2, 0], succActor: [], succCtr: [] }) assert.strictEqual(backend.blocks[0].numOps, 2) assert.strictEqual(backend.blocks[0].lastKey, undefined) assert.strictEqual(backend.blocks[0].numVisible, 1) assert.strictEqual(backend.blocks[0].lastObjectActor, 0) assert.strictEqual(backend.blocks[0].lastObjectCtr, 1) assert.strictEqual(backend.blocks[0].firstVisibleActor, 0) assert.strictEqual(backend.blocks[0].firstVisibleCtr, 2) assert.strictEqual(backend.blocks[0].lastVisibleActor, 0) assert.strictEqual(backend.blocks[0].lastVisibleCtr, 2) assert.strictEqual(bloomFilterContains(backend.blocks[0].bloom, 0, 2), true) assert.strictEqual(bloomFilterContains(backend.blocks[0].bloom, 0, 3), false) }) it('should insert text characters', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeText', obj: '_root', key: 'text', insert: false, pred: []}, {action: 'set', obj: `1@${actor}`, elemId: '_head', insert: true, value: 'a', pred: []}, {action: 'set', obj: `1@${actor}`, elemId: `2@${actor}`, insert: true, value: 'b', pred: []} ]} const change2 = {actor, seq: 2, startOp: 4, time: 0, deps: [hash(change1)], ops: [ {action: 'set', obj: `1@${actor}`, elemId: `3@${actor}`, insert: true, value: 'c', pred: []}, {action: 'set', obj: `1@${actor}`, elemId: `4@${actor}`, insert: true, value: 'd', pred: []} ]} const backend = new BackendDoc() assert.deepStrictEqual(backend.applyChanges([encodeChange(change1)]), { maxOp: 3, clock: {[actor]: 1}, deps: [hash(change1)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {text: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'text', edits: [ {action: 'multi-insert', index: 0, elemId: `2@${actor}`, values: ['a', 'b']} ] }}}} }) assert.deepStrictEqual(backend.applyChanges([encodeChange(change2)]), { maxOp: 5, clock: {[actor]: 2}, deps: [hash(change2)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {text: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'text', edits: [ {action: 'multi-insert', index: 2, elemId: `4@${actor}`, values: ['c', 'd']} ] }}}} }) checkColumns(backend.blocks[0], { objActor: [0, 1, 4, 0], objCtr: [0, 1, 4, 1], keyActor: [0, 2, 3, 0], keyCtr: [0, 1, 0x7e, 0, 2, 2, 1], // null, 0, 2, 3, 4 keyStr: [0x7f, 4, 0x74, 0x65, 0x78, 0x74, 0, 4], // 'text', 4x null idActor: [5, 0], idCtr: [5, 1], insert: [1, 4], action: [0x7f, 4, 4, 1], // makeText, 4x set valLen: [0x7f, 0, 4, 0x16], // null, 4x 1-byte string valRaw: [0x61, 0x62, 0x63, 0x64], // 'a', 'b', 'c', 'd' succNum: [5, 0], succActor: [], succCtr: [] }) assert.strictEqual(backend.blocks[0].numOps, 5) assert.strictEqual(backend.blocks[0].lastKey, undefined) assert.strictEqual(backend.blocks[0].numVisible, 4) assert.strictEqual(backend.blocks[0].lastObjectActor, 0) assert.strictEqual(backend.blocks[0].lastObjectCtr, 1) assert.strictEqual(backend.blocks[0].firstVisibleActor, 0) assert.strictEqual(backend.blocks[0].firstVisibleCtr, 2) assert.strictEqual(backend.blocks[0].lastVisibleActor, 0) assert.strictEqual(backend.blocks[0].lastVisibleCtr, 5) assert.strictEqual(bloomFilterContains(backend.blocks[0].bloom, 0, 2), true) assert.strictEqual(bloomFilterContains(backend.blocks[0].bloom, 0, 3), true) assert.strictEqual(bloomFilterContains(backend.blocks[0].bloom, 0, 4), true) assert.strictEqual(bloomFilterContains(backend.blocks[0].bloom, 0, 5), true) assert.strictEqual(bloomFilterContains(backend.blocks[0].bloom, 1, 2), false) }) it('should throw an error if the reference element of an insertion does not exist', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeText', obj: '_root', key: 'text', insert: false, pred: []}, {action: 'set', obj: `1@${actor}`, elemId: '_head', insert: true, value: 'a', pred: []}, {action: 'set', obj: `1@${actor}`, elemId: `2@${actor}`, insert: true, value: 'b', pred: []}, {action: 'makeMap', obj: '_root', key: 'map', insert: false, pred: []}, {action: 'set', obj: `4@${actor}`, key: 'foo', insert: false, value: 'c', pred: []} ]} const change2 = {actor, seq: 2, startOp: 6, time: 0, deps: [], ops: [ {action: 'set', obj: `1@${actor}`, elemId: `4@${actor}`, insert: true, value: 'd', pred: []} ]} const backend = new BackendDoc() assert.deepStrictEqual(backend.applyChanges([encodeChange(change1)]), { maxOp: 5, clock: {[actor]: 1}, deps: [hash(change1)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: { text: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'text', edits: [ {action: 'multi-insert', index: 0, elemId: `2@${actor}`, values: ['a', 'b']} ] }}, map: {[`4@${actor}`]: {objectId: `4@${actor}`, type: 'map', props: { foo: {[`5@${actor}`]: {type: 'value', value: 'c'}} }}} }} }) assert.strictEqual(backend.blocks[0].lastObjectActor, 0) assert.strictEqual(backend.blocks[0].lastObjectCtr, 4) assert.throws(() => { backend.applyChanges([encodeChange(change2)]) }, /Reference element not found/) }) it('should handle non-consecutive insertions', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeText', obj: '_root', key: 'text', insert: false, pred: []}, {action: 'set', obj: `1@${actor}`, elemId: '_head', insert: true, value: 'a', pred: []}, {action: 'set', obj: `1@${actor}`, elemId: `2@${actor}`, insert: true, value: 'c', pred: []} ]} const change2 = {actor, seq: 2, startOp: 4, time: 0, deps: [hash(change1)], ops: [ {action: 'set', obj: `1@${actor}`, elemId: `2@${actor}`, insert: true, value: 'b', pred: []}, {action: 'set', obj: `1@${actor}`, elemId: `3@${actor}`, insert: true, value: 'd', pred: []} ]} const backend = new BackendDoc() assert.deepStrictEqual(backend.applyChanges([encodeChange(change1)]), { maxOp: 3, clock: {[actor]: 1}, deps: [hash(change1)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {text: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'text', edits: [ {action: 'multi-insert', index: 0, elemId: `2@${actor}`, values: ['a', 'c']} ] }}}} }) assert.deepStrictEqual(backend.applyChanges([encodeChange(change2)]), { maxOp: 5, clock: {[actor]: 2}, deps: [hash(change2)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {text: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'text', edits: [ {action: 'insert', index: 1, elemId: `4@${actor}`, opId: `4@${actor}`, value: {type: 'value', value: 'b'}}, {action: 'insert', index: 3, elemId: `5@${actor}`, opId: `5@${actor}`, value: {type: 'value', value: 'd'}} ] }}}} }) checkColumns(backend.blocks[0], { objActor: [0, 1, 4, 0], objCtr: [0, 1, 4, 1], keyActor: [0, 2, 3, 0], keyCtr: [0, 1, 0x7c, 0, 2, 0, 1], // null, 0, 2, 2, 3 keyStr: [0x7f, 4, 0x74, 0x65, 0x78, 0x74, 0, 4], // 'text', 4x null idActor: [5, 0], idCtr: [2, 1, 0x7d, 2, 0x7f, 2], // 1, 2, 4, 3, 5 insert: [1, 4], action: [0x7f, 4, 4, 1], // makeText, 4x set valLen: [0x7f, 0, 4, 0x16], // null, 4x 1-byte string valRaw: [0x61, 0x62, 0x63, 0x64], // 'a', 'b', 'c', 'd' succNum: [5, 0], succActor: [], succCtr: [] }) assert.strictEqual(backend.blocks[0].numOps, 5) assert.strictEqual(backend.blocks[0].lastKey, undefined) assert.strictEqual(backend.blocks[0].numVisible, 4) assert.strictEqual(backend.blocks[0].lastObjectActor, 0) assert.strictEqual(backend.blocks[0].lastObjectCtr, 1) assert.strictEqual(backend.blocks[0].firstVisibleActor, 0) assert.strictEqual(backend.blocks[0].firstVisibleCtr, 2) assert.strictEqual(backend.blocks[0].lastVisibleActor, 0) assert.strictEqual(backend.blocks[0].lastVisibleCtr, 5) }) it('should delete the first character', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeText', obj: '_root', key: 'text', insert: false, pred: []}, {action: 'set', obj: `1@${actor}`, elemId: '_head', insert: true, value: 'a', pred: []} ]} const change2 = {actor, seq: 2, startOp: 3, time: 0, deps: [hash(change1)], ops: [ {action: 'del', obj: `1@${actor}`, elemId: `2@${actor}`, pred: [`2@${actor}`]} ]} const backend = new BackendDoc() assert.deepStrictEqual(backend.applyChanges([encodeChange(change1)]), { maxOp: 2, clock: {[actor]: 1}, deps: [hash(change1)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {text: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'text', edits: [ {action: 'insert', index: 0, elemId: `2@${actor}`, opId: `2@${actor}`, value: {type: 'value', value: 'a'}} ] }}}} }) assert.deepStrictEqual(backend.applyChanges([encodeChange(change2)]), { maxOp: 3, clock: {[actor]: 2}, deps: [hash(change2)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {text: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'text', edits: [{action: 'remove', index: 0, count: 1}] }}}} }) checkColumns(backend.blocks[0], { objActor: [0, 1, 0x7f, 0], objCtr: [0, 1, 0x7f, 1], keyActor: [], keyCtr: [0, 1, 0x7f, 0], keyStr: [0x7f, 4, 0x74, 0x65, 0x78, 0x74, 0, 1], // 'text', null idActor: [2, 0], idCtr: [2, 1], insert: [1, 1], action: [0x7e, 4, 1], valLen: [0x7e, 0, 0x16], valRaw: [0x61], succNum: [0x7e, 0, 1], succActor: [0x7f, 0], succCtr: [0x7f, 3] }) assert.strictEqual(backend.blocks[0].numOps, 2) assert.strictEqual(backend.blocks[0].numVisible, 0) assert.strictEqual(backend.blocks[0].lastObjectActor, 0) assert.strictEqual(backend.blocks[0].lastObjectCtr, 1) assert.strictEqual(backend.blocks[0].firstVisibleActor, undefined) assert.strictEqual(backend.blocks[0].firstVisibleCtr, undefined) assert.strictEqual(backend.blocks[0].lastVisibleActor, undefined) assert.strictEqual(backend.blocks[0].lastVisibleCtr, undefined) assert.strictEqual(bloomFilterContains(backend.blocks[0].bloom, 0, 2), true) }) it('should delete a character in the middle', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeText', obj: '_root', key: 'text', insert: false, pred: []}, {action: 'set', obj: `1@${actor}`, elemId: '_head', insert: true, value: 'a', pred: []}, {action: 'set', obj: `1@${actor}`, elemId: `2@${actor}`, insert: true, value: 'b', pred: []}, {action: 'set', obj: `1@${actor}`, elemId: `3@${actor}`, insert: true, value: 'c', pred: []} ]} const change2 = {actor, seq: 2, startOp: 5, time: 0, deps: [hash(change1)], ops: [ {action: 'del', obj: `1@${actor}`, elemId: `3@${actor}`, insert: false, pred: [`3@${actor}`]} ]} const backend = new BackendDoc() assert.deepStrictEqual(backend.applyChanges([encodeChange(change1)]), { maxOp: 4, clock: {[actor]: 1}, deps: [hash(change1)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {text: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'text', edits: [ {action: 'multi-insert', index: 0, elemId: `2@${actor}`, values: ['a', 'b', 'c']} ] }}}} }) assert.deepStrictEqual(backend.applyChanges([encodeChange(change2)]), { maxOp: 5, clock: {[actor]: 2}, deps: [hash(change2)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {text: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'text', edits: [{action: 'remove', index: 1, count: 1}] }}}} }) checkColumns(backend.blocks[0], { objActor: [0, 1, 3, 0], objCtr: [0, 1, 3, 1], keyActor: [0, 2, 2, 0], keyCtr: [0, 1, 0x7d, 0, 2, 1], // null, 0, 2, 3 keyStr: [0x7f, 4, 0x74, 0x65, 0x78, 0x74, 0, 3], // 'text', 3x null idActor: [4, 0], idCtr: [4, 1], insert: [1, 3], action: [0x7f, 4, 3, 1], // makeText, set, set, set valLen: [0x7f, 0, 3, 0x16], // null, 3x 1-byte string valRaw: [0x61, 0x62, 0x63], // 'a', 'b', 'c' succNum: [2, 0, 0x7e, 1, 0], succActor: [0x7f, 0], succCtr: [0x7f, 5] }) assert.strictEqual(backend.blocks[0].numOps, 4) assert.strictEqual(backend.blocks[0].numVisible, 2) assert.strictEqual(backend.blocks[0].lastObjectActor, 0) assert.strictEqual(backend.blocks[0].lastObjectCtr, 1) assert.strictEqual(backend.blocks[0].firstVisibleActor, 0) assert.strictEqual(backend.blocks[0].firstVisibleCtr, 2) assert.strictEqual(backend.blocks[0].lastVisibleActor, 0) assert.strictEqual(backend.blocks[0].lastVisibleCtr, 4) }) it('should throw an error if a deleted element does not exist', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeText', obj: '_root', key: 'text', insert: false, pred: []}, {action: 'set', obj: `1@${actor}`, elemId: '_head', insert: true, value: 'a', pred: []}, {action: 'set', obj: `1@${actor}`, elemId: `2@${actor}`, insert: true, value: 'b', pred: []} ]} const change2 = {actor, seq: 2, startOp: 4, time: 0, deps: [], ops: [ {action: 'del', obj: `1@${actor}`, elemId: `1@${actor}`, insert: false, pred: [`1@${actor}`]} ]} const backend = new BackendDoc() backend.applyChanges([encodeChange(change1)]) assert.throws(() => { backend.applyChanges([encodeChange(change2)]) }, /Reference element not found/) }) it('should apply concurrent insertions at the same position', () => { const actor1 = '01234567', actor2 = '89abcdef' const change1 = {actor: actor1, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeText', obj: '_root', key: 'text', insert: false, pred: []}, {action: 'set', obj: `1@${actor1}`, elemId: '_head', insert: true, value: 'a', pred: []} ]} const change2 = {actor: actor1, seq: 2, startOp: 3, time: 0, deps: [hash(change1)], ops: [ {action: 'set', obj: `1@${actor1}`, elemId: `2@${actor1}`, insert: true, value: 'c', pred: []} ]} const change3 = {actor: actor2, seq: 1, startOp: 3, time: 0, deps: [hash(change1)], ops: [ {action: 'set', obj: `1@${actor1}`, elemId: `2@${actor1}`, insert: true, value: 'b', pred: []} ]} const backend1 = new BackendDoc(), backend2 = new BackendDoc() assert.deepStrictEqual(backend1.applyChanges([encodeChange(change1)]), { maxOp: 2, clock: {[actor1]: 1}, deps: [hash(change1)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {text: {[`1@${actor1}`]: { objectId: `1@${actor1}`, type: 'text', edits: [ {action: 'insert', index: 0, elemId: `2@${actor1}`, opId: `2@${actor1}`, value: {type: 'value', value: 'a'}} ] }}}} }) assert.deepStrictEqual(backend1.applyChanges([encodeChange(change2)]), { maxOp: 3, clock: {[actor1]: 2}, deps: [hash(change2)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {text: {[`1@${actor1}`]: { objectId: `1@${actor1}`, type: 'text', edits: [ {action: 'insert', index: 1, elemId: `3@${actor1}`, opId: `3@${actor1}`, value: {type: 'value', value: 'c'}} ] }}}} }) assert.deepStrictEqual(backend1.applyChanges([encodeChange(change3)]), { maxOp: 3, clock: {[actor1]: 2, [actor2]: 1}, deps: [hash(change2), hash(change3)].sort(), pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {text: {[`1@${actor1}`]: { objectId: `1@${actor1}`, type: 'text', edits: [ {action: 'insert', index: 1, elemId: `3@${actor2}`, opId: `3@${actor2}`, value: {type: 'value', value: 'b'}} ] }}}} }) assert.deepStrictEqual(backend2.applyChanges([encodeChange(change1)]), { maxOp: 2, clock: {[actor1]: 1}, deps: [hash(change1)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {text: {[`1@${actor1}`]: { objectId: `1@${actor1}`, type: 'text', edits: [ {action: 'insert', index: 0, elemId: `2@${actor1}`, opId: `2@${actor1}`, value: {type: 'value', value: 'a'}} ] }}}} }) assert.deepStrictEqual(backend2.applyChanges([encodeChange(change3)]), { maxOp: 3, clock: {[actor1]: 1, [actor2]: 1}, deps: [hash(change3)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {text: {[`1@${actor1}`]: { objectId: `1@${actor1}`, type: 'text', edits: [ {action: 'insert', index: 1, elemId: `3@${actor2}`, opId: `3@${actor2}`, value: {type: 'value', value: 'b'}} ] }}}} }) assert.deepStrictEqual(backend2.applyChanges([encodeChange(change2)]), { maxOp: 3, clock: {[actor1]: 2, [actor2]: 1}, deps: [hash(change2), hash(change3)].sort(), pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {text: {[`1@${actor1}`]: { objectId: `1@${actor1}`, type: 'text', edits: [ {action: 'insert', index: 2, elemId: `3@${actor1}`, opId: `3@${actor1}`, value: {type: 'value', value: 'c'}} ] }}}} }) for (let backend of [backend1, backend2]) { checkColumns(backend.blocks[0], { objActor: [0, 1, 3, 0], objCtr: [0, 1, 3, 1], keyActor: [0, 2, 2, 0], keyCtr: [0, 1, 0x7d, 0, 2, 0], // null, 0, 2, 2 keyStr: [0x7f, 4, 0x74, 0x65, 0x78, 0x74, 0, 3], // 'text', 3x null idActor: [2, 0, 0x7e, 1, 0], // 0, 0, 1, 0 idCtr: [3, 1, 0x7f, 0], // 1, 2, 3, 3 insert: [1, 3], // false, true, true, true action: [0x7f, 4, 3, 1], // makeText, set, set, set valLen: [0x7f, 0, 3, 0x16], // null, 3x 1-byte string valRaw: [0x61, 0x62, 0x63], // 'a', 'b', 'c' succNum: [4, 0], succActor: [], succCtr: [] }) assert.strictEqual(backend.blocks[0].numOps, 4) assert.strictEqual(backend.blocks[0].numVisible, 3) assert.strictEqual(backend.blocks[0].lastObjectActor, 0) assert.strictEqual(backend.blocks[0].lastObjectCtr, 1) assert.strictEqual(backend.blocks[0].firstVisibleActor, 0) assert.strictEqual(backend.blocks[0].firstVisibleCtr, 2) assert.strictEqual(backend.blocks[0].lastVisibleActor, 0) assert.strictEqual(backend.blocks[0].lastVisibleCtr, 3) } }) it('should apply concurrent insertions at the head', () => { const actor1 = '01234567', actor2 = '89abcdef' const change1 = {actor: actor1, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeText', obj: '_root', key: 'text', insert: false, pred: []}, {action: 'set', obj: `1@${actor1}`, elemId: '_head', insert: true, value: 'd', pred: []} ]} const change2 = {actor: actor1, seq: 2, startOp: 3, time: 0, deps: [hash(change1)], ops: [ {action: 'set', obj: `1@${actor1}`, elemId: '_head', insert: true, value: 'c', pred: []} ]} const change3 = {actor: actor2, seq: 1, startOp: 3, time: 0, deps: [hash(change1)], ops: [ {action: 'set', obj: `1@${actor1}`, elemId: '_head', insert: true, value: 'a', pred: []}, {action: 'set', obj: `1@${actor1}`, elemId: `3@${actor2}`, insert: true, value: 'b', pred: []} ]} const backend1 = new BackendDoc(), backend2 = new BackendDoc() assert.deepStrictEqual(backend1.applyChanges([encodeChange(change1)]), { maxOp: 2, clock: {[actor1]: 1}, deps: [hash(change1)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {text: {[`1@${actor1}`]: { objectId: `1@${actor1}`, type: 'text', edits: [ {action: 'insert', index: 0, elemId: `2@${actor1}`, opId: `2@${actor1}`, value: {type: 'value', value: 'd'}} ] }}}} }) assert.deepStrictEqual(backend1.applyChanges([encodeChange(change2)]), { maxOp: 3, clock: {[actor1]: 2}, deps: [hash(change2)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {text: {[`1@${actor1}`]: { objectId: `1@${actor1}`, type: 'text', edits: [ {action: 'insert', index: 0, elemId: `3@${actor1}`, opId: `3@${actor1}`, value: {type: 'value', value: 'c'}} ] }}}} }) assert.deepStrictEqual(backend1.applyChanges([encodeChange(change3)]), { maxOp: 4, clock: {[actor1]: 2, [actor2]: 1}, deps: [hash(change2), hash(change3)].sort(), pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {text: {[`1@${actor1}`]: { objectId: `1@${actor1}`, type: 'text', edits: [ {action: 'multi-insert', index: 0, elemId: `3@${actor2}`, values: ['a', 'b']} ] }}}} }) assert.deepStrictEqual(backend2.applyChanges([encodeChange(change1)]), { maxOp: 2, clock: {[actor1]: 1}, deps: [hash(change1)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {text: {[`1@${actor1}`]: { objectId: `1@${actor1}`, type: 'text', edits: [ {action: 'insert', index: 0, elemId: `2@${actor1}`, opId: `2@${actor1}`, value: {type: 'value', value: 'd'}} ] }}}} }) assert.deepStrictEqual(backend2.applyChanges([encodeChange(change3)]), { maxOp: 4, clock: {[actor1]: 1, [actor2]: 1}, deps: [hash(change3)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {text: {[`1@${actor1}`]: { objectId: `1@${actor1}`, type: 'text', edits: [ {action: 'multi-insert', index: 0, elemId: `3@${actor2}`, values: ['a', 'b']} ] }}}} }) assert.deepStrictEqual(backend2.applyChanges([encodeChange(change2)]), { maxOp: 4, clock: {[actor1]: 2, [actor2]: 1}, deps: [hash(change2), hash(change3)].sort(), pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {text: {[`1@${actor1}`]: { objectId: `1@${actor1}`, type: 'text', edits: [ {action: 'insert', index: 2, elemId: `3@${actor1}`, opId: `3@${actor1}`, value: {type: 'value', value: 'c'}} ] }}}} }) for (let backend of [backend1, backend2]) { checkColumns(backend.blocks[0], { objActor: [0, 1, 4, 0], objCtr: [0, 1, 4, 1], keyActor: [0, 2, 0x7f, 1, 0, 2], // null, null, 1, null, null keyCtr: [0, 1, 0x7c, 0, 3, 0x7d, 0], // null, 0, 3, 0, 0 keyStr: [0x7f, 4, 0x74, 0x65, 0x78, 0x74, 0, 4], // 'text', 4x null idActor: [0x7f, 0, 2, 1, 2, 0], // 0, 1, 1, 0, 0 idCtr: [0x7d, 1, 2, 1, 2, 0x7f], // 1, 3, 4, 3, 2 insert: [1, 4], // false, true, true, true, true action: [0x7f, 4, 4, 1], // makeText, set, set, set, set valLen: [0x7f, 0, 4, 0x16], // null, 4x 1-byte string valRaw: [0x61, 0x62, 0x63, 0x64], // 'a', 'b', 'c', 'd' succNum: [5, 0], succActor: [], succCtr: [] }) assert.strictEqual(backend.blocks[0].numOps, 5) assert.strictEqual(backend.blocks[0].numVisible, 4) assert.strictEqual(backend.blocks[0].lastObjectActor, 0) assert.strictEqual(backend.blocks[0].lastObjectCtr, 1) // firstVisible is incorrect -- it should strictly be (1,3) rather than (0,2) -- but that // doesn't matter since in any case it'll be different from the previous block's lastVisible assert.strictEqual(backend.blocks[0].firstVisibleActor, 0) assert.strictEqual(backend.blocks[0].firstVisibleCtr, 2) assert.strictEqual(backend.blocks[0].lastVisibleActor, 0) assert.strictEqual(backend.blocks[0].lastVisibleCtr, 2) assert.strictEqual(bloomFilterContains(backend.blocks[0].bloom, 0, 2), true) assert.strictEqual(bloomFilterContains(backend.blocks[0].bloom, 0, 3), true) assert.strictEqual(bloomFilterContains(backend.blocks[0].bloom, 1, 3), true) assert.strictEqual(bloomFilterContains(backend.blocks[0].bloom, 1, 4), true) // The chance of a false positive is extremely low since the filter only contains 4 elements for (let i = 5; i < 100; i++) assert.strictEqual(bloomFilterContains(backend.blocks[0].bloom, 1, i), false) } }) it('should perform multiple list element updates', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeText', obj: '_root', key: 'text', insert: false, pred: []}, {action: 'set', obj: `1@${actor}`, elemId: '_head', insert: true, value: 'a', pred: []}, {action: 'set', obj: `1@${actor}`, elemId: `2@${actor}`, insert: true, value: 'b', pred: []}, {action: 'set', obj: `1@${actor}`, elemId: `3@${actor}`, insert: true, value: 'c', pred: []} ]} const change2 = {actor, seq: 2, startOp: 5, time: 0, deps: [hash(change1)], ops: [ {action: 'set', obj: `1@${actor}`, elemId: `2@${actor}`, insert: false, value: 'A', pred: [`2@${actor}`]}, {action: 'set', obj: `1@${actor}`, elemId: `4@${actor}`, insert: false, value: 'C', pred: [`4@${actor}`]} ]} const backend = new BackendDoc() assert.deepStrictEqual(backend.applyChanges([encodeChange(change1)]), { maxOp: 4, clock: {[actor]: 1}, deps: [hash(change1)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {text: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'text', edits: [ {action: 'multi-insert', index: 0, elemId: `2@${actor}`, values: ['a', 'b', 'c']} ] }}}} }) assert.deepStrictEqual(backend.applyChanges([encodeChange(change2)]), { maxOp: 6, clock: {[actor]: 2}, deps: [hash(change2)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {text: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'text', edits: [ {action: 'update', index: 0, opId: `5@${actor}`, value: {type: 'value', value: 'A'}}, {action: 'update', index: 2, opId: `6@${actor}`, value: {type: 'value', value: 'C'}} ] }}}} }) checkColumns(backend.blocks[0], { objActor: [0, 1, 5, 0], objCtr: [0, 1, 5, 1], keyActor: [0, 2, 4, 0], keyCtr: [0, 1, 0x7d, 0, 2, 0, 2, 1], // null, 0, 2, 2, 3, 4 keyStr: [0x7f, 4, 0x74, 0x65, 0x78, 0x74, 0, 5], // 'text', 5x null idActor: [6, 0], idCtr: [2, 1, 0x7c, 3, 0x7e, 1, 2], // 1, 2, 5, 3, 4, 6 insert: [1, 1, 1, 2, 1], // false, true, false, true, true, false action: [0x7f, 4, 5, 1], // makeText, 5x set valLen: [0x7f, 0, 5, 0x16], // null, 5x 1-byte string valRaw: [0x61, 0x41, 0x62, 0x63, 0x43], // 'a', 'A', 'b', 'c', 'C' succNum: [0x7e, 0, 1, 2, 0, 0x7e, 1, 0], // 0, 1, 0, 0, 1, 0 succActor: [2, 0], succCtr: [0x7e, 5, 1] // 5, 6 }) assert.strictEqual(backend.blocks[0].numOps, 6) assert.strictEqual(backend.blocks[0].numVisible, 3) assert.strictEqual(backend.blocks[0].lastObjectActor, 0) assert.strictEqual(backend.blocks[0].lastObjectCtr, 1) assert.strictEqual(backend.blocks[0].firstVisibleActor, 0) assert.strictEqual(backend.blocks[0].firstVisibleCtr, 2) assert.strictEqual(backend.blocks[0].lastVisibleActor, 0) assert.strictEqual(backend.blocks[0].lastVisibleCtr, 4) }) it('should allow list element updates in reverse order', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeText', obj: '_root', key: 'text', insert: false, pred: []}, {action: 'set', obj: `1@${actor}`, elemId: '_head', insert: true, value: 'a', pred: []}, {action: 'set', obj: `1@${actor}`, elemId: `2@${actor}`, insert: true, value: 'b', pred: []}, {action: 'set', obj: `1@${actor}`, elemId: `3@${actor}`, insert: true, value: 'c', pred: []} ]} const change2 = {actor, seq: 2, startOp: 5, time: 0, deps: [hash(change1)], ops: [ {action: 'set', obj: `1@${actor}`, elemId: `4@${actor}`, insert: false, value: 'C', pred: [`4@${actor}`]}, {action: 'set', obj: `1@${actor}`, elemId: `2@${actor}`, insert: false, value: 'A', pred: [`2@${actor}`]} ]} const backend = new BackendDoc() backend.applyChanges([encodeChange(change1)]) assert.deepStrictEqual(backend.applyChanges([encodeChange(change2)]), { maxOp: 6, clock: {[actor]: 2}, deps: [hash(change2)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {text: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'text', edits: [ {action: 'update', index: 2, opId: `5@${actor}`, value: {type: 'value', value: 'C'}}, {action: 'update', index: 0, opId: `6@${actor}`, value: {type: 'value', value: 'A'}} ] }}}} }) checkColumns(backend.blocks[0], { objActor: [0, 1, 5, 0], objCtr: [0, 1, 5, 1], keyActor: [0, 2, 4, 0], // null, null, 0, 0, 0, 0 keyCtr: [0, 1, 0x7d, 0, 2, 0, 2, 1], // null, 0, 2, 2, 3, 4 keyStr: [0x7f, 4, 0x74, 0x65, 0x78, 0x74, 0, 5], // 'text', 5x null idActor: [6, 0], idCtr: [2, 1, 0x7e, 4, 0x7d, 2, 1], // 1, 2, 6, 3, 4, 5 insert: [1, 1, 1, 2, 1], // false, true, false, true, true, false action: [0x7f, 4, 5, 1], // makeText, 5x set valLen: [0x7f, 0, 5, 0x16], // null, 5x 1-byte string valRaw: [0x61, 0x41, 0x62, 0x63, 0x43], // 'a', 'A', 'b', 'c', 'C' succNum: [0x7e, 0, 1, 2, 0, 0x7e, 1, 0], // 0, 1, 0, 0, 1, 0 succActor: [2, 0], succCtr: [0x7e, 6, 0x7f] // 6, 5 }) assert.strictEqual(backend.blocks[0].numOps, 6) assert.strictEqual(backend.blocks[0].numVisible, 3) assert.strictEqual(backend.blocks[0].lastObjectActor, 0) assert.strictEqual(backend.blocks[0].lastObjectCtr, 1) assert.strictEqual(backend.blocks[0].firstVisibleActor, 0) assert.strictEqual(backend.blocks[0].firstVisibleCtr, 2) assert.strictEqual(backend.blocks[0].lastVisibleActor, 0) assert.strictEqual(backend.blocks[0].lastVisibleCtr, 4) }) it('should handle nested objects inside list elements', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeList', obj: '_root', key: 'list', insert: false, pred: []}, {action: 'set', obj: `1@${actor}`, elemId: '_head', insert: true, datatype: 'uint', value: 1, pred: []}, {action: 'makeMap', obj: `1@${actor}`, elemId: `2@${actor}`, insert: true, pred: []} ]} const change2 = {actor, seq: 2, startOp: 4, time: 0, deps: [hash(change1)], ops: [ {action: 'set', obj: `3@${actor}`, key: 'x', insert: false, datatype: 'uint', value: 2, pred: []} ]} const backend = new BackendDoc() assert.deepStrictEqual(backend.applyChanges([encodeChange(change1)]), { maxOp: 3, clock: {[actor]: 1}, deps: [hash(change1)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {list: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'list', edits: [ {action: 'insert', index: 0, elemId: `2@${actor}`, opId: `2@${actor}`, value: { type: 'value', value: 1, datatype: 'uint' }}, {action: 'insert', index: 1, elemId: `3@${actor}`, opId: `3@${actor}`, value: { objectId: `3@${actor}`, type: 'map', props: {} }} ] }}}} }) assert.deepStrictEqual(backend.applyChanges([encodeChange(change2)]), { maxOp: 4, clock: {[actor]: 2}, deps: [hash(change2)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {list: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'list', edits: [ {action: 'update', index: 1, opId: `3@${actor}`, value: { objectId: `3@${actor}`, type: 'map', props: {x: {[`4@${actor}`]: { type: 'value', value: 2, datatype: 'uint' }}} }} ] }}}} }) checkColumns(backend.blocks[0], { objActor: [0, 1, 3, 0], objCtr: [0, 1, 2, 1, 0x7f, 3], // null, 1, 1, 3 keyActor: [0, 2, 0x7f, 0, 0, 1], // null, null, 0, null keyCtr: [0, 1, 0x7e, 0, 2, 0, 1], // null, 0, 2, null keyStr: [0x7f, 4, 0x6c, 0x69, 0x73, 0x74, 0, 2, 0x7f, 1, 0x78], // 'list', null, null, 'x' idActor: [4, 0], idCtr: [4, 1], insert: [1, 2, 1], // false, true, true, false action: [0x7c, 2, 1, 0, 1], // makeList, set, makeMap, set valLen: [0x7c, 0, 0x13, 0, 0x13], // null, uint, null, uint valRaw: [1, 2], succNum: [4, 0], succActor: [], succCtr: [] }) assert.strictEqual(backend.blocks[0].numOps, 4) assert.strictEqual(backend.blocks[0].lastKey, 'x') assert.strictEqual(backend.blocks[0].numVisible, 0) assert.strictEqual(backend.blocks[0].lastObjectActor, 0) assert.strictEqual(backend.blocks[0].lastObjectCtr, 3) assert.strictEqual(backend.blocks[0].firstVisibleActor, 0) assert.strictEqual(backend.blocks[0].firstVisibleCtr, 2) assert.strictEqual(backend.blocks[0].lastVisibleActor, undefined) assert.strictEqual(backend.blocks[0].lastVisibleCtr, undefined) }) it('should handle multiple list objects', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeList', obj: '_root', key: 'list1', insert: false, pred: []}, {action: 'set', obj: `1@${actor}`, elemId: '_head', insert: true, datatype: 'uint', value: 1, pred: []}, {action: 'makeList', obj: '_root', key: 'list2', insert: false, pred: []}, {action: 'set', obj: `3@${actor}`, elemId: '_head', insert: true, datatype: 'uint', value: 2, pred: []} ]} const change2 = {actor, seq: 2, startOp: 5, time: 0, deps: [hash(change1)], ops: [ {action: 'set', obj: `1@${actor}`, elemId: `2@${actor}`, insert: true, datatype: 'uint', value: 3, pred: []} ]} const backend = new BackendDoc() assert.deepStrictEqual(backend.applyChanges([encodeChange(change1)]), { maxOp: 4, clock: {[actor]: 1}, deps: [hash(change1)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: { list1: {[`1@${actor}`]: {objectId: `1@${actor}`, type: 'list', edits: [ {action: 'insert', index: 0, elemId: `2@${actor}`, opId: `2@${actor}`, value: { type: 'value', value: 1, datatype: 'uint' }} ]}}, list2: {[`3@${actor}`]: {objectId: `3@${actor}`, type: 'list', edits: [ {action: 'insert', index: 0, elemId: `4@${actor}`, opId: `4@${actor}`, value: { type: 'value', value: 2, datatype: 'uint' }} ]}} }} }) assert.deepStrictEqual(backend.applyChanges([encodeChange(change2)]), { maxOp: 5, clock: {[actor]: 2}, deps: [hash(change2)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: { list1: {[`1@${actor}`]: {objectId: `1@${actor}`, type: 'list', edits: [ {action: 'insert', index: 1, elemId: `5@${actor}`, opId: `5@${actor}`, value: { type: 'value', value: 3, datatype: 'uint' }} ]}} }} }) checkColumns(backend.blocks[0], { objActor: [0, 2, 3, 0], objCtr: [0, 2, 2, 1, 0x7f, 3], // null, null, 1, 1, 3 keyActor: [0, 3, 0x7f, 0, 0, 1], // null, null, null, 0, null keyCtr: [0, 2, 0x7d, 0, 2, 0x7e], // null, null, 0, 2, 0 keyStr: [0x7e, 5, 0x6c, 0x69, 0x73, 0x74, 0x31, 5, 0x6c, 0x69, 0x73, 0x74, 0x32, 0, 3], // 'list1', 'list2', null, null, null idActor: [5, 0], idCtr: [0x7b, 1, 2, 0x7f, 3, 0x7f], // 1, 3, 2, 5, 4 insert: [2, 3], // false, false, true, true, true action: [2, 2, 3, 1], // 2x makeList, 3x set valLen: [2, 0, 3, 0x13], // 2x null, 3x uint valRaw: [1, 3, 2], succNum: [5, 0], succActor: [], succCtr: [] }) assert.strictEqual(backend.blocks[0].numOps, 5) assert.strictEqual(backend.blocks[0].lastKey, undefined) assert.strictEqual(backend.blocks[0].numVisible, 1) assert.strictEqual(backend.blocks[0].lastObjectActor, 0) assert.strictEqual(backend.blocks[0].lastObjectCtr, 3) assert.strictEqual(backend.blocks[0].firstVisibleActor, 0) assert.strictEqual(backend.blocks[0].firstVisibleCtr, 2) assert.strictEqual(backend.blocks[0].lastVisibleActor, 0) assert.strictEqual(backend.blocks[0].lastVisibleCtr, 4) }) it('should handle a counter inside a map', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'set', obj: '_root', key: 'counter', value: 1, datatype: 'counter', pred: []} ]} const change2 = {actor, seq: 2, startOp: 2, time: 0, deps: [hash(change1)], ops: [ {action: 'inc', obj: '_root', key: 'counter', datatype: 'uint', value: 2, pred: [`1@${actor}`]} ]} const change3 = {actor, seq: 3, startOp: 3, time: 0, deps: [hash(change2)], ops: [ {action: 'inc', obj: '_root', key: 'counter', datatype: 'uint', value: 3, pred: [`1@${actor}`]} ]} const backend = new BackendDoc() assert.deepStrictEqual(backend.applyChanges([encodeChange(change1)]), { maxOp: 1, clock: {[actor]: 1}, deps: [hash(change1)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: { counter: {[`1@${actor}`]: {type: 'value', value: 1, datatype: 'counter'}} }} }) assert.deepStrictEqual(backend.applyChanges([encodeChange(change2)]), { maxOp: 2, clock: {[actor]: 2}, deps: [hash(change2)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: { counter: {[`1@${actor}`]: {type: 'value', value: 3, datatype: 'counter'}} }} }) assert.deepStrictEqual(backend.applyChanges([encodeChange(change3)]), { maxOp: 3, clock: {[actor]: 3}, deps: [hash(change3)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: { counter: {[`1@${actor}`]: {type: 'value', value: 6, datatype: 'counter'}} }} }) checkColumns(backend.blocks[0], { objActor: [], objCtr: [], keyActor: [], keyCtr: [], keyStr: [3, 7, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72], // 3x 'counter' idActor: [3, 0], idCtr: [3, 1], insert: [3], action: [0x7f, 1, 2, 5], // set, inc, inc valLen: [0x7f, 0x18, 2, 0x13], // counter, uint, uint valRaw: [1, 2, 3], succNum: [0x7f, 2, 2, 0], // 2, 0, 0 succActor: [2, 0], succCtr: [0x7e, 2, 1] // 2, 3 }) assert.strictEqual(backend.blocks[0].lastKey, 'counter') assert.strictEqual(backend.blocks[0].numOps, 3) assert.strictEqual(backend.blocks[0].lastObjectActor, null) assert.strictEqual(backend.blocks[0].lastObjectCtr, null) }) it('should handle a counter inside a list element', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeList', obj: '_root', key: 'list', insert: false, pred: []}, {action: 'set', obj: `1@${actor}`, elemId: '_head', insert: true, pred: [], value: 1, datatype: 'counter'} ]} const change2 = {actor, seq: 2, startOp: 3, time: 0, deps: [hash(change1)], ops: [ {action: 'inc', obj: `1@${actor}`, elemId: `2@${actor}`, datatype: 'uint', value: 2, pred: [`2@${actor}`]} ]} const backend = new BackendDoc() assert.deepStrictEqual(backend.applyChanges([encodeChange(change1)]), { maxOp: 2, clock: {[actor]: 1}, deps: [hash(change1)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {list: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'list', edits: [ {action: 'insert', index: 0, elemId: `2@${actor}`, opId: `2@${actor}`, value: { type: 'value', value: 1, datatype: 'counter' }} ] }}}} }) assert.deepStrictEqual(backend.applyChanges([encodeChange(change2)]), { maxOp: 3, clock: {[actor]: 2}, deps: [hash(change2)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {list: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'list', edits: [ {action: 'update', index: 0, opId: `2@${actor}`, value: { type: 'value', value: 3, datatype: 'counter' }} ] }}}} }) checkColumns(backend.blocks[0], { objActor: [0, 1, 2, 0], objCtr: [0, 1, 2, 1], keyActor: [0, 2, 0x7f, 0], // null, null, 0 keyCtr: [0, 1, 0x7e, 0, 2], // null, 0, 2 keyStr: [0x7f, 4, 0x6c, 0x69, 0x73, 0x74, 0, 2], // 'list', null, null idActor: [3, 0], idCtr: [3, 1], // 1, 2, 3 insert: [1, 1, 1], // false, true, false action: [0x7d, 2, 1, 5], // makeList, set, inc valLen: [0x7d, 0, 0x18, 0x13], // null, counter, uint valRaw: [1, 2], succNum: [0x7d, 0, 1, 0], succActor: [0x7f, 0], succCtr: [0x7f, 3] }) assert.strictEqual(backend.blocks[0].numOps, 3) assert.strictEqual(backend.blocks[0].lastKey, undefined) assert.strictEqual(backend.blocks[0].numVisible, 1) assert.strictEqual(backend.blocks[0].lastObjectActor, 0) assert.strictEqual(backend.blocks[0].lastObjectCtr, 1) assert.strictEqual(backend.blocks[0].firstVisibleActor, 0) assert.strictEqual(backend.blocks[0].firstVisibleCtr, 2) assert.strictEqual(backend.blocks[0].lastVisibleActor, 0) assert.strictEqual(backend.blocks[0].lastVisibleCtr, 2) }) it('should delete a counter from a map', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'set', obj: '_root', key: 'counter', value: 1, datatype: 'counter', pred: []} ]} const change2 = {actor, seq: 2, startOp: 2, time: 0, deps: [hash(change1)], ops: [ {action: 'inc', obj: '_root', key: 'counter', value: 2, datatype: 'uint', pred: [`1@${actor}`]} ]} const change3 = {actor, seq: 3, startOp: 3, time: 0, deps: [hash(change2)], ops: [ {action: 'del', obj: '_root', key: 'counter', pred: [`1@${actor}`]} ]} const backend = new BackendDoc() backend.applyChanges([encodeChange(change1)]) assert.deepStrictEqual(backend.applyChanges([encodeChange(change2)]), { maxOp: 2, clock: {[actor]: 2}, deps: [hash(change2)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: { counter: {[`1@${actor}`]: {type: 'value', value: 3, datatype: 'counter'}} }} }) assert.deepStrictEqual(backend.applyChanges([encodeChange(change3)]), { maxOp: 3, clock: {[actor]: 3}, deps: [hash(change3)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {counter: {}}} }) assert.strictEqual(backend.blocks[0].lastKey, 'counter') assert.strictEqual(backend.blocks[0].numOps, 2) assert.strictEqual(backend.blocks[0].lastObjectActor, null) assert.strictEqual(backend.blocks[0].lastObjectCtr, null) }) it('should handle conflicts inside list elements', () => { const actor1 = '01234567', actor2 = '89abcdef' const change1 = {actor: actor1, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeList', obj: '_root', key: 'list', insert: false, pred: []}, {action: 'set', obj: `1@${actor1}`, elemId: '_head', insert: true, datatype: 'uint', value: 1, pred: []} ]} const change2 = {actor: actor1, seq: 2, startOp: 3, time: 0, deps: [hash(change1)], ops: [ {action: 'set', obj: `1@${actor1}`, elemId: `2@${actor1}`, insert: false, datatype: 'uint', value: 2, pred: [`2@${actor1}`]} ]} const change3 = {actor: actor2, seq: 1, startOp: 3, time: 0, deps: [hash(change1)], ops: [ {action: 'set', obj: `1@${actor1}`, elemId: `2@${actor1}`, insert: false, datatype: 'uint', value: 3, pred: [`2@${actor1}`]} ]} const backend1 = new BackendDoc(), backend2 = new BackendDoc() assert.deepStrictEqual(backend1.applyChanges([encodeChange(change1)]), { maxOp: 2, clock: {[actor1]: 1}, deps: [hash(change1)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {list: {[`1@${actor1}`]: { objectId: `1@${actor1}`, type: 'list', edits: [ {action: 'insert', index: 0, elemId: `2@${actor1}`, opId: `2@${actor1}`, value: { type: 'value', value: 1, datatype: 'uint' }} ] }}}} }) assert.deepStrictEqual(backend1.applyChanges([encodeChange(change2)]), { maxOp: 3, clock: {[actor1]: 2}, deps: [hash(change2)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {list: {[`1@${actor1}`]: { objectId: `1@${actor1}`, type: 'list', edits: [ {action: 'update', index: 0, opId: `3@${actor1}`, value: {type: 'value', value: 2, datatype: 'uint'}} ] }}}} }) assert.deepStrictEqual(backend1.applyChanges([encodeChange(change3)]), { maxOp: 3, clock: {[actor1]: 2, [actor2]: 1}, deps: [hash(change2), hash(change3)].sort(), pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {list: {[`1@${actor1}`]: { objectId: `1@${actor1}`, type: 'list', edits: [ {action: 'update', index: 0, opId: `3@${actor1}`, value: {type: 'value', value: 2, datatype: 'uint'}}, {action: 'update', index: 0, opId: `3@${actor2}`, value: {type: 'value', value: 3, datatype: 'uint'}} ] }}}} }) backend2.applyChanges([encodeChange(change1)]) assert.deepStrictEqual(backend2.applyChanges([encodeChange(change3)]), { maxOp: 3, clock: {[actor1]: 1, [actor2]: 1}, deps: [hash(change3)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {list: {[`1@${actor1}`]: { objectId: `1@${actor1}`, type: 'list', edits: [ {action: 'update', index: 0, opId: `3@${actor2}`, value: {type: 'value', value: 3, datatype: 'uint'}} ] }}}} }) assert.deepStrictEqual(backend2.applyChanges([encodeChange(change2)]), { maxOp: 3, clock: {[actor1]: 2, [actor2]: 1}, deps: [hash(change2), hash(change3)].sort(), pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {list: {[`1@${actor1}`]: { objectId: `1@${actor1}`, type: 'list', edits: [ {action: 'update', index: 0, opId: `3@${actor1}`, value: {type: 'value', value: 2, datatype: 'uint'}}, {action: 'update', index: 0, opId: `3@${actor2}`, value: {type: 'value', value: 3, datatype: 'uint'}} ] }}}} }) for (let backend of [backend1, backend2]) { checkColumns(backend.blocks[0], { objActor: [0, 1, 3, 0], objCtr: [0, 1, 3, 1], keyActor: [0, 2, 2, 0], keyCtr: [0, 1, 0x7d, 0, 2, 0], // null, 0, 2, 2 keyStr: [0x7f, 4, 0x6c, 0x69, 0x73, 0x74, 0, 3], // 'list', 3x null idActor: [3, 0, 0x7f, 1], idCtr: [3, 1, 0x7f, 0], // 1, 2, 3, 3 insert: [1, 1, 2], // false, true, false, false action: [0x7f, 2, 3, 1], // makeList, 3x set valLen: [0x7f, 0, 3, 0x13], // null, 3x uint valRaw: [1, 2, 3], succNum: [0x7e, 0, 2, 2, 0], // 0, 1, 0, 0 succActor: [0x7e, 0, 1], succCtr: [0x7e, 3, 0] // 3, 3 }) assert.strictEqual(backend.blocks[0].numOps, 4) assert.strictEqual(backend.blocks[0].lastKey, undefined) assert.strictEqual(backend.blocks[0].numVisible, 1) assert.strictEqual(backend.blocks[0].lastObjectActor, 0) assert.strictEqual(backend.blocks[0].lastObjectCtr, 1) assert.strictEqual(backend.blocks[0].firstVisibleActor, 0) assert.strictEqual(backend.blocks[0].firstVisibleCtr, 2) assert.strictEqual(backend.blocks[0].lastVisibleActor, 0) assert.strictEqual(backend.blocks[0].lastVisibleCtr, 2) } }) it('should allow conflicts to be introduced by a single change', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeText', obj: '_root', key: 'text', insert: false, pred: []}, {action: 'set', obj: `1@${actor}`, elemId: '_head', insert: true, value: 'a', pred: []}, {action: 'set', obj: `1@${actor}`, elemId: `2@${actor}`, insert: true, value: 'b', pred: []} ]} const change2 = {actor, seq: 2, startOp: 4, time: 0, deps: [hash(change1)], ops: [ {action: 'set', obj: `1@${actor}`, elemId: `2@${actor}`, insert: false, value: 'x', pred: [`2@${actor}`]}, {action: 'set', obj: `1@${actor}`, elemId: `2@${actor}`, insert: false, value: 'y', pred: [`2@${actor}`]} ]} const backend = new BackendDoc() assert.deepStrictEqual(backend.applyChanges([encodeChange(change1)]), { maxOp: 3, clock: {[actor]: 1}, deps: [hash(change1)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {text: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'text', edits: [ {action: 'multi-insert', index: 0, elemId: `2@${actor}`, values: ['a', 'b']} ] }}}} }) assert.deepStrictEqual(backend.applyChanges([encodeChange(change2)]), { maxOp: 5, clock: {[actor]: 2}, deps: [hash(change2)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {text: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'text', edits: [ {action: 'update', index: 0, opId: `4@${actor}`, value: {type: 'value', value: 'x'}}, {action: 'update', index: 0, opId: `5@${actor}`, value: {type: 'value', value: 'y'}} ] }}}} }) checkColumns(backend.blocks[0], { objActor: [0, 1, 4, 0], objCtr: [0, 1, 4, 1], keyActor: [0, 2, 3, 0], keyCtr: [0, 1, 0x7e, 0, 2, 2, 0], // null, 0, 2, 2, 2 keyStr: [0x7f, 4, 0x74, 0x65, 0x78, 0x74, 0, 4], // 'text', 4x null idActor: [5, 0], idCtr: [2, 1, 0x7d, 2, 1, 0x7e], // 1, 2, 4, 5, 3 insert: [1, 1, 2, 1], // false, true, false, false, true action: [0x7f, 4, 4, 1], // makeText, 4x set valLen: [0x7f, 0, 4, 0x16], // null, 4x 1-byte string valRaw: [0x61, 0x78, 0x79, 0x62], // 'a', 'x', 'y', 'b' succNum: [0x7e, 0, 2, 3, 0], // 0, 2, 0, 0, 0 succActor: [2, 0], succCtr: [0x7e, 4, 1] // 4, 5 }) assert.strictEqual(backend.blocks[0].numOps, 5) assert.strictEqual(backend.blocks[0].lastKey, undefined) assert.strictEqual(backend.blocks[0].numVisible, 2) assert.strictEqual(backend.blocks[0].lastObjectActor, 0) assert.strictEqual(backend.blocks[0].lastObjectCtr, 1) assert.strictEqual(backend.blocks[0].firstVisibleActor, 0) assert.strictEqual(backend.blocks[0].firstVisibleCtr, 2) assert.strictEqual(backend.blocks[0].lastVisibleActor, 0) assert.strictEqual(backend.blocks[0].lastVisibleCtr, 3) }) it('should allow conflicts to arise on a multi-inserted element', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeText', obj: '_root', key: 'text', insert: false, pred: []}, {action: 'set', obj: `1@${actor}`, elemId: '_head', insert: true, value: 'a', pred: []}, {action: 'set', obj: `1@${actor}`, elemId: `2@${actor}`, insert: true, value: 'b', pred: []} ]} const change2 = {actor, seq: 2, startOp: 4, time: 0, deps: [hash(change1)], ops: [ {action: 'set', obj: `1@${actor}`, elemId: `3@${actor}`, insert: false, value: 'x', pred: [`3@${actor}`]}, {action: 'set', obj: `1@${actor}`, elemId: `3@${actor}`, insert: false, value: 'y', pred: [`3@${actor}`]} ]} const backend = new BackendDoc() assert.deepStrictEqual(backend.applyChanges([encodeChange(change1), encodeChange(change2)]), { maxOp: 5, clock: {[actor]: 2}, deps: [hash(change2)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {text: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'text', edits: [ {action: 'multi-insert', index: 0, elemId: `2@${actor}`, values: ['a']}, {action: 'insert', index: 1, elemId: `3@${actor}`, opId: `4@${actor}`, value: {type: 'value', value: 'x'}}, {action: 'update', index: 1, opId: `5@${actor}`, value: {type: 'value', value: 'y'}} ] }}}} }) checkColumns(backend.blocks[0], { objActor: [0, 1, 4, 0], objCtr: [0, 1, 4, 1], keyActor: [0, 2, 3, 0], keyCtr: [0, 1, 0x7c, 0, 2, 1, 0], // null, 0, 2, 3, 3 keyStr: [0x7f, 4, 0x74, 0x65, 0x78, 0x74, 0, 4], // 'text', 4x null idActor: [5, 0], idCtr: [5, 1], // 1, 2, 3, 4, 5 insert: [1, 2, 2], // false, true, true, false, false action: [0x7f, 4, 4, 1], // makeText, 4x set valLen: [0x7f, 0, 4, 0x16], // null, 4x 1-byte string valRaw: [0x61, 0x62, 0x78, 0x79], // 'a', 'b', 'x', 'y' succNum: [2, 0, 0x7f, 2, 2, 0], // 0, 0, 2, 0, 0 succActor: [2, 0], succCtr: [0x7e, 4, 1] // 4, 5 }) assert.strictEqual(backend.blocks[0].numOps, 5) assert.strictEqual(backend.blocks[0].lastKey, undefined) assert.strictEqual(backend.blocks[0].numVisible, 2) assert.strictEqual(backend.blocks[0].lastObjectActor, 0) assert.strictEqual(backend.blocks[0].lastObjectCtr, 1) assert.strictEqual(backend.blocks[0].firstVisibleActor, 0) assert.strictEqual(backend.blocks[0].firstVisibleCtr, 2) assert.strictEqual(backend.blocks[0].lastVisibleActor, 0) assert.strictEqual(backend.blocks[0].lastVisibleCtr, 3) }) it('should convert inserts to updates when needed', () => { const actor1 = '01234567', actor2 = '89abcdef' const change1 = {actor: actor1, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeText', obj: '_root', key: 'text', insert: false, pred: []}, {action: 'set', obj: `1@${actor1}`, elemId: '_head', insert: true, value: 'c', pred: []} ]} const change2 = {actor: actor1, seq: 2, startOp: 3, time: 0, deps: [hash(change1)], ops: [ {action: 'set', obj: `1@${actor1}`, elemId: '_head', insert: true, value: 'a', pred: []}, {action: 'set', obj: `1@${actor1}`, elemId: `3@${actor1}`, insert: true, value: 'b', pred: []}, {action: 'set', obj: `1@${actor1}`, elemId: `2@${actor1}`, insert: false, value: 'C', pred: [`2@${actor1}`]} ]} const change3 = {actor: actor2, seq: 1, startOp: 3, time: 0, deps: [hash(change1)], ops: [ {action: 'set', obj: `1@${actor1}`, elemId: `2@${actor1}`, insert: false, value: 'x', pred: [`2@${actor1}`]}, {action: 'set', obj: `1@${actor1}`, elemId: `2@${actor1}`, insert: false, value: 'y', pred: [`2@${actor1}`]} ]} const backend = new BackendDoc() assert.deepStrictEqual(backend.applyChanges([encodeChange(change1), encodeChange(change2)]), { maxOp: 5, clock: {[actor1]: 2}, deps: [hash(change2)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {text: {[`1@${actor1}`]: { objectId: `1@${actor1}`, type: 'text', edits: [ {action: 'insert', index: 0, elemId: `2@${actor1}`, opId: `2@${actor1}`, value: {type: 'value', value: 'c'}}, {action: 'multi-insert', index: 0, elemId: `3@${actor1}`, values: ['a', 'b']}, {action: 'update', index: 2, opId: `5@${actor1}`, value: {type: 'value', value: 'C'}} ] }}}} }) assert.deepStrictEqual(backend.applyChanges([encodeChange(change3)]), { maxOp: 5, clock: {[actor1]: 2, [actor2]: 1}, deps: [hash(change2), hash(change3)].sort(), pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {text: {[`1@${actor1}`]: { objectId: `1@${actor1}`, type: 'text', edits: [ {action: 'update', index: 2, opId: `3@${actor2}`, value: {type: 'value', value: 'x'}}, {action: 'update', index: 2, opId: `4@${actor2}`, value: {type: 'value', value: 'y'}}, {action: 'update', index: 2, opId: `5@${actor1}`, value: {type: 'value', value: 'C'}} ] }}}} }) // Order of operations in the document: // {action: 'makeText', id: `1@${actor1}`, obj: '_root', key: 'text', insert: false, succ: []}, // {action: 'set', id: `3@${actor1}`, obj: `1@${actor1}`, elemId: '_head', insert: true, value: 'a', succ: []}, // {action: 'set', id: `4@${actor1}`, obj: `1@${actor1}`, elemId: `3@${actor1}`, insert: true, value: 'b', succ: []}, // {action: 'set', id: `2@${actor1}`, obj: `1@${actor1}`, elemId: '_head', insert: true, value: 'c', succ: [`3@${actor2}`, `4@${actor2}`, `5@${actor1}`]}, // {action: 'set', id: `3@${actor2}`, obj: `1@${actor1}`, elemId: `2@${actor1}`, insert: false, value: 'x', succ: []}, // {action: 'set', id: `4@${actor2}`, obj: `1@${actor1}`, elemId: `2@${actor1}`, insert: false, value: 'y', succ: []}, // {action: 'set', id: `5@${actor1}`, obj: `1@${actor1}`, elemId: `2@${actor1}`, insert: false, value: 'C', succ: []} checkColumns(backend.blocks[0], { objActor: [0, 1, 6, 0], objCtr: [0, 1, 6, 1], keyActor: [0, 2, 0x7f, 0, 0, 1, 3, 0], // null, null, 0, null, 0, 0, 0 keyCtr: [0, 1, 0x7c, 0, 3, 0x7d, 2, 2, 0], // null, 0, 3, 0, 2, 2, 2 keyStr: [0x7f, 4, 0x74, 0x65, 0x78, 0x74, 0, 6], // 'text', 6x null idActor: [4, 0, 2, 1, 0x7f, 0], // 4x actor1, 2x actor2, 1x actor1 idCtr: [0x7c, 1, 2, 1, 0x7e, 3, 1], // 1, 3, 4, 2, 3, 4, 5 insert: [1, 3, 3], // 1x false, 3x true, 3x false action: [0x7f, 4, 6, 1], // makeText, 6x set valLen: [0x7f, 0, 6, 0x16], // null, 6x 1-byte string valRaw: [0x61, 0x62, 0x63, 0x78, 0x79, 0x43], // 'a', 'b', 'c', 'x', 'y', 'C' succNum: [3, 0, 0x7f, 3, 3, 0], // 0, 0, 0, 3, 0, 0, 0 succActor: [2, 1, 0x7f, 0], // actor2, actor2, actor1 succCtr: [0x7f, 3, 2, 1] // 3, 4, 5 }) assert.strictEqual(backend.blocks[0].numOps, 7) assert.strictEqual(backend.blocks[0].lastKey, undefined) assert.strictEqual(backend.blocks[0].numVisible, 3) assert.strictEqual(backend.blocks[0].lastObjectActor, 0) assert.strictEqual(backend.blocks[0].lastObjectCtr, 1) // firstVisible is incorrect -- it should strictly be (0,3) rather than (0,2) -- but that // doesn't matter since in any case it'll be different from the previous block's lastVisible assert.strictEqual(backend.blocks[0].firstVisibleActor, 0) assert.strictEqual(backend.blocks[0].firstVisibleCtr, 2) assert.strictEqual(backend.blocks[0].lastVisibleActor, 0) assert.strictEqual(backend.blocks[0].lastVisibleCtr, 2) }) it('should allow a further conflict to be added to an existing conflict', () => { const actor1 = '01234567', actor2 = '89abcdef' const change1 = {actor: actor1, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeText', obj: '_root', key: 'text', insert: false, pred: []}, {action: 'set', obj: `1@${actor1}`, elemId: '_head', insert: true, value: 'a', pred: []} ]} const change2 = {actor: actor1, seq: 2, startOp: 3, time: 0, deps: [hash(change1)], ops: [ {action: 'set', obj: `1@${actor1}`, elemId: `2@${actor1}`, insert: false, value: 'b', pred: [`2@${actor1}`]}, {action: 'set', obj: `1@${actor1}`, elemId: `2@${actor1}`, insert: false, value: 'c', pred: [`2@${actor1}`]} ]} const change3 = {actor: actor2, seq: 1, startOp: 3, time: 0, deps: [hash(change1)], ops: [ {action: 'set', obj: `1@${actor1}`, elemId: `2@${actor1}`, insert: false, value: 'x', pred: [`2@${actor1}`]} ]} const backend = new BackendDoc() assert.deepStrictEqual(backend.applyChanges([change1, change2, change3].map(encodeChange)), { maxOp: 4, clock: {[actor1]: 2, [actor2]: 1}, deps: [hash(change2), hash(change3)].sort(), pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {text: {[`1@${actor1}`]: { objectId: `1@${actor1}`, type: 'text', edits: [ {action: 'insert', index: 0, elemId: `2@${actor1}`, opId: `3@${actor1}`, value: {type: 'value', value: 'b'}}, {action: 'update', index: 0, opId: `3@${actor2}`, value: {type: 'value', value: 'x'}}, {action: 'update', index: 0, opId: `4@${actor1}`, value: {type: 'value', value: 'c'}} ] }}}} }) // Order of operations in the document: // {action: 'makeText', id: `1@${actor1}`, obj: '_root', key: 'text', insert: false, succ: []}, // {action: 'set', id: `2@${actor1}`, obj: `1@${actor1}`, elemId: '_head', insert: true, value: 'a', succ: [`3@${actor1}`, `3@${actor2}`, `4@${actor1}`]}, // {action: 'set', id: `3@${actor1}`, obj: `1@${actor1}`, elemId: `2@${actor1}`, insert: false, value: 'b', succ: []}, // {action: 'set', id: `3@${actor2}`, obj: `1@${actor1}`, elemId: `2@${actor1}`, insert: false, value: 'x', succ: []}, // {action: 'set', id: `4@${actor1}`, obj: `1@${actor1}`, elemId: `2@${actor1}`, insert: false, value: 'c', succ: []} checkColumns(backend.blocks[0], { objActor: [0, 1, 4, 0], objCtr: [0, 1, 4, 1], keyActor: [0, 2, 3, 0], keyCtr: [0, 1, 0x7e, 0, 2, 2, 0], // null, 0, 2, 2, 2 keyStr: [0x7f, 4, 0x74, 0x65, 0x78, 0x74, 0, 4], // 'text', 4x null idActor: [3, 0, 0x7e, 1, 0], // 3x actor1, 1x actor2, 1x actor1 idCtr: [3, 1, 0x7e, 0, 1], // 1, 2, 3, 3, 4 insert: [1, 1, 3], // false, true, false, false, false action: [0x7f, 4, 4, 1], // makeText, 4x set valLen: [0x7f, 0, 4, 0x16], // null, 4x 1-byte string valRaw: [0x61, 0x62, 0x78, 0x63], // 'a', 'b', 'x', 'c' succNum: [0x7e, 0, 3, 3, 0], // 0, 3, 0, 0, 0 succActor: [0x7d, 0, 1, 0], // actor1, actor2, actor1 succCtr: [0x7d, 3, 0, 1] // 3, 3, 4 }) assert.strictEqual(backend.blocks[0].numOps, 5) assert.strictEqual(backend.blocks[0].lastKey, undefined) assert.strictEqual(backend.blocks[0].numVisible, 1) assert.strictEqual(backend.blocks[0].lastObjectActor, 0) assert.strictEqual(backend.blocks[0].lastObjectCtr, 1) assert.strictEqual(backend.blocks[0].firstVisibleActor, 0) assert.strictEqual(backend.blocks[0].firstVisibleCtr, 2) assert.strictEqual(backend.blocks[0].lastVisibleActor, 0) assert.strictEqual(backend.blocks[0].lastVisibleCtr, 2) }) it('should allow element deletes and overwrites in the same change', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeText', obj: '_root', key: 'text', insert: false, pred: []}, {action: 'set', obj: `1@${actor}`, elemId: '_head', insert: true, value: 'a', pred: []}, {action: 'set', obj: `1@${actor}`, elemId: `2@${actor}`, insert: true, value: 'b', pred: []} ]} const change2 = {actor, seq: 2, startOp: 4, time: 0, deps: [hash(change1)], ops: [ {action: 'del', obj: `1@${actor}`, elemId: `2@${actor}`, insert: false, pred: [`2@${actor}`]}, {action: 'set', obj: `1@${actor}`, elemId: `3@${actor}`, insert: false, value: 'x', pred: [`3@${actor}`]} ]} const backend = new BackendDoc() assert.deepStrictEqual(backend.applyChanges([encodeChange(change1), encodeChange(change2)]), { maxOp: 5, clock: {[actor]: 2}, deps: [hash(change2)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {text: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'text', edits: [ {action: 'multi-insert', index: 0, elemId: `2@${actor}`, values: ['a', 'b']}, {action: 'remove', index: 0, count: 1}, {action: 'update', index: 0, opId: `5@${actor}`, value: {type: 'value', value: 'x'}} ] }}}} }) checkColumns(backend.blocks[0], { objActor: [0, 1, 3, 0], objCtr: [0, 1, 3, 1], keyActor: [0, 2, 2, 0], keyCtr: [0, 1, 0x7d, 0, 2, 1], // null, 0, 2, 3 keyStr: [0x7f, 4, 0x74, 0x65, 0x78, 0x74, 0, 3], // 'text', 3x null idActor: [4, 0], idCtr: [3, 1, 0x7f, 2], // 1, 2, 3, 5 insert: [1, 2, 1], // false, true, true, false action: [0x7f, 4, 3, 1], // makeText, 3x set valLen: [0x7f, 0, 3, 0x16], // null, 3x 1-byte string valRaw: [0x61, 0x62, 0x78], // 'a', 'b', 'x' succNum: [0x7f, 0, 2, 1, 0x7f, 0], // 0, 1, 1, 0 succActor: [2, 0], succCtr: [0x7e, 4, 1] // 4, 5 }) assert.strictEqual(backend.blocks[0].numOps, 4) assert.strictEqual(backend.blocks[0].lastKey, undefined) assert.strictEqual(backend.blocks[0].numVisible, 1) assert.strictEqual(backend.blocks[0].lastObjectActor, 0) assert.strictEqual(backend.blocks[0].lastObjectCtr, 1) assert.strictEqual(backend.blocks[0].firstVisibleActor, 0) assert.strictEqual(backend.blocks[0].firstVisibleCtr, 3) assert.strictEqual(backend.blocks[0].lastVisibleActor, 0) assert.strictEqual(backend.blocks[0].lastVisibleCtr, 3) }) it('should allow concurrent deletion and assignment of the same list element', () => { const actor1 = '01234567', actor2 = '89abcdef' const change1 = {actor: actor1, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeList', obj: '_root', key: 'list', insert: false, pred: []}, {action: 'set', obj: `1@${actor1}`, elemId: '_head', insert: true, datatype: 'uint', value: 1, pred: []} ]} const change2 = {actor: actor1, seq: 2, startOp: 3, time: 0, deps: [hash(change1)], ops: [ {action: 'del', obj: `1@${actor1}`, elemId: `2@${actor1}`, insert: false, pred: [`2@${actor1}`]} ]} const change3 = {actor: actor2, seq: 1, startOp: 3, time: 0, deps: [hash(change1)], ops: [ {action: 'set', obj: `1@${actor1}`, elemId: `2@${actor1}`, insert: false, datatype: 'uint', value: 2, pred: [`2@${actor1}`]} ]} const backend1 = new BackendDoc(), backend2 = new BackendDoc() assert.deepStrictEqual(backend1.applyChanges([encodeChange(change1), encodeChange(change2)]), { maxOp: 3, clock: {[actor1]: 2}, deps: [hash(change2)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {list: {[`1@${actor1}`]: { objectId: `1@${actor1}`, type: 'list', edits: [ {action: 'insert', index: 0, elemId: `2@${actor1}`, opId: `2@${actor1}`, value: { type: 'value', value: 1, datatype: 'uint' }}, {action: 'remove', index: 0, count: 1} ] }}}} }) assert.deepStrictEqual(backend1.applyChanges([encodeChange(change3)]), { maxOp: 3, clock: {[actor1]: 2, [actor2]: 1}, deps: [hash(change2), hash(change3)].sort(), pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {list: {[`1@${actor1}`]: { objectId: `1@${actor1}`, type: 'list', edits: [ {action: 'insert', index: 0, elemId: `2@${actor1}`, opId: `3@${actor2}`, value: { type: 'value', value: 2, datatype: 'uint' }} ] }}}} }) assert.deepStrictEqual(backend2.applyChanges([encodeChange(change1), encodeChange(change3)]), { maxOp: 3, clock: {[actor1]: 1, [actor2]: 1}, deps: [hash(change3)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {list: {[`1@${actor1}`]: { objectId: `1@${actor1}`, type: 'list', edits: [ {action: 'insert', index: 0, elemId: `2@${actor1}`, opId: `3@${actor2}`, value: { type: 'value', value: 2, datatype: 'uint' }} ] }}}} }) assert.deepStrictEqual(backend2.applyChanges([encodeChange(change2)]), { maxOp: 3, clock: {[actor1]: 2, [actor2]: 1}, deps: [hash(change2), hash(change3)].sort(), pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {list: {[`1@${actor1}`]: { objectId: `1@${actor1}`, type: 'list', edits: [ {action: 'update', index: 0, opId: `3@${actor2}`, value: { type: 'value', value: 2, datatype: 'uint' }} ] }}}} }) for (let backend of [backend1, backend2]) { checkColumns(backend.blocks[0], { objActor: [0, 1, 2, 0], // null, actor1, actor1 objCtr: [0, 1, 2, 1], // null, 1, 1 keyActor: [0, 2, 0x7f, 0], // null, null, actor1 keyCtr: [0, 1, 0x7e, 0, 2], // null, 0, 2 keyStr: [0x7f, 4, 0x6c, 0x69, 0x73, 0x74, 0, 2], // 'list', null, null idActor: [2, 0, 0x7f, 1], // actor1, actor1, actor2 idCtr: [3, 1], // 1, 2, 3 insert: [1, 1, 1], // false, true, false action: [0x7f, 2, 2, 1], // makeList, 2x set valLen: [0x7f, 0, 2, 0x13], // null, 2x 1-byte uint valRaw: [1, 2], succNum: [0x7d, 0, 2, 0], // 0, 2, 0 succActor: [0x7e, 0, 1], // 0, 1 succCtr: [0x7e, 3, 0] // 3, 3 }) assert.strictEqual(backend.blocks[0].numOps, 3) assert.strictEqual(backend.blocks[0].lastKey, undefined) assert.strictEqual(backend.blocks[0].numVisible, 1) assert.strictEqual(backend.blocks[0].lastObjectActor, 0) assert.strictEqual(backend.blocks[0].lastObjectCtr, 1) assert.strictEqual(backend.blocks[0].firstVisibleActor, 0) assert.strictEqual(backend.blocks[0].firstVisibleCtr, 2) assert.strictEqual(backend.blocks[0].lastVisibleActor, 0) assert.strictEqual(backend.blocks[0].lastVisibleCtr, 2) } }) it('should handle updates inside conflicted properties', () => { const actor1 = '01234567', actor2 = '89abcdef' const change1 = {actor: actor1, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeMap', obj: '_root', key: 'map', pred: []}, {action: 'set', obj: `1@${actor1}`, key: 'x', datatype: 'uint', value: 1, pred: []} ]} const change2 = {actor: actor2, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeMap', obj: '_root', key: 'map', pred: []}, {action: 'set', obj: `1@${actor2}`, key: 'y', datatype: 'uint', value: 2, pred: []} ]} const change3 = {actor: actor1, seq: 2, startOp: 3, time: 0, deps: [hash(change1), hash(change2)], ops: [ {action: 'set', obj: `1@${actor1}`, key: 'x', datatype: 'uint', value: 3, pred: [`2@${actor1}`]} ]} const backend = new BackendDoc() assert.deepStrictEqual(backend.applyChanges([encodeChange(change1)]), { maxOp: 2, clock: {[actor1]: 1}, deps: [hash(change1)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {map: { [`1@${actor1}`]: {objectId: `1@${actor1}`, type: 'map', props: {x: {[`2@${actor1}`]: { type: 'value', value: 1, datatype: 'uint' }}}} }}} }) assert.deepStrictEqual(backend.applyChanges([encodeChange(change2)]), { maxOp: 2, clock: {[actor1]: 1, [actor2]: 1}, deps: [hash(change1), hash(change2)].sort(), pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {map: { [`1@${actor1}`]: {objectId: `1@${actor1}`, type: 'map', props: {}}, [`1@${actor2}`]: {objectId: `1@${actor2}`, type: 'map', props: {y: {[`2@${actor2}`]: { type: 'value', value: 2, datatype: 'uint' }}}} }}} }) assert.deepStrictEqual(backend.applyChanges([encodeChange(change3)]), { maxOp: 3, clock: {[actor1]: 2, [actor2]: 1}, deps: [hash(change3)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {map: { [`1@${actor1}`]: {objectId: `1@${actor1}`, type: 'map', props: {x: {[`3@${actor1}`]: { type: 'value', value: 3, datatype: 'uint' }}}}, [`1@${actor2}`]: {objectId: `1@${actor2}`, type: 'map', props: {}} }}} }) checkColumns(backend.blocks[0], { objActor: [0, 2, 2, 0, 0x7f, 1], objCtr: [0, 2, 3, 1], keyActor: [], keyCtr: [], keyStr: [2, 3, 0x6d, 0x61, 0x70, 2, 1, 0x78, 0x7f, 1, 0x79], // 'map', 'map', 'x', 'x', 'y' idActor: [0x7e, 0, 1, 2, 0, 0x7f, 1], // 0, 1, 0, 0, 1 idCtr: [0x7e, 1, 0, 2, 1, 0x7f, 0x7f], // 1, 1, 2, 3, 2 insert: [5], action: [2, 0, 3, 1], // 2x makeMap, 3x set valLen: [2, 0, 3, 0x13], // 2x null, 3x uint valRaw: [1, 3, 2], succNum: [2, 0, 0x7f, 1, 2, 0], // 0, 0, 1, 0, 0 succActor: [0x7f, 0], succCtr: [0x7f, 3] }) assert.strictEqual(backend.blocks[0].lastKey, 'y') assert.strictEqual(backend.blocks[0].numOps, 5) assert.strictEqual(backend.blocks[0].lastObjectActor, 1) assert.strictEqual(backend.blocks[0].lastObjectCtr, 1) }) it('should allow a conflict consisting of a nested object and a value', () => { const actor1 = '01234567', actor2 = '89abcdef' const change1 = {actor: actor1, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeMap', obj: '_root', key: 'x', pred: []}, {action: 'set', obj: `1@${actor1}`, key: 'y', datatype: 'uint', value: 2, pred: []} ]} const change2 = {actor: actor2, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'set', obj: '_root', key: 'x', datatype: 'uint', value: 1, pred: []} ]} const change3 = {actor: actor1, seq: 2, startOp: 3, time: 0, deps: [hash(change1), hash(change2)], ops: [ {action: 'set', obj: `1@${actor1}`, key: 'y', datatype: 'uint', value: 3, pred: [`2@${actor1}`]} ]} const backend = new BackendDoc() assert.deepStrictEqual(backend.applyChanges([encodeChange(change1)]), { maxOp: 2, clock: {[actor1]: 1}, deps: [hash(change1)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {x: { [`1@${actor1}`]: {objectId: `1@${actor1}`, type: 'map', props: {y: {[`2@${actor1}`]: { type: 'value', value: 2, datatype: 'uint' }}}} }}} }) assert.deepStrictEqual(backend.applyChanges([encodeChange(change2)]), { maxOp: 2, clock: {[actor1]: 1, [actor2]: 1}, deps: [hash(change1), hash(change2)].sort(), pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {x: { [`1@${actor1}`]: {objectId: `1@${actor1}`, type: 'map', props: {}}, [`1@${actor2}`]: {type: 'value', value: 1, datatype: 'uint'} }}} }) assert.deepStrictEqual(backend.applyChanges([encodeChange(change3)]), { maxOp: 3, clock: {[actor1]: 2, [actor2]: 1}, deps: [hash(change3)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {x: { [`1@${actor1}`]: {objectId: `1@${actor1}`, type: 'map', props: {y: {[`3@${actor1}`]: { type: 'value', value: 3, datatype: 'uint' }}}}, [`1@${actor2}`]: {type: 'value', value: 1, datatype: 'uint'} }}} }) checkColumns(backend.blocks[0], { objActor: [0, 2, 2, 0], objCtr: [0, 2, 2, 1], keyActor: [], keyCtr: [], keyStr: [2, 1, 0x78, 2, 1, 0x79], // 'x', 'x', 'y', 'y' idActor: [0x7e, 0, 1, 2, 0], // 0, 1, 0, 0 idCtr: [0x7e, 1, 0, 2, 1], // 1, 1, 2, 3 insert: [4], action: [0x7f, 0, 3, 1], // makeMap, 3x set valLen: [0x7f, 0, 3, 0x13], // null, 3x uint valRaw: [1, 2, 3], succNum: [2, 0, 0x7e, 1, 0], // 0, 0, 1, 0 succActor: [0x7f, 0], succCtr: [0x7f, 3] }) assert.strictEqual(backend.blocks[0].lastKey, 'y') assert.strictEqual(backend.blocks[0].numOps, 4) assert.strictEqual(backend.blocks[0].lastObjectActor, 0) assert.strictEqual(backend.blocks[0].lastObjectCtr, 1) }) it('should allow changes containing unknown columns, actions, and datatypes', () => { const change = new Uint8Array([ 0x85, 0x6f, 0x4a, 0x83, // magic bytes 0xad, 0xfb, 0x1a, 0x69, // checksum 1, 51, 0, 2, 0x12, 0x34, // chunkType: change, length, deps, actor '1234' 1, 1, 0, 0, // seq, startOp, time, message 0, 9, // actor list, column count 0x15, 3, 0x34, 1, 0x42, 2, // keyStr, insert, action 0x56, 2, 0x57, 4, 0x70, 2, // valLen, valRaw, predNum 0xf0, 1, 2, 0xf1, 1, 2, 0xf3, 1, 2, // unknown column group (3 columns of type GROUP_CARD, ACTOR_ID, INT_DELTA) 0x7f, 1, 0x78, // keyStr: 'x' 1, // insert: false 0x7f, 17, // unknown action type: 17 0x7f, 0x4e, // valLen: 4 bytes of unknown type 14 1, 2, 3, 4, // valRaw: 4 bytes 0x7f, 0, // predNum: 0 0x7f, 2, // unknown cardinality column: 2 values 2, 0, // unknown actor column: 0, 0 2, 1 // unknown delta column: 1, 2 ]) const backend = new BackendDoc() assert.deepStrictEqual(backend.applyChanges([change]), { maxOp: 1, clock: {'1234': 1}, deps: [decodeChange(change).hash], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {x: {}}} }) checkColumns(backend.blocks[0], { objActor: [], objCtr: [], keyActor: [], keyCtr: [], keyStr: [0x7f, 1, 0x78], idActor: [0x7f, 0], idCtr: [0x7f, 1], insert: [1], action: [0x7f, 17], valLen: [0x7f, 0x4e], valRaw: [1, 2, 3, 4], succNum: [0x7f, 0], succActor: [], succCtr: [], 240: [0x7f, 2], 241: [2, 0], 243: [2, 1] }) assert.strictEqual(backend.blocks[0].lastKey, 'x') assert.strictEqual(backend.blocks[0].numOps, 1) assert.strictEqual(backend.blocks[0].lastObjectActor, null) assert.strictEqual(backend.blocks[0].lastObjectCtr, null) }) it('should split a long insertion into multiple blocks', () => { const actor = uuid() const change = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeText', obj: '_root', key: 'text', insert: false, pred: []}, {action: 'set', obj: `1@${actor}`, elemId: '_head', insert: true, value: 'a', pred: []} ]} for (let i = 2; i <= MAX_BLOCK_SIZE; i++) { change.ops.push({action: 'set', obj: `1@${actor}`, elemId: `${i}@${actor}`, insert: true, value: 'a', pred: []}) } const backend = new BackendDoc() const patch = backend.applyChanges([encodeChange(change)]) const edits = patch.diffs.props.text[`1@${actor}`].edits assert.strictEqual(edits.length, 1) assert.strictEqual(edits[0].action, 'multi-insert') assert.strictEqual(edits[0].values.length, MAX_BLOCK_SIZE) assert.strictEqual(backend.blocks.length, 2) assert.strictEqual(bloomFilterContains(backend.blocks[0].bloom, 0, 2), true) assert.strictEqual(bloomFilterContains(backend.blocks[0].bloom, 0, MAX_BLOCK_SIZE / 2 + 1), true) assert.strictEqual(bloomFilterContains(backend.blocks[0].bloom, 0, MAX_BLOCK_SIZE / 2 + 2), false) assert.strictEqual(bloomFilterContains(backend.blocks[0].bloom, 0, MAX_BLOCK_SIZE + 1), false) assert.strictEqual(bloomFilterContains(backend.blocks[1].bloom, 0, 2), false) assert.strictEqual(bloomFilterContains(backend.blocks[1].bloom, 0, MAX_BLOCK_SIZE / 2 + 1), false) assert.strictEqual(bloomFilterContains(backend.blocks[1].bloom, 0, MAX_BLOCK_SIZE / 2 + 2), true) assert.strictEqual(bloomFilterContains(backend.blocks[1].bloom, 0, MAX_BLOCK_SIZE + 1), true) const sizeByte1 = 0x80 | 0x7f & (MAX_BLOCK_SIZE / 2), sizeByte2 = (MAX_BLOCK_SIZE / 2) >>> 7 checkColumns(backend.blocks[0], { objActor: [0, 1, sizeByte1, sizeByte2, 0], objCtr: [0, 1, sizeByte1, sizeByte2, 1], keyActor: [0, 2, sizeByte1 - 1, sizeByte2, 0], keyCtr: [0, 1, 0x7e, 0, 2, sizeByte1 - 2, sizeByte2, 1], // null, 0, 2, 3, 4, ... keyStr: [0x7f, 4, 0x74, 0x65, 0x78, 0x74, 0, sizeByte1, sizeByte2], // 'text', nulls idActor: [sizeByte1 + 1, sizeByte2, 0], idCtr: [sizeByte1 + 1, sizeByte2, 1], insert: [1, sizeByte1, sizeByte2], action: [0x7f, 4, sizeByte1, sizeByte2, 1], valLen: [0x7f, 0, sizeByte1, sizeByte2, 0x16], valRaw: new Array(MAX_BLOCK_SIZE / 2).fill(0x61), succNum: [sizeByte1 + 1, sizeByte2, 0], succActor: [], succCtr: [] }) checkColumns(backend.blocks[1], { objActor: [sizeByte1, sizeByte2, 0], objCtr: [sizeByte1, sizeByte2, 1], keyActor: [sizeByte1, sizeByte2, 0], keyCtr: [0x7f, sizeByte1 + 1, sizeByte2, sizeByte1 - 1, sizeByte2, 1], keyStr: [], idActor: [sizeByte1, sizeByte2, 0], idCtr: [0x7f, sizeByte1 + 2, sizeByte2, sizeByte1 - 1, sizeByte2, 1], insert: [0, sizeByte1, sizeByte2], action: [sizeByte1, sizeByte2, 1], valLen: [sizeByte1, sizeByte2, 0x16], valRaw: new Array(MAX_BLOCK_SIZE / 2).fill(0x61), succNum: [sizeByte1, sizeByte2, 0], succActor: [], succCtr: [] }) }) it('should split a sequence of short insertions into multiple blocks', () => { const actor = uuid(), backend = new BackendDoc() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeText', obj: '_root', key: 'text', insert: false, pred: []}, {action: 'set', obj: `1@${actor}`, elemId: '_head', insert: true, value: 'a', pred: []} ]} backend.applyChanges([encodeChange(change1)]) for (let i = 2; i <= MAX_BLOCK_SIZE; i++) { const change2 = {actor, seq: i, startOp: i + 1, time: 0, deps: backend.heads, ops: [ {action: 'set', obj: `1@${actor}`, elemId: `${i}@${actor}`, insert: true, value: 'a', pred: []} ]} assert.deepStrictEqual(backend.applyChanges([encodeChange(change2)]), { maxOp: i + 1, clock: {[actor]: i}, deps: [hash(change2)], pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {text: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'text', edits: [ {action: 'insert', index: i - 1, elemId: `${i + 1}@${actor}`, opId: `${i + 1}@${actor}`, value: {type: 'value', value: 'a'}} ] }}}} }) } assert.strictEqual(backend.blocks.length, 2) assert.strictEqual(bloomFilterContains(backend.blocks[0].bloom, 0, 2), true) assert.strictEqual(bloomFilterContains(backend.blocks[0].bloom, 0, MAX_BLOCK_SIZE / 2 + 1), true) assert.strictEqual(bloomFilterContains(backend.blocks[0].bloom, 0, MAX_BLOCK_SIZE / 2 + 2), false) assert.strictEqual(bloomFilterContains(backend.blocks[0].bloom, 0, MAX_BLOCK_SIZE + 1), false) assert.strictEqual(bloomFilterContains(backend.blocks[1].bloom, 0, 2), false) assert.strictEqual(bloomFilterContains(backend.blocks[1].bloom, 0, MAX_BLOCK_SIZE / 2 + 1), false) assert.strictEqual(bloomFilterContains(backend.blocks[1].bloom, 0, MAX_BLOCK_SIZE / 2 + 2), true) assert.strictEqual(bloomFilterContains(backend.blocks[1].bloom, 0, MAX_BLOCK_SIZE + 1), true) const sizeByte1 = 0x80 | 0x7f & (MAX_BLOCK_SIZE / 2), sizeByte2 = (MAX_BLOCK_SIZE / 2) >>> 7 checkColumns(backend.blocks[0], { objActor: [0, 1, sizeByte1, sizeByte2, 0], objCtr: [0, 1, sizeByte1, sizeByte2, 1], keyActor: [0, 2, sizeByte1 - 1, sizeByte2, 0], keyCtr: [0, 1, 0x7e, 0, 2, sizeByte1 - 2, sizeByte2, 1], // null, 0, 2, 3, 4, ... keyStr: [0x7f, 4, 0x74, 0x65, 0x78, 0x74, 0, sizeByte1, sizeByte2], // 'text', nulls idActor: [sizeByte1 + 1, sizeByte2, 0], idCtr: [sizeByte1 + 1, sizeByte2, 1], insert: [1, sizeByte1, sizeByte2], action: [0x7f, 4, sizeByte1, sizeByte2, 1], valLen: [0x7f, 0, sizeByte1, sizeByte2, 0x16], valRaw: new Array(MAX_BLOCK_SIZE / 2).fill(0x61), succNum: [sizeByte1 + 1, sizeByte2, 0], succActor: [], succCtr: [] }) checkColumns(backend.blocks[1], { objActor: [sizeByte1, sizeByte2, 0], objCtr: [sizeByte1, sizeByte2, 1], keyActor: [sizeByte1, sizeByte2, 0], keyCtr: [0x7f, sizeByte1 + 1, sizeByte2, sizeByte1 - 1, sizeByte2, 1], keyStr: [], idActor: [sizeByte1, sizeByte2, 0], idCtr: [0x7f, sizeByte1 + 2, sizeByte2, sizeByte1 - 1, sizeByte2, 1], insert: [0, sizeByte1, sizeByte2], action: [sizeByte1, sizeByte2, 1], valLen: [sizeByte1, sizeByte2, 0x16], valRaw: new Array(MAX_BLOCK_SIZE / 2).fill(0x61), succNum: [sizeByte1, sizeByte2, 0], succActor: [], succCtr: [] }) }) it('should handle insertions with Bloom filter false positives', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeText', obj: '_root', key: 'text', insert: false, pred: []}, {action: 'set', obj: `1@${actor}`, elemId: '_head', insert: true, value: 'a', pred: []} ]} for (let i = 2; i <= 2 * MAX_BLOCK_SIZE; i++) { change1.ops.push({action: 'set', obj: `1@${actor}`, elemId: `${i}@${actor}`, insert: true, value: 'a', pred: []}) } const backend = new BackendDoc(), startOp = 2 * MAX_BLOCK_SIZE + 2 backend.applyChanges([encodeChange(change1)]) assert.strictEqual(backend.blocks.length, 3) let keyCtr = backend.blocks[1].firstVisibleCtr while (keyCtr <= backend.blocks[backend.blocks.length - 1].lastVisibleCtr) { if (bloomFilterContains(backend.blocks[0].bloom, 0, keyCtr)) break keyCtr++ } if (keyCtr > backend.blocks[backend.blocks.length - 1].lastVisibleCtr) { throw new Error('no false positive found') } const change2 = {actor, seq: 2, startOp, time: 0, deps: [hash(change1)], ops: [ {action: 'set', obj: `1@${actor}`, elemId: `${keyCtr}@${actor}`, insert: true, value: 'a', pred: []} ]} const patch = backend.applyChanges([encodeChange(change2)]) assert.deepStrictEqual(patch.diffs.props.text[`1@${actor}`].edits, [{ action: 'insert', index: keyCtr - 1, elemId: `${startOp}@${actor}`, opId: `${startOp}@${actor}`, value: {type: 'value', value: 'a'} }]) }) it('should delete many consecutive characters', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeText', obj: '_root', key: 'text', insert: false, pred: []}, {action: 'set', obj: `1@${actor}`, elemId: '_head', insert: true, value: 'a', pred: []} ]} for (let i = 2; i <= MAX_BLOCK_SIZE; i++) { change1.ops.push({action: 'set', obj: `1@${actor}`, elemId: `${i}@${actor}`, insert: true, value: 'a', pred: []}) } const change2 = {actor, seq: 2, startOp: MAX_BLOCK_SIZE + 3, time: 0, deps: [], ops: []} for (let i = 2; i <= MAX_BLOCK_SIZE + 1; i++) { change2.ops.push({action: 'del', obj: `1@${actor}`, elemId: `${i}@${actor}`, insert: false, pred: [`${i}@${actor}`]}) } const backend = new BackendDoc() backend.applyChanges([encodeChange(change1)]) const patch = backend.applyChanges([encodeChange(change2)]) assert.deepStrictEqual(patch.diffs.props.text[`1@${actor}`].edits, [{action: 'remove', index: 0, count: MAX_BLOCK_SIZE}]) assert.strictEqual(backend.blocks.length, 2) const sizeByte1 = 0x80 | 0x7f & (MAX_BLOCK_SIZE / 2), sizeByte2 = (MAX_BLOCK_SIZE / 2) >>> 7 const firstSucc = MAX_BLOCK_SIZE + 3, secondSucc = MAX_BLOCK_SIZE + 3 + MAX_BLOCK_SIZE / 2 checkColumns(backend.blocks[0], { objActor: [0, 1, sizeByte1, sizeByte2, 0], objCtr: [0, 1, sizeByte1, sizeByte2, 1], keyActor: [0, 2, sizeByte1 - 1, sizeByte2, 0], keyCtr: [0, 1, 0x7e, 0, 2, sizeByte1 - 2, sizeByte2, 1], // null, 0, 2, 3, 4, ... keyStr: [0x7f, 4, 0x74, 0x65, 0x78, 0x74, 0, sizeByte1, sizeByte2], // 'text', nulls idActor: [sizeByte1 + 1, sizeByte2, 0], idCtr: [sizeByte1 + 1, sizeByte2, 1], insert: [1, sizeByte1, sizeByte2], action: [0x7f, 4, sizeByte1, sizeByte2, 1], valLen: [0x7f, 0, sizeByte1, sizeByte2, 0x16], valRaw: new Array(MAX_BLOCK_SIZE / 2).fill(0x61), succNum: [0x7f, 0, sizeByte1, sizeByte2, 1], succActor: [sizeByte1, sizeByte2, 0], succCtr: [0x7f, 0x80 | (0x7f & firstSucc), firstSucc >>> 7, sizeByte1 - 1, sizeByte2, 1] }) checkColumns(backend.blocks[1], { objActor: [sizeByte1, sizeByte2, 0], objCtr: [sizeByte1, sizeByte2, 1], keyActor: [sizeByte1, sizeByte2, 0], keyCtr: [0x7f, sizeByte1 + 1, sizeByte2, sizeByte1 - 1, sizeByte2, 1], keyStr: [], idActor: [sizeByte1, sizeByte2, 0], idCtr: [0x7f, sizeByte1 + 2, sizeByte2, sizeByte1 - 1, sizeByte2, 1], insert: [0, sizeByte1, sizeByte2], action: [sizeByte1, sizeByte2, 1], valLen: [sizeByte1, sizeByte2, 0x16], valRaw: new Array(MAX_BLOCK_SIZE / 2).fill(0x61), succNum: [sizeByte1, sizeByte2, 1], succActor: [sizeByte1, sizeByte2, 0], succCtr: [0x7f, 0x80 | (0x7f & secondSucc), secondSucc >>> 7, sizeByte1 - 1, sizeByte2, 1] }) }) it('should update an object that appears after a long text object', () => { const actor = uuid() const change1 = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeText', obj: '_root', key: 'text1', insert: false, pred: []}, {action: 'makeText', obj: '_root', key: 'text2', insert: false, pred: []}, {action: 'set', obj: `2@${actor}`, elemId: '_head', insert: true, value: 'x', pred: []}, {action: 'set', obj: `1@${actor}`, elemId: '_head', insert: true, value: 'a', pred: []} ]} for (let i = 4; i <= MAX_BLOCK_SIZE; i++) { change1.ops.push({action: 'set', obj: `1@${actor}`, elemId: `${i}@${actor}`, insert: true, value: 'a', pred: []}) } const change2 = {actor, seq: 2, startOp: MAX_BLOCK_SIZE + 3, time: 0, deps: [], ops: [ {action: 'set', obj: `2@${actor}`, elemId: `3@${actor}`, insert: true, value: 'x', pred: []} ]} const backend = new BackendDoc() backend.applyChanges([encodeChange(change1)]) assert.deepStrictEqual(backend.applyChanges([encodeChange(change2)]).diffs.props, {text2: {[`2@${actor}`]: { objectId: `2@${actor}`, type: 'text', edits: [{ action: 'insert', index: 1, opId: `${MAX_BLOCK_SIZE + 3}@${actor}`, elemId: `${MAX_BLOCK_SIZE + 3}@${actor}`, value: {type: 'value', value: 'x'} }] }}}) }) it('should place root object operations before a long text object', () => { const actor = uuid() const change = {actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeText', obj: '_root', key: 'text', insert: false, pred: []}, {action: 'set', obj: `1@${actor}`, elemId: '_head', insert: true, value: 'a', pred: []} ]} for (let i = 2; i <= MAX_BLOCK_SIZE; i++) { change.ops.push({action: 'set', obj: `1@${actor}`, elemId: `${i}@${actor}`, insert: true, value: 'a', pred: []}) } change.ops.push({action: 'set', obj: '_root', key: 'z', insert: false, value: 'zzz', pred: []}) const backend = new BackendDoc() backend.applyChanges([encodeChange(change)]) const sizeByte1 = 0x80 | 0x7f & (MAX_BLOCK_SIZE / 2), sizeByte2 = (MAX_BLOCK_SIZE / 2) >>> 7 checkColumns(backend.blocks[0], { objActor: [0, 2, sizeByte1, sizeByte2, 0], objCtr: [0, 2, sizeByte1, sizeByte2, 1], keyActor: [0, 3, sizeByte1 - 1, sizeByte2, 0], keyCtr: [0, 2, 0x7e, 0, 2, sizeByte1 - 2, sizeByte2, 1], // null, null, 0, 2, 3, 4, ... keyStr: [0x7e, 4, 0x74, 0x65, 0x78, 0x74, 1, 0x7a, 0, sizeByte1, sizeByte2], // 'text', 'z', nulls idActor: [sizeByte1 + 2, sizeByte2, 0], idCtr: [0x7d, 1, 0x80 | 0x7f & (MAX_BLOCK_SIZE + 1), 0x7f & (MAX_BLOCK_SIZE + 1) >>> 7, 0x80 | 0x7f & -MAX_BLOCK_SIZE, 0x7f & -MAX_BLOCK_SIZE >>> 7, sizeByte1 - 1, sizeByte2, 1], insert: [2, sizeByte1, sizeByte2], action: [0x7f, 4, sizeByte1 + 1, sizeByte2, 1], valLen: [0x7e, 0, 0x36, sizeByte1, sizeByte2, 0x16], valRaw: [0x7a, 0x7a, 0x7a].concat(new Array(MAX_BLOCK_SIZE / 2).fill(0x61)), succNum: [sizeByte1 + 2, sizeByte2, 0], succActor: [], succCtr: [] }) checkColumns(backend.blocks[1], { objActor: [sizeByte1, sizeByte2, 0], objCtr: [sizeByte1, sizeByte2, 1], keyActor: [sizeByte1, sizeByte2, 0], keyCtr: [0x7f, sizeByte1 + 1, sizeByte2, sizeByte1 - 1, sizeByte2, 1], keyStr: [], idActor: [sizeByte1, sizeByte2, 0], idCtr: [0x7f, sizeByte1 + 2, sizeByte2, sizeByte1 - 1, sizeByte2, 1], insert: [0, sizeByte1, sizeByte2], action: [sizeByte1, sizeByte2, 1], valLen: [sizeByte1, sizeByte2, 0x16], valRaw: new Array(MAX_BLOCK_SIZE / 2).fill(0x61), succNum: [sizeByte1, sizeByte2, 0], succActor: [], succCtr: [] }) }) }) ================================================ FILE: test/observable_test.js ================================================ const assert = require('assert') const Automerge = process.env.TEST_DIST === '1' ? require('../dist/automerge') : require('../src/automerge') describe('Automerge.Observable', () => { it('allows registering a callback on the root object', () => { let observable = new Automerge.Observable(), callbackChanges let doc = Automerge.init({observable}), actor = Automerge.getActorId(doc) observable.observe(doc, (diff, before, after, local, changes) => { callbackChanges = changes assert.deepStrictEqual(diff, { objectId: '_root', type: 'map', props: {bird: {[`1@${actor}`]: {type: 'value', value: 'Goldfinch'}}} }) assert.deepStrictEqual(before, {}) assert.deepStrictEqual(after, {bird: 'Goldfinch'}) assert.strictEqual(local, true) assert.strictEqual(changes.length, 1) }) doc = Automerge.change(doc, doc => doc.bird = 'Goldfinch') assert.strictEqual(callbackChanges.length, 1) assert.ok(callbackChanges[0] instanceof Uint8Array) assert.strictEqual(callbackChanges[0], Automerge.getLastLocalChange(doc)) }) it('allows registering a callback on a text object', () => { let observable = new Automerge.Observable(), callbackCalled = false let doc = Automerge.from({text: new Automerge.Text()}, {observable}) let actor = Automerge.getActorId(doc) observable.observe(doc.text, (diff, before, after, local) => { callbackCalled = true assert.deepStrictEqual(diff, { objectId: `1@${actor}`, type: 'text', edits: [ {action: 'multi-insert', index: 0, elemId: `2@${actor}`, values: ['a', 'b', 'c']} ] }) assert.deepStrictEqual(before.toString(), '') assert.deepStrictEqual(after.toString(), 'abc') assert.deepStrictEqual(local, true) }) doc = Automerge.change(doc, doc => doc.text.insertAt(0, 'a', 'b', 'c')) assert.strictEqual(callbackCalled, true) }) it('should call the callback when applying remote changes', () => { let observable = new Automerge.Observable(), callbackChanges let local = Automerge.from({text: new Automerge.Text()}, {observable}) let remote = Automerge.init() const localId = Automerge.getActorId(local), remoteId = Automerge.getActorId(remote) observable.observe(local.text, (diff, before, after, local, changes) => { callbackChanges = changes assert.deepStrictEqual(diff, { objectId: `1@${localId}`, type: 'text', edits: [ {action: 'insert', index: 0, elemId: `2@${remoteId}`, opId: `2@${remoteId}`, value: {type: 'value', value: 'a'}} ] }) assert.deepStrictEqual(before.toString(), '') assert.deepStrictEqual(after.toString(), 'a') assert.deepStrictEqual(local, false) }) ;[remote] = Automerge.applyChanges(remote, Automerge.getAllChanges(local)) remote = Automerge.change(remote, doc => doc.text.insertAt(0, 'a')) const allChanges = Automerge.getAllChanges(remote) ;[local] = Automerge.applyChanges(local, allChanges) assert.strictEqual(callbackChanges, allChanges) }) it('should observe objects nested inside list elements', () => { let observable = new Automerge.Observable(), callbackCalled = false let doc = Automerge.from({todos: [{title: 'Buy milk', done: false}]}, {observable}) const actor = Automerge.getActorId(doc) observable.observe(doc.todos[0], (diff, before, after, local) => { callbackCalled = true assert.deepStrictEqual(diff, { objectId: `2@${actor}`, type: 'map', props: {done: {[`5@${actor}`]: {type: 'value', value: true}}} }) assert.deepStrictEqual(before, {title: 'Buy milk', done: false}) assert.deepStrictEqual(after, {title: 'Buy milk', done: true}) assert.strictEqual(local, true) }) doc = Automerge.change(doc, doc => doc.todos[0].done = true) assert.strictEqual(callbackCalled, true) }) it('should provide before and after states if list indexes changed', () => { let observable = new Automerge.Observable(), callbackCalled = false let doc = Automerge.from({todos: [{title: 'Buy milk', done: false}]}, {observable}) const actor = Automerge.getActorId(doc) observable.observe(doc.todos[0], (diff, before, after, local) => { callbackCalled = true assert.deepStrictEqual(diff, { objectId: `2@${actor}`, type: 'map', props: {done: {[`8@${actor}`]: {type: 'value', value: true}}} }) assert.deepStrictEqual(before, {title: 'Buy milk', done: false}) assert.deepStrictEqual(after, {title: 'Buy milk', done: true}) assert.strictEqual(local, true) }) doc = Automerge.change(doc, doc => { doc.todos.unshift({title: 'Water plants', done: false}) doc.todos[1].done = true }) assert.strictEqual(callbackCalled, true) }) it('should observe rows inside tables', () => { let observable = new Automerge.Observable(), callbackCalled = false let doc = Automerge.init({observable}), actor = Automerge.getActorId(doc), rowId doc = Automerge.change(doc, doc => { doc.todos = new Automerge.Table() rowId = doc.todos.add({title: 'Buy milk', done: false}) }) observable.observe(doc.todos.byId(rowId), (diff, before, after, local) => { callbackCalled = true assert.deepStrictEqual(diff, { objectId: `2@${actor}`, type: 'map', props: {done: {[`5@${actor}`]: {type: 'value', value: true}}} }) assert.deepStrictEqual(before, {id: rowId, title: 'Buy milk', done: false}) assert.deepStrictEqual(after, {id: rowId, title: 'Buy milk', done: true}) assert.strictEqual(local, true) }) doc = Automerge.change(doc, doc => doc.todos.byId(rowId).done = true) assert.strictEqual(callbackCalled, true) }) it('should observe nested objects inside text', () => { let observable = new Automerge.Observable(), callbackCalled = false let doc = Automerge.init({observable}), actor = Automerge.getActorId(doc) doc = Automerge.change(doc, doc => { doc.text = new Automerge.Text() doc.text.insertAt(0, 'a', 'b', {start: 'bold'}, 'c', {end: 'bold'}) }) observable.observe(doc.text.get(2), (diff, before, after, local) => { callbackCalled = true assert.deepStrictEqual(diff, { objectId: `4@${actor}`, type: 'map', props: {start: {[`9@${actor}`]: {type: 'value', value: 'italic'}}} }) assert.deepStrictEqual(before, {start: 'bold'}) assert.deepStrictEqual(after, {start: 'italic'}) assert.strictEqual(local, true) }) doc = Automerge.change(doc, doc => doc.text.get(2).start = 'italic') assert.strictEqual(callbackCalled, true) }) it('should not allow observers on non-document objects', () => { let observable = new Automerge.Observable() let doc = Automerge.init({observable}) assert.throws(() => { Automerge.change(doc, doc => { const text = new Automerge.Text() doc.text = text observable.observe(text, () => {}) }) }, /The observed object must be part of an Automerge document/) }) it('should allow multiple observers', () => { let observable = new Automerge.Observable(), called1 = false, called2 = false let doc = Automerge.init({observable}) observable.observe(doc, () => { called1 = true }) observable.observe(doc, () => { called2 = true }) Automerge.change(doc, doc => doc.foo = 'bar') assert.strictEqual(called1, true) assert.strictEqual(called2, true) }) }) ================================================ FILE: test/proxies_test.js ================================================ const assert = require('assert') const Automerge = process.env.TEST_DIST === '1' ? require('../dist/automerge') : require('../src/automerge') const { assertEqualsOneOf } = require('./helpers') const UUID_PATTERN = /^[0-9a-f]{32}$/ describe('Automerge proxy API', () => { describe('root object', () => { it('should have a fixed object ID', () => { Automerge.change(Automerge.init(), doc => { assert.strictEqual(Automerge.getObjectId(doc), '_root') }) }) it('should know its actor ID', () => { Automerge.change(Automerge.init(), doc => { assert(UUID_PATTERN.test(Automerge.getActorId(doc).toString())) assert.notEqual(Automerge.getActorId(doc), '_root') assert.strictEqual(Automerge.getActorId(Automerge.init('01234567')), '01234567') }) }) it('should expose keys as object properties', () => { Automerge.change(Automerge.init(), doc => { doc.key1 = 'value1' assert.strictEqual(doc.key1, 'value1') }) }) it('should return undefined for unknown properties', () => { Automerge.change(Automerge.init(), doc => { assert.strictEqual(doc.someProperty, undefined) }) }) it('should support the "in" operator', () => { Automerge.change(Automerge.init(), doc => { assert.strictEqual('key1' in doc, false) doc.key1 = 'value1' assert.strictEqual('key1' in doc, true) }) }) it('should support Object.keys()', () => { Automerge.change(Automerge.init(), doc => { assert.deepStrictEqual(Object.keys(doc), []) doc.key1 = 'value1' assert.deepStrictEqual(Object.keys(doc), ['key1']) doc.key2 = 'value2' assertEqualsOneOf(Object.keys(doc), ['key1', 'key2'], ['key2', 'key1']) }) }) it('should support Object.getOwnPropertyNames()', () => { Automerge.change(Automerge.init(), doc => { assert.deepStrictEqual(Object.getOwnPropertyNames(doc), []) doc.key1 = 'value1' assert.deepStrictEqual(Object.getOwnPropertyNames(doc), ['key1']) doc.key2 = 'value2' assertEqualsOneOf(Object.getOwnPropertyNames(doc), ['key1', 'key2'], ['key2', 'key1']) }) }) it('should support bulk assignment with Object.assign()', () => { Automerge.change(Automerge.init(), doc => { Object.assign(doc, {key1: 'value1', key2: 'value2'}) assert.deepStrictEqual(doc, {key1: 'value1', key2: 'value2'}) }) }) it('should support JSON.stringify()', () => { Automerge.change(Automerge.init(), doc => { assert.deepStrictEqual(JSON.stringify(doc), '{}') doc.key1 = 'value1' assert.deepStrictEqual(JSON.stringify(doc), '{"key1":"value1"}') doc.key2 = 'value2' assert.deepStrictEqual(JSON.parse(JSON.stringify(doc)), { key1: 'value1', key2: 'value2' }) }) }) it('should allow access to an object by id', () => { const doc = Automerge.change(Automerge.init(), doc => { doc.deepObj = {} doc.deepObj.deepList = [] const listId = Automerge.getObjectId(doc.deepObj.deepList) assert.throws(() => { Automerge.getObjectById(doc, listId) }, /Cannot use getObjectById in a change callback/) }) const objId = Automerge.getObjectId(doc.deepObj) assert.strictEqual(Automerge.getObjectById(doc, objId), doc.deepObj) const listId = Automerge.getObjectId(doc.deepObj.deepList) assert.strictEqual(Automerge.getObjectById(doc, listId), doc.deepObj.deepList) }) }) describe('list object', () => { let root beforeEach(() => { root = Automerge.change(Automerge.init(), doc => { doc.list = [1, 2, 3] doc.empty = [] doc.listObjects = [ {id: "first"}, {id: "second"} ] }) }) it('should look like a JavaScript array', () => { Automerge.change(root, doc => { assert.strictEqual(Array.isArray(doc.list), true) assert.strictEqual(typeof doc.list, 'object') assert.strictEqual(toString.call(doc.list), '[object Array]') }) }) it('should have a length property', () => { Automerge.change(root, doc => { assert.strictEqual(doc.empty.length, 0) assert.strictEqual(doc.list.length, 3) }) }) it('should allow entries to be fetched by index', () => { Automerge.change(root, doc => { assert.strictEqual(doc.list[0], 1) assert.strictEqual(doc.list['0'], 1) assert.strictEqual(doc.list[1], 2) assert.strictEqual(doc.list['1'], 2) assert.strictEqual(doc.list[2], 3) assert.strictEqual(doc.list['2'], 3) assert.strictEqual(doc.list[3], undefined) assert.strictEqual(doc.list['3'], undefined) assert.strictEqual(doc.list[-1], undefined) assert.strictEqual(doc.list.someProperty, undefined) }) }) it('should support the "in" operator', () => { Automerge.change(root, doc => { assert.strictEqual(0 in doc.list, true) assert.strictEqual('0' in doc.list, true) assert.strictEqual(3 in doc.list, false) assert.strictEqual('3' in doc.list, false) assert.strictEqual('length' in doc.list, true) assert.strictEqual('someProperty' in doc.list, false) }) }) it('should support Object.keys()', () => { Automerge.change(root, doc => { assert.deepStrictEqual(Object.keys(doc.list), ['0', '1', '2']) }) }) it('should support Object.getOwnPropertyNames()', () => { Automerge.change(root, doc => { assert.deepStrictEqual(Object.getOwnPropertyNames(doc.list), ['length', '0', '1', '2']) }) }) it('should support JSON.stringify()', () => { Automerge.change(root, doc => { assert.deepStrictEqual(JSON.parse(JSON.stringify(doc)), { list: [1, 2, 3], empty: [], listObjects: [ {id: "first"}, {id: "second"} ] }) assert.deepStrictEqual(JSON.stringify(doc.list), '[1,2,3]') }) }) it('should support iteration', () => { Automerge.change(root, doc => { let copy = [] for (let x of doc.list) copy.push(x) assert.deepStrictEqual(copy, [1, 2, 3]) // spread operator also uses iteration protocol assert.deepStrictEqual([0, ...doc.list, 4], [0, 1, 2, 3, 4]) }) }) describe('should support standard array read-only operations', () => { it('concat()', () => { Automerge.change(root, doc => { assert.deepStrictEqual(doc.list.concat([4, 5, 6]), [1, 2, 3, 4, 5, 6]) assert.deepStrictEqual(doc.list.concat([4], [5, [6]]), [1, 2, 3, 4, 5, [6]]) }) }) it('entries()', () => { Automerge.change(root, doc => { let copy = [] for (let x of doc.list.entries()) copy.push(x) assert.deepStrictEqual(copy, [[0, 1], [1, 2], [2, 3]]) assert.deepStrictEqual([...doc.list.entries()], [[0, 1], [1, 2], [2, 3]]) }) }) it('every()', () => { Automerge.change(root, doc => { assert.strictEqual(doc.empty.every(() => false), true) assert.strictEqual(doc.list.every(val => val > 0), true) assert.strictEqual(doc.list.every(val => val > 2), false) assert.strictEqual(doc.list.every((val, index) => index < 3), true) // check that in the callback, 'this' is set to the second argument of 'every' doc.list.every(function () { assert.strictEqual(this.hello, 'world'); return true }, {hello: 'world'}) }) }) it('filter()', () => { Automerge.change(root, doc => { assert.deepStrictEqual(doc.empty.filter(() => false), []) assert.deepStrictEqual(doc.list.filter(num => num % 2 === 1), [1, 3]) assert.deepStrictEqual(doc.list.filter(() => true), [1, 2, 3]) doc.list.filter(function () { assert.strictEqual(this.hello, 'world'); return true }, {hello: 'world'}) }) }) it('find()', () => { Automerge.change(root, doc => { assert.strictEqual(doc.empty.find(() => true), undefined) assert.strictEqual(doc.list.find(num => num >= 2), 2) assert.strictEqual(doc.list.find(num => num >= 4), undefined) doc.list.find(function () { assert.strictEqual(this.hello, 'world'); return true }, {hello: 'world'}) }) }) it('findIndex()', () => { Automerge.change(root, doc => { assert.strictEqual(doc.empty.findIndex(() => true), -1) assert.strictEqual(doc.list.findIndex(num => num >= 2), 1) assert.strictEqual(doc.list.findIndex(num => num >= 4), -1) doc.list.findIndex(function () { assert.strictEqual(this.hello, 'world'); return true }, {hello: 'world'}) }) }) it('forEach()', () => { Automerge.change(root, doc => { doc.empty.forEach(() => { assert.fail('was called', 'not called', 'callback error') }) let binary = [] doc.list.forEach(num => binary.push(num.toString(2))) assert.deepStrictEqual(binary, ['1', '10', '11']) doc.list.forEach(function () { assert.strictEqual(this.hello, 'world'); return true }, {hello: 'world'}) }) }) it('includes()', () => { Automerge.change(root, doc => { assert.strictEqual(doc.empty.includes(3), false) assert.strictEqual(doc.list.includes(3), true) assert.strictEqual(doc.list.includes(1, 1), false) assert.strictEqual(doc.list.includes(2, -2), true) assert.strictEqual(doc.list.includes(0), false) }) }) it('indexOf()', () => { Automerge.change(root, doc => { assert.strictEqual(doc.empty.indexOf(3), -1) assert.strictEqual(doc.list.indexOf(3), 2) assert.strictEqual(doc.list.indexOf(1, 1), -1) assert.strictEqual(doc.list.indexOf(2, -2), 1) assert.strictEqual(doc.list.indexOf(0), -1) assert.strictEqual(doc.list.indexOf(undefined), -1) }) }) it('indexOf() with objects', () => { Automerge.change(root, doc => { assert.strictEqual(doc.listObjects.indexOf(doc.listObjects[0]), 0) assert.strictEqual(doc.listObjects.indexOf(doc.listObjects[1]), 1) assert.strictEqual(doc.listObjects.indexOf(doc.listObjects[0], 0), 0) assert.strictEqual(doc.listObjects.indexOf(doc.listObjects[0], 1), -1) assert.strictEqual(doc.listObjects.indexOf(doc.listObjects[1], 0), 1) assert.strictEqual(doc.listObjects.indexOf(doc.listObjects[1], 1), 1) }) }) it('join()', () => { Automerge.change(root, doc => { assert.strictEqual(doc.empty.join(', '), '') assert.strictEqual(doc.list.join(), '1,2,3') assert.strictEqual(doc.list.join(''), '123') assert.strictEqual(doc.list.join(', '), '1, 2, 3') }) }) it('keys()', () => { Automerge.change(root, doc => { let keys = [] for (let x of doc.list.keys()) keys.push(x) assert.deepStrictEqual(keys, [0, 1, 2]) assert.deepStrictEqual([...doc.list.keys()], [0, 1, 2]) }) }) it('lastIndexOf()', () => { Automerge.change(root, doc => { assert.strictEqual(doc.empty.lastIndexOf(3), -1) assert.strictEqual(doc.list.lastIndexOf(3), 2) assert.strictEqual(doc.list.lastIndexOf(3, 1), -1) assert.strictEqual(doc.list.lastIndexOf(3, -1), 2) assert.strictEqual(doc.list.lastIndexOf(0), -1) }) }) it('map()', () => { Automerge.change(root, doc => { assert.deepStrictEqual(doc.empty.map(num => num * 2), []) assert.deepStrictEqual(doc.list.map(num => num * 2), [2, 4, 6]) assert.deepStrictEqual(doc.list.map((num, index) => index + '->' + num), ['0->1', '1->2', '2->3']) doc.list.map(function () { assert.strictEqual(this.hello, 'world'); return true }, {hello: 'world'}) }) }) it('reduce()', () => { Automerge.change(root, doc => { assert.strictEqual(doc.empty.reduce((sum, val) => sum + val, 0), 0) assert.strictEqual(doc.list.reduce((sum, val) => sum + val, 0), 6) assert.strictEqual(doc.list.reduce((sum, val) => sum + val, ''), '123') assert.strictEqual(doc.list.reduce((sum, val) => sum + val), 6) assert.strictEqual(doc.list.reduce((sum, val, index) => ((index % 2 === 0) ? (sum + val) : sum), 0), 4) }) }) it('reduceRight()', () => { Automerge.change(root, doc => { assert.strictEqual(doc.empty.reduceRight((sum, val) => sum + val, 0), 0) assert.strictEqual(doc.list.reduceRight((sum, val) => sum + val, 0), 6) assert.strictEqual(doc.list.reduceRight((sum, val) => sum + val, ''), '321') assert.strictEqual(doc.list.reduceRight((sum, val) => sum + val), 6) assert.strictEqual(doc.list.reduceRight((sum, val, index) => ((index % 2 === 0) ? (sum + val) : sum), 0), 4) }) }) it('slice()', () => { Automerge.change(root, doc => { assert.deepStrictEqual(doc.empty.slice(), []) assert.deepStrictEqual(doc.list.slice(2), [3]) assert.deepStrictEqual(doc.list.slice(-2), [2, 3]) assert.deepStrictEqual(doc.list.slice(0, 0), []) assert.deepStrictEqual(doc.list.slice(0, 1), [1]) assert.deepStrictEqual(doc.list.slice(0, -1), [1, 2]) }) }) it('some()', () => { Automerge.change(root, doc => { assert.strictEqual(doc.empty.some(() => true), false) assert.strictEqual(doc.list.some(val => val > 2), true) assert.strictEqual(doc.list.some(val => val > 4), false) assert.strictEqual(doc.list.some((val, index) => index > 2), false) doc.list.some(function () { assert.strictEqual(this.hello, 'world'); return true }, {hello: 'world'}) }) }) it('toString()', () => { Automerge.change(root, doc => { assert.strictEqual(doc.empty.toString(), '') assert.strictEqual(doc.list.toString(), '1,2,3') }) }) it('values()', () => { Automerge.change(root, doc => { let values = [] for (let x of doc.list.values()) values.push(x) assert.deepStrictEqual(values, [1, 2, 3]) assert.deepStrictEqual([...doc.list.values()], [1, 2, 3]) }) }) it('should allow mutation of objects returned from built in list iteration', () => { root = Automerge.change(Automerge.init({freeze: true}), doc => { doc.objects = [{id: 1, value: 'one'}, {id: 2, value: 'two'}] }) root = Automerge.change(root, doc => { for (let obj of doc.objects) if (obj.id === 1) obj.value = 'ONE!' }) assert.deepStrictEqual(root, {objects: [{id: 1, value: 'ONE!'}, {id: 2, value: 'two'}]}) }) it('should allow mutation of objects returned from readonly list methods', () => { root = Automerge.change(Automerge.init({freeze: true}), doc => { doc.objects = [{id: 1, value: 'one'}, {id: 2, value: 'two'}] }) root = Automerge.change(root, doc => { doc.objects.find(obj => obj.id === 1).value = 'ONE!' }) assert.deepStrictEqual(root, {objects: [{id: 1, value: 'ONE!'}, {id: 2, value: 'two'}]}) }) }) describe('should support standard mutation methods', () => { it('fill()', () => { root = Automerge.change(root, doc => doc.list.fill('a')) assert.deepStrictEqual(root.list, ['a', 'a', 'a']) root = Automerge.change(root, doc => doc.list.fill('c', 1).fill('b', 1, 2)) assert.deepStrictEqual(root.list, ['a', 'b', 'c']) }) it('pop()', () => { root = Automerge.change(root, doc => assert.strictEqual(doc.list.pop(), 3)) assert.deepStrictEqual(root.list, [1, 2]) root = Automerge.change(root, doc => assert.strictEqual(doc.list.pop(), 2)) assert.deepStrictEqual(root.list, [1]) root = Automerge.change(root, doc => assert.strictEqual(doc.list.pop(), 1)) assert.deepStrictEqual(root.list, []) root = Automerge.change(root, doc => assert.strictEqual(doc.list.pop(), undefined)) assert.deepStrictEqual(root.list, []) }) it('push()', () => { root = Automerge.change(root, doc => doc.noodles = []) root = Automerge.change(root, doc => doc.noodles.push('udon', 'soba')) root = Automerge.change(root, doc => doc.noodles.push('ramen')) assert.deepStrictEqual(root.noodles, ['udon', 'soba', 'ramen']) assert.strictEqual(root.noodles[0], 'udon') assert.strictEqual(root.noodles[1], 'soba') assert.strictEqual(root.noodles[2], 'ramen') assert.strictEqual(root.noodles.length, 3) }) it('shift()', () => { root = Automerge.change(root, doc => assert.strictEqual(doc.list.shift(), 1)) assert.deepStrictEqual(root.list, [2, 3]) root = Automerge.change(root, doc => assert.strictEqual(doc.list.shift(), 2)) assert.deepStrictEqual(root.list, [3]) root = Automerge.change(root, doc => assert.strictEqual(doc.list.shift(), 3)) assert.deepStrictEqual(root.list, []) root = Automerge.change(root, doc => assert.strictEqual(doc.list.shift(), undefined)) assert.deepStrictEqual(root.list, []) }) it('splice()', () => { root = Automerge.change(root, doc => assert.deepStrictEqual(doc.list.splice(1), [2, 3])) assert.deepStrictEqual(root.list, [1]) root = Automerge.change(root, doc => assert.deepStrictEqual(doc.list.splice(0, 0, 'a', 'b', 'c'), [])) assert.deepStrictEqual(root.list, ['a', 'b', 'c', 1]) root = Automerge.change(root, doc => assert.deepStrictEqual(doc.list.splice(1, 2, '-->'), ['b', 'c'])) assert.deepStrictEqual(root.list, ['a', '-->', 1]) root = Automerge.change(root, doc => assert.deepStrictEqual(doc.list.splice(2, 200, 2), [1])) assert.deepStrictEqual(root.list, ['a', '-->', 2]) }) it('unshift()', () => { root = Automerge.change(root, doc => doc.noodles = []) root = Automerge.change(root, doc => doc.noodles.unshift('soba', 'udon')) root = Automerge.change(root, doc => doc.noodles.unshift('ramen')) assert.deepStrictEqual(root.noodles, ['ramen', 'soba', 'udon']) assert.strictEqual(root.noodles[0], 'ramen') assert.strictEqual(root.noodles[1], 'soba') assert.strictEqual(root.noodles[2], 'udon') assert.strictEqual(root.noodles.length, 3) }) }) }) }) ================================================ FILE: test/sync_test.js ================================================ const assert = require('assert') const Automerge = process.env.TEST_DIST === '1' ? require('../dist/automerge') : require('../src/automerge') const { BloomFilter } = require('../backend/sync') const { decodeChangeMeta } = require('../backend/columnar') const { decodeSyncMessage, encodeSyncMessage, decodeSyncState, encodeSyncState, initSyncState } = Automerge.Backend function getHeads(doc) { return Automerge.Backend.getHeads(Automerge.Frontend.getBackendState(doc)) } function getMissingDeps(doc) { return Automerge.Backend.getMissingDeps(Automerge.Frontend.getBackendState(doc)) } function sync(a, b, aSyncState = initSyncState(), bSyncState = initSyncState()) { const MAX_ITER = 10 let aToBmsg = null, bToAmsg = null, i = 0 do { [aSyncState, aToBmsg] = Automerge.generateSyncMessage(a, aSyncState) ;[bSyncState, bToAmsg] = Automerge.generateSyncMessage(b, bSyncState) if (aToBmsg) { [b, bSyncState] = Automerge.receiveSyncMessage(b, bSyncState, aToBmsg) } if (bToAmsg) { [a, aSyncState] = Automerge.receiveSyncMessage(a, aSyncState, bToAmsg) } if (i++ > MAX_ITER) { throw new Error(`Did not synchronize within ${MAX_ITER} iterations. Do you have a bug causing an infinite loop?`) } } while (aToBmsg || bToAmsg) return [a, b, aSyncState, bSyncState] } describe('Data sync protocol', () => { describe('with docs already in sync', () => { describe('an empty local doc', () => { it('should send a sync message implying no local data', () => { let n1 = Automerge.init() let s1 = initSyncState() let m1 ;[s1, m1] = Automerge.generateSyncMessage(n1, s1) const message = decodeSyncMessage(m1) assert.deepStrictEqual(message.heads, []) assert.deepStrictEqual(message.need, []) assert.deepStrictEqual(message.have.length, 1) assert.deepStrictEqual(message.have[0].lastSync, []) assert.deepStrictEqual(message.have[0].bloom.byteLength, 0) assert.deepStrictEqual(message.changes, []) }) it('should not reply if we have no data as well', () => { let n1 = Automerge.init(), n2 = Automerge.init() let s1 = initSyncState(), s2 = initSyncState() let m1 = null, m2 = null ;[s1, m1] = Automerge.generateSyncMessage(n1, s1) ;[n2, s2] = Automerge.receiveSyncMessage(n2, s2, m1) ;[s2, m2] = Automerge.generateSyncMessage(n2, s2) assert.deepStrictEqual(m2, null) }) }) describe('documents with data', () => { it('repos with equal heads do not need a reply message', () => { let n1 = Automerge.init(), n2 = Automerge.init() let s1 = initSyncState(), s2 = initSyncState() let m1 = null, m2 = null // make two nodes with the same changes n1 = Automerge.change(n1, {time: 0}, doc => doc.n = []) for (let i = 0; i < 10; i++) n1 = Automerge.change(n1, {time: 0}, doc => doc.n.push(i)) ;[n2] = Automerge.applyChanges(n2, Automerge.getAllChanges(n1)) assert.deepStrictEqual(n1, n2) // generate a naive sync message ;[s1, m1] = Automerge.generateSyncMessage(n1, s1) assert.deepStrictEqual(s1.lastSentHeads, getHeads(n1)) // heads are equal so this message should be null ;[n2, s2] = Automerge.receiveSyncMessage(n2, s2, m1) ;[s2, m2] = Automerge.generateSyncMessage(n2, s2) assert.strictEqual(m2, null) }) it('n1 should offer all changes to n2 when starting from nothing', () => { let n1 = Automerge.init(), n2 = Automerge.init() // make changes for n1 that n2 should request n1 = Automerge.change(n1, {time: 0}, doc => doc.n = []) for (let i = 0; i < 10; i++) n1 = Automerge.change(n1, {time: 0}, doc => doc.n.push(i)) assert.notDeepStrictEqual(n1, n2) const [after1, after2] = sync(n1, n2) assert.deepStrictEqual(after1, after2) }) it('should sync peers where one has commits the other does not', () => { let n1 = Automerge.init(), n2 = Automerge.init() // make changes for n1 that n2 should request n1 = Automerge.change(n1, {time: 0}, doc => doc.n = []) for (let i = 0; i < 10; i++) n1 = Automerge.change(n1, {time: 0}, doc => doc.n.push(i)) assert.notDeepStrictEqual(n1, n2) ;[n1, n2] = sync(n1, n2) assert.deepStrictEqual(n1, n2) }) it('should work with prior sync state', () => { // create & synchronize two nodes let n1 = Automerge.init(), n2 = Automerge.init() let s1 = initSyncState(), s2 = initSyncState() for (let i = 0; i < 5; i++) n1 = Automerge.change(n1, {time: 0}, doc => doc.x = i) ;[n1, n2, s1, s2] = sync(n1, n2) // modify the first node further for (let i = 5; i < 10; i++) n1 = Automerge.change(n1, {time: 0}, doc => doc.x = i) assert.notDeepStrictEqual(n1, n2) ;[n1, n2, s1, s2] = sync(n1, n2, s1, s2) assert.deepStrictEqual(n1, n2) }) it('should not generate messages once synced', () => { // create & synchronize two nodes let n1 = Automerge.init('abc123'), n2 = Automerge.init('def456') let s1 = initSyncState(), s2 = initSyncState() let message, patch for (let i = 0; i < 5; i++) n1 = Automerge.change(n1, {time: 0}, doc => doc.x = i) for (let i = 0; i < 5; i++) n2 = Automerge.change(n2, {time: 0}, doc => doc.y = i) // n1 reports what it has ;[s1, message] = Automerge.generateSyncMessage(n1, s1, n1) // n2 receives that message and sends changes along with what it has ;[n2, s2, patch] = Automerge.receiveSyncMessage(n2, s2, message) ;[s2, message] = Automerge.generateSyncMessage(n2, s2) assert.deepStrictEqual(decodeSyncMessage(message).changes.length, 5) assert.deepStrictEqual(patch, null) // no changes arrived // n1 receives the changes and replies with the changes it now knows n2 needs ;[n1, s1, patch] = Automerge.receiveSyncMessage(n1, s1, message) ;[s1, message] = Automerge.generateSyncMessage(n1, s1) assert.deepStrictEqual(decodeSyncMessage(message).changes.length, 5) assert.deepStrictEqual(patch.diffs.props, {y: {'5@def456': {type: 'value', value: 4, datatype: 'int'}}}) // changes arrived // n2 applies the changes and sends confirmation ending the exchange ;[n2, s2, patch] = Automerge.receiveSyncMessage(n2, s2, message) ;[s2, message] = Automerge.generateSyncMessage(n2, s2) assert.deepStrictEqual(patch.diffs.props, {x: {'5@abc123': {type: 'value', value: 4, datatype: 'int'}}}) // changes arrived // n1 receives the message and has nothing more to say ;[n1, s1, patch] = Automerge.receiveSyncMessage(n1, s1, message) ;[s1, message] = Automerge.generateSyncMessage(n1, s1) assert.deepStrictEqual(message, null) assert.deepStrictEqual(patch, null) // no changes arrived // n2 also has nothing left to say ;[s2, message] = Automerge.generateSyncMessage(n2, s2) assert.deepStrictEqual(message, null) }) it('should allow simultaneous messages during synchronization', () => { // create & synchronize two nodes let n1 = Automerge.init('abc123'), n2 = Automerge.init('def456') let s1 = initSyncState(), s2 = initSyncState() for (let i = 0; i < 5; i++) n1 = Automerge.change(n1, {time: 0}, doc => doc.x = i) for (let i = 0; i < 5; i++) n2 = Automerge.change(n2, {time: 0}, doc => doc.y = i) const head1 = getHeads(n1)[0], head2 = getHeads(n2)[0] // both sides report what they have but have no shared peer state let msg1to2, msg2to1 ;[s1, msg1to2] = Automerge.generateSyncMessage(n1, s1) ;[s2, msg2to1] = Automerge.generateSyncMessage(n2, s2) assert.deepStrictEqual(decodeSyncMessage(msg1to2).changes.length, 0) assert.deepStrictEqual(decodeSyncMessage(msg1to2).have[0].lastSync.length, 0) assert.deepStrictEqual(decodeSyncMessage(msg2to1).changes.length, 0) assert.deepStrictEqual(decodeSyncMessage(msg2to1).have[0].lastSync.length, 0) // n1 and n2 receives that message and update sync state but make no patch let patch1, patch2 ;[n1, s1, patch1] = Automerge.receiveSyncMessage(n1, s1, msg2to1) assert.deepStrictEqual(patch1, null) // no changes arrived, so no patch ;[n2, s2, patch2] = Automerge.receiveSyncMessage(n2, s2, msg1to2) assert.deepStrictEqual(patch2, null) // no changes arrived, so no patch // now both reply with their local changes the other lacks // (standard warning that 1% of the time this will result in a "need" message) ;[s1, msg1to2] = Automerge.generateSyncMessage(n1, s1) assert.deepStrictEqual(decodeSyncMessage(msg1to2).changes.length, 5) ;[s2, msg2to1] = Automerge.generateSyncMessage(n2, s2) assert.deepStrictEqual(decodeSyncMessage(msg2to1).changes.length, 5) // both should now apply the changes and update the frontend ;[n1, s1, patch1] = Automerge.receiveSyncMessage(n1, s1, msg2to1) assert.deepStrictEqual(getMissingDeps(n1), []) assert.notDeepStrictEqual(patch1, null) assert.deepStrictEqual(n1, {x: 4, y: 4}) ;[n2, s2, patch2] = Automerge.receiveSyncMessage(n2, s2, msg1to2) assert.deepStrictEqual(getMissingDeps(n2), []) assert.notDeepStrictEqual(patch2, null) assert.deepStrictEqual(n2, {x: 4, y: 4}) // The response acknowledges the changes received, and sends no further changes ;[s1, msg1to2] = Automerge.generateSyncMessage(n1, s1) assert.deepStrictEqual(decodeSyncMessage(msg1to2).changes.length, 0) ;[s2, msg2to1] = Automerge.generateSyncMessage(n2, s2) assert.deepStrictEqual(decodeSyncMessage(msg2to1).changes.length, 0) // After receiving acknowledgements, their shared heads should be equal ;[n1, s1, patch1] = Automerge.receiveSyncMessage(n1, s1, msg2to1) ;[n2, s2, patch2] = Automerge.receiveSyncMessage(n2, s2, msg1to2) assert.deepStrictEqual(s1.sharedHeads, [head1, head2].sort()) assert.deepStrictEqual(s2.sharedHeads, [head1, head2].sort()) assert.deepStrictEqual(patch1, null) assert.deepStrictEqual(patch2, null) // We're in sync, no more messages required ;[s1, msg1to2] = Automerge.generateSyncMessage(n1, s1) ;[s2, msg2to1] = Automerge.generateSyncMessage(n2, s2) assert.deepStrictEqual(msg1to2, null) assert.deepStrictEqual(msg2to1, null) // If we make one more change, and start another sync, its lastSync should be updated n1 = Automerge.change(n1, {time: 0}, doc => doc.x = 5) ;[s1, msg1to2] = Automerge.generateSyncMessage(n1, s1) assert.deepStrictEqual(decodeSyncMessage(msg1to2).have[0].lastSync, [head1, head2].sort()) }) it('should assume sent changes were recieved until we hear otherwise', () => { let n1 = Automerge.init('01234567'), n2 = Automerge.init('89abcdef') let s1 = initSyncState(), message = null n1 = Automerge.change(n1, {time: 0}, doc => doc.items = []) ;[n1, n2, s1, /* s2 */] = sync(n1, n2) n1 = Automerge.change(n1, {time: 0}, doc => doc.items.push('x')) ;[s1, message] = Automerge.generateSyncMessage(n1, s1) assert.deepStrictEqual(decodeSyncMessage(message).changes.length, 1) n1 = Automerge.change(n1, {time: 0}, doc => doc.items.push('y')) ;[s1, message] = Automerge.generateSyncMessage(n1, s1) assert.deepStrictEqual(decodeSyncMessage(message).changes.length, 1) n1 = Automerge.change(n1, {time: 0}, doc => doc.items.push('z')) ;[s1, message] = Automerge.generateSyncMessage(n1, s1) assert.deepStrictEqual(decodeSyncMessage(message).changes.length, 1) }) it('should work regardless of who initiates the exchange', () => { // create & synchronize two nodes let n1 = Automerge.init(), n2 = Automerge.init() let s1 = initSyncState(), s2 = initSyncState() for (let i = 0; i < 5; i++) n1 = Automerge.change(n1, {time: 0}, doc => doc.x = i) ;[n1, n2, s1, s2] = sync(n1, n2, s1, s2) // modify the first node further for (let i = 5; i < 10; i++) n1 = Automerge.change(n1, {time: 0}, doc => doc.x = i) assert.notDeepStrictEqual(n1, n2) ;[n1, n2, s1, s2] = sync(n1, n2, s1, s2) assert.deepStrictEqual(n1, n2) }) }) }) describe('with diverged documents', () => { it('should work without prior sync state', () => { // Scenario: ,-- c10 <-- c11 <-- c12 <-- c13 <-- c14 // c0 <-- c1 <-- c2 <-- c3 <-- c4 <-- c5 <-- c6 <-- c7 <-- c8 <-- c9 <-+ // `-- c15 <-- c16 <-- c17 // lastSync is undefined. // create two peers both with divergent commits let n1 = Automerge.init('01234567'), n2 = Automerge.init('89abcdef') for (let i = 0; i < 10; i++) n1 = Automerge.change(n1, {time: 0}, doc => doc.x = i) ;[n1, n2] = sync(n1, n2) for (let i = 10; i < 15; i++) n1 = Automerge.change(n1, {time: 0}, doc => doc.x = i) for (let i = 15; i < 18; i++) n2 = Automerge.change(n2, {time: 0}, doc => doc.x = i) assert.notDeepStrictEqual(n1, n2) ;[n1, n2] = sync(n1, n2) assert.deepStrictEqual(getHeads(n1), getHeads(n2)) assert.deepStrictEqual(n1, n2) }) it('should work with prior sync state', () => { // Scenario: ,-- c10 <-- c11 <-- c12 <-- c13 <-- c14 // c0 <-- c1 <-- c2 <-- c3 <-- c4 <-- c5 <-- c6 <-- c7 <-- c8 <-- c9 <-+ // `-- c15 <-- c16 <-- c17 // lastSync is c9. // create two peers both with divergent commits let n1 = Automerge.init('01234567'), n2 = Automerge.init('89abcdef') let s1 = initSyncState(), s2 = initSyncState() for (let i = 0; i < 10; i++) n1 = Automerge.change(n1, {time: 0}, doc => doc.x = i) ;[n1, n2, s1, s2] = sync(n1, n2, s1, s2) for (let i = 10; i < 15; i++) n1 = Automerge.change(n1, {time: 0}, doc => doc.x = i) for (let i = 15; i < 18; i++) n2 = Automerge.change(n2, {time: 0}, doc => doc.x = i) s1 = decodeSyncState(encodeSyncState(s1)) s2 = decodeSyncState(encodeSyncState(s2)) assert.notDeepStrictEqual(n1, n2) ;[n1, n2, s1, s2] = sync(n1, n2, s1, s2) assert.deepStrictEqual(getHeads(n1), getHeads(n2)) assert.deepStrictEqual(n1, n2) }) it('should ensure non-empty state after sync', () => { let n1 = Automerge.init('01234567'), n2 = Automerge.init('89abcdef') let s1 = initSyncState(), s2 = initSyncState() for (let i = 0; i < 3; i++) n1 = Automerge.change(n1, {time: 0}, doc => doc.x = i) ;[n1, n2, s1, s2] = sync(n1, n2, s1, s2) assert.deepStrictEqual(s1.sharedHeads, getHeads(n1)) assert.deepStrictEqual(s2.sharedHeads, getHeads(n1)) }) it('should re-sync after one node crashed with data loss', () => { // Scenario: (r) (n2) (n1) // c0 <-- c1 <-- c2 <-- c3 <-- c4 <-- c5 <-- c6 <-- c7 <-- c8 // n2 has changes {c0, c1, c2}, n1's lastSync is c5, and n2's lastSync is c2. // we want to successfully sync (n1) with (r), even though (n1) believes it's talking to (n2) let n1 = Automerge.init('01234567'), n2 = Automerge.init('89abcdef') let s1 = initSyncState(), s2 = initSyncState() // n1 makes three changes, which we sync to n2 for (let i = 0; i < 3; i++) n1 = Automerge.change(n1, {time: 0}, doc => doc.x = i) ;[n1, n2, s1, s2] = sync(n1, n2, s1, s2) // save a copy of n2 as "r" to simulate recovering from crash let r, rSyncState ;[r, rSyncState] = [Automerge.clone(n2), s2] // sync another few commits for (let i = 3; i < 6; i++) n1 = Automerge.change(n1, {time: 0}, doc => doc.x = i) ;[n1, n2, s1, s2] = sync(n1, n2, s1, s2) // everyone should be on the same page here assert.deepStrictEqual(getHeads(n1), getHeads(n2)) assert.deepStrictEqual(n1, n2) // now make a few more changes, then attempt to sync the fully-up-to-date n1 with the confused r for (let i = 6; i < 9; i++) n1 = Automerge.change(n1, {time: 0}, doc => doc.x = i) s1 = decodeSyncState(encodeSyncState(s1)) rSyncState = decodeSyncState(encodeSyncState(rSyncState)) assert.notDeepStrictEqual(getHeads(n1), getHeads(r)) assert.notDeepStrictEqual(n1, r) assert.deepStrictEqual(n1, {x: 8}) assert.deepStrictEqual(r, {x: 2}) ;[n1, r, s1, rSyncState] = sync(n1, r, s1, rSyncState) assert.deepStrictEqual(getHeads(n1), getHeads(r)) assert.deepStrictEqual(n1, r) }) it('should resync after one node experiences data loss without disconnecting', () => { let n1 = Automerge.init('01234567'), n2 = Automerge.init('89abcdef') let s1 = initSyncState(), s2 = initSyncState() // n1 makes three changes, which we sync to n2 for (let i = 0; i < 3; i++) n1 = Automerge.change(n1, {time: 0}, doc => doc.x = i) ;[n1, n2, s1, s2] = sync(n1, n2, s1, s2) assert.deepStrictEqual(getHeads(n1), getHeads(n2)) assert.deepStrictEqual(n1, n2) let n2AfterDataLoss = Automerge.init('89abcdef') // "n2" now has no data, but n1 still thinks it does. Note we don't do // decodeSyncState(encodeSyncState(s1)) in order to simulate data loss without disconnecting ;[n1, n2, s1, s2] = sync(n1, n2AfterDataLoss, s1, initSyncState()) assert.deepStrictEqual(getHeads(n1), getHeads(n2)) assert.deepStrictEqual(n1, n2) }) it('should handle changes concurrent to the last sync heads', () => { let n1 = Automerge.init('01234567'), n2 = Automerge.init('89abcdef'), n3 = Automerge.init('fedcba98') let s12 = initSyncState(), s21 = initSyncState(), s23 = initSyncState(), s32 = initSyncState() // Change 1 is known to all three nodes n1 = Automerge.change(n1, {time: 0}, doc => doc.x = 1) ;[n1, n2, s12, s21] = sync(n1, n2, s12, s21) ;[n2, n3, s23, s32] = sync(n2, n3, s23, s32) // Change 2 is known to n1 and n2 n1 = Automerge.change(n1, {time: 0}, doc => doc.x = 2) ;[n1, n2, s12, s21] = sync(n1, n2, s12, s21) // Each of the three nodes makes one change (changes 3, 4, 5) n1 = Automerge.change(n1, {time: 0}, doc => doc.x = 3) n2 = Automerge.change(n2, {time: 0}, doc => doc.x = 4) n3 = Automerge.change(n3, {time: 0}, doc => doc.x = 5) // Apply n3's latest change to n2. If running in Node, turn the Uint8Array into a Buffer, to // simulate transmission over a network (see https://github.com/automerge/automerge/pull/362) let change = Automerge.getLastLocalChange(n3) if (typeof Buffer === 'function') change = Buffer.from(change) ;[n2] = Automerge.applyChanges(n2, [change]) // Now sync n1 and n2. n3's change is concurrent to n1 and n2's last sync heads ;[n1, n2, s12, s21] = sync(n1, n2, s12, s21) assert.deepStrictEqual(getHeads(n1), getHeads(n2)) assert.deepStrictEqual(n1, n2) }) it('should handle histories with lots of branching and merging', () => { let n1 = Automerge.init('01234567'), n2 = Automerge.init('89abcdef'), n3 = Automerge.init('fedcba98') n1 = Automerge.change(n1, {time: 0}, doc => doc.x = 0) ;[n2] = Automerge.applyChanges(n2, [Automerge.getLastLocalChange(n1)]) ;[n3] = Automerge.applyChanges(n3, [Automerge.getLastLocalChange(n1)]) n3 = Automerge.change(n3, {time: 0}, doc => doc.x = 1) // - n1c1 <------ n1c2 <------ n1c3 <-- etc. <-- n1c20 <------ n1c21 // / \/ \/ \/ // / /\ /\ /\ // c0 <---- n2c1 <------ n2c2 <------ n2c3 <-- etc. <-- n2c20 <------ n2c21 // \ / // ---------------------------------------------- n3c1 <----- for (let i = 1; i < 20; i++) { n1 = Automerge.change(n1, {time: 0}, doc => doc.n1 = i) n2 = Automerge.change(n2, {time: 0}, doc => doc.n2 = i) const change1 = Automerge.getLastLocalChange(n1) const change2 = Automerge.getLastLocalChange(n2) ;[n1] = Automerge.applyChanges(n1, [change2]) ;[n2] = Automerge.applyChanges(n2, [change1]) } let s1 = initSyncState(), s2 = initSyncState() ;[n1, n2, s1, s2] = sync(n1, n2, s1, s2) // Having n3's last change concurrent to the last sync heads forces us into the slower code path ;[n2] = Automerge.applyChanges(n2, [Automerge.getLastLocalChange(n3)]) n1 = Automerge.change(n1, {time: 0}, doc => doc.n1 = 'final') n2 = Automerge.change(n2, {time: 0}, doc => doc.n2 = 'final') ;[n1, n2, s1, s2] = sync(n1, n2, s1, s2) assert.deepStrictEqual(getHeads(n1), getHeads(n2)) assert.deepStrictEqual(n1, n2) }) }) describe('with false positives', () => { // NOTE: the following tests use brute force to search for Bloom filter false positives. The // tests make change hashes deterministic by fixing the actorId and change timestamp to be // constants. The loop that searches for false positives is then initialised such that it finds // a false positive on its first iteration. However, if anything changes about the encoding of // changes (causing their hashes to change) or if the Bloom filter configuration is changed, // then the false positive will no longer be the first loop iteration. The tests should still // pass because the loop will run until a false positive is found, but they will be slower. it('should handle a false-positive head', () => { // Scenario: ,-- n1 // c0 <-- c1 <-- c2 <-- c3 <-- c4 <-- c5 <-- c6 <-- c7 <-- c8 <-- c9 <-+ // `-- n2 // where n2 is a false positive in the Bloom filter containing {n1}. // lastSync is c9. let n1 = Automerge.init('01234567'), n2 = Automerge.init('89abcdef') let s1 = initSyncState(), s2 = initSyncState() for (let i = 0; i < 10; i++) n1 = Automerge.change(n1, {time: 0}, doc => doc.x = i) ;[n1, n2, s1, s2] = sync(n1, n2) for (let i = 1; ; i++) { // search for false positive; see comment above const n1up = Automerge.change(Automerge.clone(n1, {actorId: '01234567'}), {time: 0}, doc => doc.x = `${i} @ n1`) const n2up = Automerge.change(Automerge.clone(n2, {actorId: '89abcdef'}), {time: 0}, doc => doc.x = `${i} @ n2`) if (new BloomFilter(getHeads(n1up)).containsHash(getHeads(n2up)[0])) { n1 = n1up; n2 = n2up; break } } const allHeads = [...getHeads(n1), ...getHeads(n2)].sort() s1 = decodeSyncState(encodeSyncState(s1)) s2 = decodeSyncState(encodeSyncState(s2)) ;[n1, n2, s1, s2] = sync(n1, n2, s1, s2) assert.deepStrictEqual(getHeads(n1), allHeads) assert.deepStrictEqual(getHeads(n2), allHeads) }) describe('with a false-positive dependency', () => { let n1, n2, s1, s2, n1hash2, n2hash2 beforeEach(() => { // Scenario: ,-- n1c1 <-- n1c2 // c0 <-- c1 <-- c2 <-- c3 <-- c4 <-- c5 <-- c6 <-- c7 <-- c8 <-- c9 <-+ // `-- n2c1 <-- n2c2 // where n2c1 is a false positive in the Bloom filter containing {n1c1, n1c2}. // lastSync is c9. n1 = Automerge.init('01234567') n2 = Automerge.init('89abcdef') s1 = initSyncState() s2 = initSyncState() for (let i = 0; i < 10; i++) n1 = Automerge.change(n1, {time: 0}, doc => doc.x = i) ;[n1, n2, s1, s2] = sync(n1, n2) let n1hash1, n2hash1 for (let i = 29; ; i++) { // search for false positive; see comment above const n1us1 = Automerge.change(Automerge.clone(n1, {actorId: '01234567'}), {time: 0}, doc => doc.x = `${i} @ n1`) const n2us1 = Automerge.change(Automerge.clone(n2, {actorId: '89abcdef'}), {time: 0}, doc => doc.x = `${i} @ n2`) n1hash1 = getHeads(n1us1)[0]; n2hash1 = getHeads(n2us1)[0] const n1us2 = Automerge.change(n1us1, {time: 0}, doc => doc.x = 'final @ n1') const n2us2 = Automerge.change(n2us1, {time: 0}, doc => doc.x = 'final @ n2') n1hash2 = getHeads(n1us2)[0]; n2hash2 = getHeads(n2us2)[0] if (new BloomFilter([n1hash1, n1hash2]).containsHash(n2hash1)) { n1 = n1us2; n2 = n2us2; break } } }) it('should sync two nodes without connection reset', () => { [n1, n2, s1, s2] = sync(n1, n2, s1, s2) assert.deepStrictEqual(getHeads(n1), [n1hash2, n2hash2].sort()) assert.deepStrictEqual(getHeads(n2), [n1hash2, n2hash2].sort()) }) it('should sync two nodes with connection reset', () => { s1 = decodeSyncState(encodeSyncState(s1)) s2 = decodeSyncState(encodeSyncState(s2)) ;[n1, n2, s1, s2] = sync(n1, n2, s1, s2) assert.deepStrictEqual(getHeads(n1), [n1hash2, n2hash2].sort()) assert.deepStrictEqual(getHeads(n2), [n1hash2, n2hash2].sort()) }) it('should sync three nodes', () => { s1 = decodeSyncState(encodeSyncState(s1)) s2 = decodeSyncState(encodeSyncState(s2)) // First n1 and n2 exchange Bloom filters let m1, m2 ;[s1, m1] = Automerge.generateSyncMessage(n1, s1) ;[s2, m2] = Automerge.generateSyncMessage(n2, s2) ;[n1, s1] = Automerge.receiveSyncMessage(n1, s1, m2) ;[n2, s2] = Automerge.receiveSyncMessage(n2, s2, m1) // Then n1 and n2 send each other their changes, except for the false positive ;[s1, m1] = Automerge.generateSyncMessage(n1, s1) ;[s2, m2] = Automerge.generateSyncMessage(n2, s2) ;[n1, s1] = Automerge.receiveSyncMessage(n1, s1, m2) ;[n2, s2] = Automerge.receiveSyncMessage(n2, s2, m1) assert.strictEqual(decodeSyncMessage(m1).changes.length, 2) // n1c1 and n1c2 assert.strictEqual(decodeSyncMessage(m2).changes.length, 1) // only n2c2; change n2c1 is not sent // n3 is a node that doesn't have the missing change. Nevertheless n1 is going to ask n3 for it let n3 = Automerge.init('fedcba98'), s13 = initSyncState(), s31 = initSyncState() ;[n1, n3, s13, s31] = sync(n1, n3, s13, s31) assert.deepStrictEqual(getHeads(n1), [n1hash2]) assert.deepStrictEqual(getHeads(n3), [n1hash2]) }) }) it('should not require an additional request when a false-positive depends on a true-negative', () => { // Scenario: ,-- n1c1 <-- n1c2 <-- n1c3 // c0 <-- c1 <-- c2 <-- c3 <-- c4 <-+ // `-- n2c1 <-- n2c2 <-- n2c3 // where n2c2 is a false positive in the Bloom filter containing {n1c1, n1c2, n1c3}. // lastSync is c4. let n1 = Automerge.init('01234567'), n2 = Automerge.init('89abcdef') let s1 = initSyncState(), s2 = initSyncState() let n1hash3, n2hash3 for (let i = 0; i < 5; i++) n1 = Automerge.change(n1, {time: 0}, doc => doc.x = i) ;[n1, n2, s1, s2] = sync(n1, n2) for (let i = 86; ; i++) { // search for false positive; see comment above const n1us1 = Automerge.change(Automerge.clone(n1, {actorId: '01234567'}), {time: 0}, doc => doc.x = `${i} @ n1`) const n2us1 = Automerge.change(Automerge.clone(n2, {actorId: '89abcdef'}), {time: 0}, doc => doc.x = `${i} @ n2`) const n1hash1 = getHeads(n1us1)[0] const n1us2 = Automerge.change(n1us1, {time: 0}, doc => doc.x = `${i + 1} @ n1`) const n2us2 = Automerge.change(n2us1, {time: 0}, doc => doc.x = `${i + 1} @ n2`) const n1hash2 = getHeads(n1us2)[0], n2hash2 = getHeads(n2us2)[0] const n1up3 = Automerge.change(n1us2, {time: 0}, doc => doc.x = 'final @ n1') const n2up3 = Automerge.change(n2us2, {time: 0}, doc => doc.x = 'final @ n2') n1hash3 = getHeads(n1up3)[0]; n2hash3 = getHeads(n2up3)[0] if (new BloomFilter([n1hash1, n1hash2, n1hash3]).containsHash(n2hash2)) { n1 = n1up3; n2 = n2up3; break } } const bothHeads = [n1hash3, n2hash3].sort() s1 = decodeSyncState(encodeSyncState(s1)) s2 = decodeSyncState(encodeSyncState(s2)) ;[n1, n2, s1, s2] = sync(n1, n2, s1, s2) assert.deepStrictEqual(getHeads(n1), bothHeads) assert.deepStrictEqual(getHeads(n2), bothHeads) }) it('should handle chains of false-positives', () => { // Scenario: ,-- c5 // c0 <-- c1 <-- c2 <-- c3 <-- c4 <-+ // `-- n2c1 <-- n2c2 <-- n2c3 // where n2c1 and n2c2 are both false positives in the Bloom filter containing {c5}. // lastSync is c4. let n1 = Automerge.init('01234567'), n2 = Automerge.init('89abcdef') let s1 = initSyncState(), s2 = initSyncState() for (let i = 0; i < 5; i++) n1 = Automerge.change(n1, {time: 0}, doc => doc.x = i) ;[n1, n2, s1, s2] = sync(n1, n2, s1, s2) n1 = Automerge.change(n1, {time: 0}, doc => doc.x = 5) for (let i = 2; ; i++) { // search for false positive; see comment above const n2us1 = Automerge.change(Automerge.clone(n2, {actorId: '89abcdef'}), {time: 0}, doc => doc.x = `${i} @ n2`) if (new BloomFilter(getHeads(n1)).containsHash(getHeads(n2us1)[0])) { n2 = n2us1; break } } for (let i = 141; ; i++) { // search for false positive; see comment above const n2us2 = Automerge.change(Automerge.clone(n2, {actorId: '89abcdef'}), {time: 0}, doc => doc.x = `${i} again`) if (new BloomFilter(getHeads(n1)).containsHash(getHeads(n2us2)[0])) { n2 = n2us2; break } } n2 = Automerge.change(n2, {time: 0}, doc => doc.x = 'final @ n2') const allHeads = [...getHeads(n1), ...getHeads(n2)].sort() s1 = decodeSyncState(encodeSyncState(s1)) s2 = decodeSyncState(encodeSyncState(s2)) ;[n1, n2, s1, s2] = sync(n1, n2, s1, s2) assert.deepStrictEqual(getHeads(n1), allHeads) assert.deepStrictEqual(getHeads(n2), allHeads) }) it('should allow the false-positive hash to be explicitly requested', () => { // Scenario: ,-- n1 // c0 <-- c1 <-- c2 <-- c3 <-- c4 <-- c5 <-- c6 <-- c7 <-- c8 <-- c9 <-+ // `-- n2 // where n2 causes a false positive in the Bloom filter containing {n1}. let n1 = Automerge.init('01234567'), n2 = Automerge.init('89abcdef') let s1 = initSyncState(), s2 = initSyncState() let message for (let i = 0; i < 10; i++) n1 = Automerge.change(n1, {time: 0}, doc => doc.x = i) ;[n1, n2, s1, s2] = sync(n1, n2) s1 = decodeSyncState(encodeSyncState(s1)) s2 = decodeSyncState(encodeSyncState(s2)) for (let i = 1; ; i++) { // brute-force search for false positive; see comment above const n1up = Automerge.change(Automerge.clone(n1, {actorId: '01234567'}), {time: 0}, doc => doc.x = `${i} @ n1`) const n2up = Automerge.change(Automerge.clone(n2, {actorId: '89abcdef'}), {time: 0}, doc => doc.x = `${i} @ n2`) // check if the bloom filter on n2 will believe n1 already has a particular hash // this will mean n2 won't offer that data to n2 by receiving a sync message from n1 if (new BloomFilter(getHeads(n1up)).containsHash(getHeads(n2up)[0])) { n1 = n1up; n2 = n2up; break } } // n1 creates a sync message for n2 with an ill-fated bloom [s1, message] = Automerge.generateSyncMessage(n1, s1) assert.strictEqual(decodeSyncMessage(message).changes.length, 0) // n2 receives it and DOESN'T send a change back ;[n2, s2] = Automerge.receiveSyncMessage(n2, s2, message) ;[s2, message] = Automerge.generateSyncMessage(n2, s2) assert.strictEqual(decodeSyncMessage(message).changes.length, 0) // n1 should now realize it's missing that change and request it explicitly ;[n1, s1] = Automerge.receiveSyncMessage(n1, s1, message) ;[s1, message] = Automerge.generateSyncMessage(n1, s1) assert.deepStrictEqual(decodeSyncMessage(message).need, getHeads(n2)) // n2 should fulfill that request ;[n2, s2] = Automerge.receiveSyncMessage(n2, s2, message) ;[s2, message] = Automerge.generateSyncMessage(n2, s2) assert.strictEqual(decodeSyncMessage(message).changes.length, 1) // n1 should apply the change and the two should now be in sync ;[n1, s1] = Automerge.receiveSyncMessage(n1, s1, message) assert.deepStrictEqual(getHeads(n1), getHeads(n2)) }) }) describe('protocol features', () => { it('should allow multiple Bloom filters', () => { // Scenario: ,-- n1c1 <-- n1c2 <-- n1c3 // c0 <-- c1 <-- c2 <-+--- n2c1 <-- n2c2 <-- n2c3 // `-- n3c1 <-- n3c2 <-- n3c3 // n1 has {c0, c1, c2, n1c1, n1c2, n1c3, n2c1, n2c2}; // n2 has {c0, c1, c2, n1c1, n1c2, n2c1, n2c2, n2c3}; // n3 has {c0, c1, c2, n3c1, n3c2, n3c3}. let n1 = Automerge.init('01234567'), n2 = Automerge.init('89abcdef'), n3 = Automerge.init('76543210') let s13 = initSyncState(), s12 = initSyncState(), s21 = initSyncState() let s32 = initSyncState(), s31 = initSyncState(), s23 = initSyncState() let message1, message2, message3 for (let i = 0; i < 3; i++) n1 = Automerge.change(n1, {time: 0}, doc => doc.x = i) // sync all 3 nodes ;[n1, n2, s12, s21] = sync(n1, n2) // eslint-disable-line no-unused-vars -- kept for consistency ;[n1, n3, s13, s31] = sync(n1, n3) ;[n3, n2, s32, s23] = sync(n3, n2) for (let i = 0; i < 2; i++) n1 = Automerge.change(n1, {time: 0}, doc => doc.x = `${i} @ n1`) for (let i = 0; i < 2; i++) n2 = Automerge.change(n2, {time: 0}, doc => doc.x = `${i} @ n2`) ;[n1] = Automerge.applyChanges(n1, Automerge.getAllChanges(n2)) ;[n2] = Automerge.applyChanges(n2, Automerge.getAllChanges(n1)) n1 = Automerge.change(n1, {time: 0}, doc => doc.x = `3 @ n1`) n2 = Automerge.change(n2, {time: 0}, doc => doc.x = `3 @ n2`) for (let i = 0; i < 3; i++) n3 = Automerge.change(n3, {time: 0}, doc => doc.x = `${i} @ n3`) const n1c3 = getHeads(n1)[0], n2c3 = getHeads(n2)[0], n3c3 = getHeads(n3)[0] s13 = decodeSyncState(encodeSyncState(s13)) s31 = decodeSyncState(encodeSyncState(s31)) s23 = decodeSyncState(encodeSyncState(s23)) s32 = decodeSyncState(encodeSyncState(s32)) // Now n3 concurrently syncs with n1 and n2. Doing this naively would result in n3 receiving // changes {n1c1, n1c2, n2c1, n2c2} twice (those are the changes that both n1 and n2 have, but // that n3 does not have). We want to prevent this duplication. ;[s13, message1] = Automerge.generateSyncMessage(n1, s13) // message from n1 to n3 assert.strictEqual(decodeSyncMessage(message1).changes.length, 0) ;[n3, s31] = Automerge.receiveSyncMessage(n3, s31, message1) ;[s31, message3] = Automerge.generateSyncMessage(n3, s31) // message from n3 to n1 assert.strictEqual(decodeSyncMessage(message3).changes.length, 3) // {n3c1, n3c2, n3c3} ;[n1, s13] = Automerge.receiveSyncMessage(n1, s13, message3) // Copy the Bloom filter received from n1 into the message sent from n3 to n2. This Bloom // filter indicates what changes n3 is going to receive from n1. ;[s32, message3] = Automerge.generateSyncMessage(n3, s32) // message from n3 to n2 const modifiedMessage = decodeSyncMessage(message3) modifiedMessage.have.push(decodeSyncMessage(message1).have[0]) assert.strictEqual(modifiedMessage.changes.length, 0) ;[n2, s23] = Automerge.receiveSyncMessage(n2, s23, encodeSyncMessage(modifiedMessage)) // n2 replies to n3, sending only n2c3 (the one change that n2 has but n1 doesn't) ;[s23, message2] = Automerge.generateSyncMessage(n2, s23) assert.strictEqual(decodeSyncMessage(message2).changes.length, 1) // {n2c3} ;[n3, s32] = Automerge.receiveSyncMessage(n3, s32, message2) // n1 replies to n3 ;[s13, message1] = Automerge.generateSyncMessage(n1, s13) assert.strictEqual(decodeSyncMessage(message1).changes.length, 5) // {n1c1, n1c2, n1c3, n2c1, n2c2} ;[n3, s31] = Automerge.receiveSyncMessage(n3, s31, message1) assert.deepStrictEqual(getHeads(n3), [n1c3, n2c3, n3c3].sort()) }) it('should allow any change to be requested', () => { let n1 = Automerge.init('01234567'), n2 = Automerge.init('89abcdef') let s1 = initSyncState(), s2 = initSyncState() let message = null for (let i = 0; i < 3; i++) n1 = Automerge.change(n1, {time: 0}, doc => doc.x = i) const lastSync = getHeads(n1) for (let i = 3; i < 6; i++) n1 = Automerge.change(n1, {time: 0}, doc => doc.x = i) ;[n1, n2, s1, s2] = sync(n1, n2) s1.lastSentHeads = [] // force generateSyncMessage to return a message even though nothing changed ;[s1, message] = Automerge.generateSyncMessage(n1, s1) const modMsg = decodeSyncMessage(message) modMsg.need = lastSync // re-request change 2 ;[n2, s2] = Automerge.receiveSyncMessage(n2, s2, encodeSyncMessage(modMsg)) ;[s1, message] = Automerge.generateSyncMessage(n2, s2) assert.strictEqual(decodeSyncMessage(message).changes.length, 1) assert.strictEqual(Automerge.decodeChange(decodeSyncMessage(message).changes[0]).hash, lastSync[0]) }) it('should ignore requests for a nonexistent change', () => { let n1 = Automerge.init('01234567'), n2 = Automerge.init('89abcdef') let s1 = initSyncState(), s2 = initSyncState() let message = null for (let i = 0; i < 3; i++) n1 = Automerge.change(n1, {time: 0}, doc => doc.x = i) ;[n2] = Automerge.applyChanges(n2, Automerge.getAllChanges(n1)) ;[s1, message] = Automerge.generateSyncMessage(n1, s1) message.need = ['0000000000000000000000000000000000000000000000000000000000000000'] ;[n2, s2] = Automerge.receiveSyncMessage(n2, s2, message) ;[s2, message] = Automerge.generateSyncMessage(n2, s2) assert.strictEqual(message, null) }) it('should allow a subset of changes to be sent', () => { // ,-- c1 <-- c2 // c0 <-+ // `-- c3 <-- c4 <-- c5 <-- c6 <-- c7 <-- c8 let n1 = Automerge.init('01234567'), n2 = Automerge.init('89abcdef'), n3 = Automerge.init('76543210') let s1 = initSyncState(), s2 = initSyncState() let msg, decodedMsg n1 = Automerge.change(n1, {time: 0}, doc => doc.x = 0) n3 = Automerge.merge(n3, n1) for (let i = 1; i <= 2; i++) n1 = Automerge.change(n1, {time: 0}, doc => doc.x = i) // n1 has {c0, c1, c2} for (let i = 3; i <= 4; i++) n3 = Automerge.change(n3, {time: 0}, doc => doc.x = i) // n3 has {c0, c3, c4} const c2 = getHeads(n1)[0], c4 = getHeads(n3)[0] n2 = Automerge.merge(n2, n3) // n2 has {c0, c3, c4} // Sync n1 and n2, so their shared heads are {c2, c4} ;[n1, n2, s1, s2] = sync(n1, n2) s1 = decodeSyncState(encodeSyncState(s1)) s2 = decodeSyncState(encodeSyncState(s2)) assert.deepStrictEqual(s1.sharedHeads, [c2, c4].sort()) assert.deepStrictEqual(s2.sharedHeads, [c2, c4].sort()) // n2 and n3 apply {c5, c6, c7, c8} n3 = Automerge.change(n3, {time: 0}, doc => doc.x = 5) const change5 = Automerge.getLastLocalChange(n3) n3 = Automerge.change(n3, {time: 0}, doc => doc.x = 6) const change6 = Automerge.getLastLocalChange(n3), c6 = getHeads(n3)[0] for (let i = 7; i <= 8; i++) n3 = Automerge.change(n3, {time: 0}, doc => doc.x = i) const c8 = getHeads(n3)[0] n2 = Automerge.merge(n2, n3) // Now n1 initiates a sync with n2, and n2 replies with {c5, c6}. n2 does not send {c7, c8} ;[s1, msg] = Automerge.generateSyncMessage(n1, s1) ;[n2, s2] = Automerge.receiveSyncMessage(n2, s2, msg) ;[s2, msg] = Automerge.generateSyncMessage(n2, s2) decodedMsg = decodeSyncMessage(msg) decodedMsg.changes = [change5, change6] msg = encodeSyncMessage(decodedMsg) const sentHashes = {} sentHashes[decodeChangeMeta(change5, true).hash] = true sentHashes[decodeChangeMeta(change6, true).hash] = true s2.sentHashes = sentHashes ;[n1, s1] = Automerge.receiveSyncMessage(n1, s1, msg) assert.deepStrictEqual(s1.sharedHeads, [c2, c6].sort()) // n1 replies, confirming the receipt of {c5, c6} and requesting the remaining changes ;[s1, msg] = Automerge.generateSyncMessage(n1, s1) ;[n2, s2] = Automerge.receiveSyncMessage(n2, s2, msg) assert.deepStrictEqual(decodeSyncMessage(msg).need, [c8]) assert.deepStrictEqual(decodeSyncMessage(msg).have[0].lastSync, [c2, c6].sort()) assert.deepStrictEqual(s1.sharedHeads, [c2, c6].sort()) assert.deepStrictEqual(s2.sharedHeads, [c2, c6].sort()) // n2 sends the remaining changes {c7, c8} ;[s2, msg] = Automerge.generateSyncMessage(n2, s2) ;[n1, s1] = Automerge.receiveSyncMessage(n1, s1, msg) assert.strictEqual(decodeSyncMessage(msg).changes.length, 2) assert.deepStrictEqual(s1.sharedHeads, [c2, c8].sort()) }) }) }) ================================================ FILE: test/table_test.js ================================================ const assert = require('assert') const Automerge = process.env.TEST_DIST === '1' ? require('../dist/automerge') : require('../src/automerge') const Frontend = Automerge.Frontend const uuid = require('../src/uuid') const { assertEqualsOneOf } = require('./helpers') // Example data const DDIA = { authors: ['Kleppmann, Martin'], title: 'Designing Data-Intensive Applications', isbn: '1449373321' } const RSDP = { authors: ['Cachin, Christian', 'Guerraoui, Rachid', 'Rodrigues, Luís'], title: 'Introduction to Reliable and Secure Distributed Programming', isbn: '3-642-15259-7' } describe('Automerge.Table', () => { describe('Frontend', () => { it('should generate ops to create a table', () => { const actor = uuid() const [, change] = Frontend.change(Frontend.init(actor), doc => { doc.books = new Automerge.Table() }) assert.deepStrictEqual(change, { actor, seq: 1, time: change.time, message: '', startOp: 1, deps: [], ops: [ {obj: '_root', action: 'makeTable', key: 'books', insert: false, pred: []} ] }) }) it('should generate ops to insert a row', () => { const actor = uuid() const [doc1] = Frontend.change(Frontend.init(actor), doc => { doc.books = new Automerge.Table() }) let rowId const [doc2, change2] = Frontend.change(doc1, doc => { rowId = doc.books.add({authors: 'Kleppmann, Martin', title: 'Designing Data-Intensive Applications'}) }) const books = Frontend.getObjectId(doc2.books) const rowObjID = Frontend.getObjectId(doc2.books.entries[rowId]) assert.deepStrictEqual(change2, { actor, seq: 2, time: change2.time, message: '', startOp: 2, deps: [], ops: [ {obj: books, action: 'makeMap', key: rowId, insert: false, pred: []}, {obj: rowObjID, action: 'set', key: 'authors', insert: false, value: 'Kleppmann, Martin', pred: []}, {obj: rowObjID, action: 'set', key: 'title', insert: false, value: 'Designing Data-Intensive Applications', pred: []} ] }) }) }) describe('with one row', () => { let s1, rowId, rowWithId beforeEach(() => { s1 = Automerge.change(Automerge.init({freeze: true}), doc => { doc.books = new Automerge.Table() rowId = doc.books.add(DDIA) }) rowWithId = Object.assign({id: rowId}, DDIA) }) it('should look up a row by ID', () => { const row = s1.books.byId(rowId) assert.deepStrictEqual(row, rowWithId) }) it('should return the row count', () => { assert.strictEqual(s1.books.count, 1) }) it('should return a list of row IDs', () => { assert.deepStrictEqual(s1.books.ids, [rowId]) }) it('should allow iterating over rows', () => { assert.deepStrictEqual([...s1.books], [rowWithId]) }) it('should support standard array methods', () => { assert.deepStrictEqual(s1.books.filter(book => book.isbn === '1449373321'), [rowWithId]) assert.deepStrictEqual(s1.books.filter(book => book.isbn === '9781449373320'), []) assert.deepStrictEqual(s1.books.find(book => book.isbn === '1449373321'), rowWithId) assert.strictEqual(s1.books.find(book => book.isbn === '9781449373320'), undefined) assert.deepStrictEqual(s1.books.map(book => book.title), ['Designing Data-Intensive Applications']) }) it('should be immutable', () => { assert.strictEqual(s1.books.add, undefined) assert.throws(() => s1.books.remove(rowId), /can only be modified in a change function/) }) it('should save and reload', () => { // FIXME - the bug is in parseAllOpIds() // maps and tables with a string key that has an `@` gets // improperly encoded as an opId const s2 = Automerge.load(Automerge.save(s1)) assert.deepStrictEqual(s2.books.byId(rowId), rowWithId) }) it('should allow a row to be updated', () => { const s2 = Automerge.change(s1, doc => { doc.books.byId(rowId).isbn = '9781449373320' }) assert.deepStrictEqual(s2.books.byId(rowId), { id: rowId, authors: ['Kleppmann, Martin'], title: 'Designing Data-Intensive Applications', isbn: '9781449373320' }) }) it('should allow a row to be removed', () => { const s2 = Automerge.change(s1, doc => { doc.books.remove(rowId) }) assert.strictEqual(s2.books.count, 0) assert.deepStrictEqual([...s2.books], []) }) it('should not allow a row ID to be specified', () => { assert.throws(() => { Automerge.change(s1, doc => { doc.books.add(Object.assign({id: 'beafbfde-8e44-4a5f-b679-786e2ebba03f'}, RSDP)) }) }, /A table row must not have an "id" property/) }) it('should not allow a row ID to be modified', () => { assert.throws(() => { Automerge.change(s1, doc => { doc.books.byId(rowId).id = 'beafbfde-8e44-4a5f-b679-786e2ebba03f' }) }, /Object property "id" cannot be modified/) }) }) it('should allow concurrent row insertion', () => { const a0 = Automerge.change(Automerge.init(), doc => { doc.books = new Automerge.Table() }) const b0 = Automerge.merge(Automerge.init(), a0) let ddia, rsdp const a1 = Automerge.change(a0, doc => { ddia = doc.books.add(DDIA) }) const b1 = Automerge.change(b0, doc => { rsdp = doc.books.add(RSDP) }) const a2 = Automerge.merge(a1, b1) assert.deepStrictEqual(a2.books.byId(ddia), Object.assign({id: ddia}, DDIA)) assert.deepStrictEqual(a2.books.byId(rsdp), Object.assign({id: rsdp}, RSDP)) assert.strictEqual(a2.books.count, 2) assertEqualsOneOf(a2.books.ids, [ddia, rsdp], [rsdp, ddia]) }) it('should allow row creation, update, and deletion in the same change', () => { const doc = Automerge.change(Automerge.init(), doc => { doc.table = new Automerge.Table() const id = doc.table.add({}) doc.table.byId(id).x = 3 doc.table.remove(id) }) assert.strictEqual(doc.table.count, 0) }) it('should allow rows to be sorted in various ways', () => { let ddia, rsdp const s = Automerge.change(Automerge.init(), doc => { doc.books = new Automerge.Table() ddia = doc.books.add(DDIA) rsdp = doc.books.add(RSDP) }) const ddiaWithId = Object.assign({id: ddia}, DDIA) const rsdpWithId = Object.assign({id: rsdp}, RSDP) assert.deepStrictEqual(s.books.sort('title'), [ddiaWithId, rsdpWithId]) assert.deepStrictEqual(s.books.sort(['authors', 'title']), [rsdpWithId, ddiaWithId]) assert.deepStrictEqual(s.books.sort(row1 => ((row1.isbn === '1449373321') ? -1 : +1)), [ddiaWithId, rsdpWithId]) }) it('should allow serialization to JSON', () => { let ddia const s = Automerge.change(Automerge.init(), doc => { doc.books = new Automerge.Table() ddia = doc.books.add(DDIA) }) const ddiaWithId = Object.assign({id: ddia}, DDIA) assert.deepStrictEqual(JSON.parse(JSON.stringify(s)), {books: {[ddia]: ddiaWithId}}) }) }) ================================================ FILE: test/test.js ================================================ const assert = require('assert') const Automerge = process.env.TEST_DIST === '1' ? require('../dist/automerge') : require('../src/automerge') const { assertEqualsOneOf } = require('./helpers') const { decodeChange } = require('../backend/columnar') const UUID_PATTERN = /^[0-9a-f]{32}$/ const OPID_PATTERN = /^[0-9]+@[0-9a-f]{32}$/ describe('Automerge', () => { describe('initialization ', () => { it('should initially be an empty map', () => { const doc = Automerge.init() assert.deepStrictEqual(doc, {}) }) it('should allow instantiating from an existing object', () => { const initialState = { birds: { wrens: 3, magpies: 4 } } const doc = Automerge.from(initialState) assert.deepStrictEqual(doc, initialState) }) it('should allow merging of an object initialized with `from`', () => { let doc1 = Automerge.from({ cards: [] }) let doc2 = Automerge.merge(Automerge.init(), doc1) assert.deepStrictEqual(doc2, { cards: [] }) }) it('should allow passing an actorId when instantiating from an existing object', () => { const actorId = '1234' let doc = Automerge.from({ foo: 1 }, actorId) assert.strictEqual(Automerge.getActorId(doc), '1234') }) it('accepts an empty object as initial state', () => { const doc = Automerge.from({}) assert.deepStrictEqual(doc, {}) }) it('accepts an array as initial state, but converts it to an object', () => { const doc = Automerge.from(['a', 'b', 'c']) assert.deepStrictEqual(doc, { '0': 'a', '1': 'b', '2': 'c' }) }) it('accepts strings as initial values, but treats them as an array of characters', () => { const doc = Automerge.from('abc') assert.deepStrictEqual(doc, { '0': 'a', '1': 'b', '2': 'c' }) }) it('ignores numbers provided as initial values', () => { const doc = Automerge.from(123) assert.deepStrictEqual(doc, {}) }) it('ignores booleans provided as initial values', () => { const doc1 = Automerge.from(false) assert.deepStrictEqual(doc1, {}) const doc2 = Automerge.from(true) assert.deepStrictEqual(doc2, {}) }) }) describe('sequential use', () => { let s1, s2 beforeEach(() => { s1 = Automerge.init() }) it('should not mutate objects', () => { s2 = Automerge.change(s1, doc => doc.foo = 'bar') assert.strictEqual(s1.foo, undefined) assert.strictEqual(s2.foo, 'bar') }) it('changes should be retrievable', () => { const change1 = Automerge.getLastLocalChange(s1) s2 = Automerge.change(s1, doc => doc.foo = 'bar') const change2 = Automerge.getLastLocalChange(s2) assert.strictEqual(change1, null) const change = decodeChange(change2) assert.deepStrictEqual(change, { actor: change.actor, deps: [], seq: 1, startOp: 1, hash: change.hash, message: '', time: change.time, ops: [{obj: '_root', key: 'foo', action: 'set', insert: false, value: 'bar', pred: []}] }) }) it('should not register any conflicts on repeated assignment', () => { assert.strictEqual(Automerge.getConflicts(s1, 'foo'), undefined) s1 = Automerge.change(s1, 'change', doc => doc.foo = 'one') assert.strictEqual(Automerge.getConflicts(s1, 'foo'), undefined) s1 = Automerge.change(s1, 'change', doc => doc.foo = 'two') assert.strictEqual(Automerge.getConflicts(s1, 'foo'), undefined) }) describe('changes', () => { it('should group several changes', () => { s2 = Automerge.change(s1, 'change message', doc => { doc.first = 'one' assert.strictEqual(doc.first, 'one') doc.second = 'two' assert.deepStrictEqual(doc, { first: 'one', second: 'two' }) }) assert.deepStrictEqual(s1, {}) assert.deepStrictEqual(s2, {first: 'one', second: 'two'}) }) it('should freeze objects if desired', () => { s1 = Automerge.init({freeze: true}) s2 = Automerge.change(s1, doc => doc.foo = 'bar') try { s2.foo = 'lemon' } catch (e) { /* deliberately ignored */ } assert.strictEqual(s2.foo, 'bar') let deleted = false try { deleted = delete s2.foo } catch (e) { /* deliberately ignored */ } assert.strictEqual(s2.foo, 'bar') assert.strictEqual(deleted, false) Automerge.change(s2, () => { try { s2.foo = 'lemon' } catch (e) { /* deliberately ignored */ } assert.strictEqual(s2.foo, 'bar') }) assert.throws(() => { Object.assign(s2, {x: 4}) }) assert.strictEqual(s2.x, undefined) }) it('should allow repeated reading and writing of values', () => { s2 = Automerge.change(s1, 'change message', doc => { doc.value = 'a' assert.strictEqual(doc.value, 'a') doc.value = 'b' doc.value = 'c' assert.strictEqual(doc.value, 'c') }) assert.deepStrictEqual(s1, {}) assert.deepStrictEqual(s2, {value: 'c'}) }) it('should not record conflicts when writing the same field several times within one change', () => { s1 = Automerge.change(s1, 'change message', doc => { doc.value = 'a' doc.value = 'b' doc.value = 'c' }) assert.strictEqual(s1.value, 'c') assert.strictEqual(Automerge.getConflicts(s1, 'value'), undefined) }) it('should return the unchanged state object if nothing changed', () => { s2 = Automerge.change(s1, () => {}) assert.strictEqual(s2, s1) }) it('should ignore field updates that write the existing value', () => { s1 = Automerge.change(s1, doc => doc.field = 123) s2 = Automerge.change(s1, doc => doc.field = 123) assert.strictEqual(s2, s1) }) it('should not ignore field updates that resolve a conflict', () => { s2 = Automerge.merge(Automerge.init(), s1) s1 = Automerge.change(s1, doc => doc.field = 123) s2 = Automerge.change(s2, doc => doc.field = 321) s1 = Automerge.merge(s1, s2) assert.strictEqual(Object.keys(Automerge.getConflicts(s1, 'field')).length, 2) const resolved = Automerge.change(s1, doc => doc.field = s1.field) assert.notStrictEqual(resolved, s1) assert.deepStrictEqual(resolved, {field: s1.field}) assert.strictEqual(Automerge.getConflicts(resolved, 'field'), undefined) }) it('should ignore list element updates that write the existing value', () => { s1 = Automerge.change(s1, doc => doc.list = [123]) s2 = Automerge.change(s1, doc => doc.list[0] = 123) assert.strictEqual(s2, s1) }) it('should not ignore list element updates that resolve a conflict', () => { s1 = Automerge.change(s1, doc => doc.list = [1]) s2 = Automerge.merge(Automerge.init(), s1) s1 = Automerge.change(s1, doc => doc.list[0] = 123) s2 = Automerge.change(s2, doc => doc.list[0] = 321) s1 = Automerge.merge(s1, s2) assert.deepStrictEqual(Automerge.getConflicts(s1.list, 0), { [`3@${Automerge.getActorId(s1)}`]: 123, [`3@${Automerge.getActorId(s2)}`]: 321 }) const resolved = Automerge.change(s1, doc => doc.list[0] = s1.list[0]) assert.deepStrictEqual(resolved, s1) assert.notStrictEqual(resolved, s1) assert.strictEqual(Automerge.getConflicts(resolved.list, 0), undefined) }) it('should sanity-check arguments', () => { s1 = Automerge.change(s1, doc => doc.nested = {}) assert.throws(() => { Automerge.change({}, doc => doc.foo = 'bar') }, /must be the document root/) assert.throws(() => { Automerge.change(s1.nested, doc => doc.foo = 'bar') }, /must be the document root/) }) it('should not allow nested change blocks', () => { assert.throws(() => { Automerge.change(s1, doc1 => { Automerge.change(doc1, doc2 => { doc2.foo = 'bar' }) }) }, /Calls to Automerge.change cannot be nested/) assert.throws(() => { s1 = Automerge.change(s1, doc1 => { s2 = Automerge.change(s1, doc2 => doc2.two = 2) doc1.one = 1 }) }, /Attempting to use an outdated Automerge document/) }) it('should not allow the same base document to be used for multiple changes', () => { assert.throws(() => { Automerge.change(s1, doc => doc.one = 1) Automerge.change(s1, doc => doc.two = 2) }, /Attempting to use an outdated Automerge document/) }) it('should allow a document to be cloned', () => { s1 = Automerge.change(s1, doc => doc.zero = 0) s2 = Automerge.clone(s1) s1 = Automerge.change(s1, doc => doc.one = 1) s2 = Automerge.change(s2, doc => doc.two = 2) assert.deepStrictEqual(s1, {zero: 0, one: 1}) assert.deepStrictEqual(s2, {zero: 0, two: 2}) Automerge.free(s1) Automerge.free(s2) }) it('should apply changes to a clone', () => { s1 = Automerge.change(s1, doc => doc.x = 1) s1 = Automerge.change(s1, doc => doc.x = 2) const changes = Automerge.getAllChanges(s1) s2 = Automerge.clone(Automerge.load(Automerge.save(s1))) ;[s2] = Automerge.applyChanges(s2, changes) assert.strictEqual(s2.x, 2) }) it('should work with Object.assign merges', () => { s1 = Automerge.change(s1, doc1 => { doc1.stuff = {foo: 'bar', baz: 'blur'} }) s1 = Automerge.change(s1, doc1 => { doc1.stuff = Object.assign({}, doc1.stuff, {baz: 'updated!'}) }) assert.deepStrictEqual(s1, {stuff: {foo: 'bar', baz: 'updated!'}}) }) it('should support Date objects in maps', () => { const now = new Date() s1 = Automerge.change(s1, doc => doc.now = now) let changes = Automerge.getAllChanges(s1) ;[s2] = Automerge.applyChanges(Automerge.init(), changes) assert.strictEqual(s2.now instanceof Date, true) assert.strictEqual(s2.now.getTime(), now.getTime()) }) it('should support Date objects in lists', () => { const now = new Date() s1 = Automerge.change(s1, doc => doc.list = [now]) let changes = Automerge.getAllChanges(s1) ;[s2] = Automerge.applyChanges(Automerge.init(), changes) assert.strictEqual(s2.list[0] instanceof Date, true) assert.strictEqual(s2.list[0].getTime(), now.getTime()) }) it('should support many Date objects in lists', () => { const now1 = new Date() const now2 = new Date() const now3 = new Date() s1 = Automerge.change(s1, doc => doc.list = [now1, now2, now3]) let changes = Automerge.getAllChanges(s1) ;[s2] = Automerge.applyChanges(Automerge.init(), changes) assert.strictEqual(s2.list[0] instanceof Date, true) assert.strictEqual(s2.list[0].getTime(), now1.getTime()) assert.strictEqual(s2.list[1] instanceof Date, true) assert.strictEqual(s2.list[1].getTime(), now2.getTime()) assert.strictEqual(s2.list[2] instanceof Date, true) assert.strictEqual(s2.list[2].getTime(), now3.getTime()) }) it('should call patchCallback if supplied', () => { const callbacks = [], actor = Automerge.getActorId(s1) const s2 = Automerge.change(s1, { patchCallback: (patch, before, after, local) => callbacks.push({patch, before, after, local}) }, doc => { doc.birds = ['Goldfinch'] }) assert.strictEqual(callbacks.length, 1) assert.deepStrictEqual(callbacks[0].patch, { actor, seq: 1, maxOp: 2, deps: [], clock: {[actor]: 1}, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {birds: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'list', edits: [ {action: 'insert', index: 0, elemId: `2@${actor}`, opId: `2@${actor}`, value: {'type': 'value', value: 'Goldfinch'}} ] }}}} }) assert.strictEqual(callbacks[0].before, s1) assert.strictEqual(callbacks[0].after, s2) assert.strictEqual(callbacks[0].local, true) }) it('should call a patchCallback set up on document initialisation', () => { const callbacks = [] s1 = Automerge.init({ patchCallback: (patch, before, after, local) => callbacks.push({patch, before, after, local}) }) const s2 = Automerge.change(s1, doc => doc.bird = 'Goldfinch') const actor = Automerge.getActorId(s1) assert.strictEqual(callbacks.length, 1) assert.deepStrictEqual(callbacks[0].patch, { actor, seq: 1, maxOp: 1, deps: [], clock: {[actor]: 1}, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {bird: {[`1@${actor}`]: {type: 'value', value: 'Goldfinch'}}}} }) assert.strictEqual(callbacks[0].before, s1) assert.strictEqual(callbacks[0].after, s2) assert.strictEqual(callbacks[0].local, true) }) }) describe('emptyChange()', () => { it('should append an empty change to the history', () => { s1 = Automerge.change(s1, 'first change', doc => doc.field = 123) s2 = Automerge.emptyChange(s1, 'empty change') assert.notStrictEqual(s2, s1) assert.deepStrictEqual(s2, s1) assert.deepStrictEqual(Automerge.getHistory(s2).map(state => state.change.message), ['first change', 'empty change']) }) it('should reference dependencies', () => { s1 = Automerge.change(s1, doc => doc.field = 123) s2 = Automerge.merge(Automerge.init(), s1) s2 = Automerge.change(s2, doc => doc.other = 'hello') s1 = Automerge.emptyChange(Automerge.merge(s1, s2)) const history = Automerge.getHistory(s1) const emptyChange = history[2].change assert.deepStrictEqual(emptyChange.deps, [history[0].change.hash, history[1].change.hash].sort()) assert.deepStrictEqual(emptyChange.ops, []) }) it('should encode and decode correctly', () => { s1 = Automerge.emptyChange(s1) s1 = Automerge.change(s1, doc => doc.z = 1) s1 = Automerge.change(s1, doc => doc.z = 1000) const changes = Automerge.getAllChanges(Automerge.load(Automerge.save(s1))) ;[s2] = Automerge.applyChanges(Automerge.init(), changes) const heads1 = Automerge.Backend.getHeads(Automerge.Frontend.getBackendState(s1)) const heads2 = Automerge.Backend.getHeads(Automerge.Frontend.getBackendState(s2)) assert.deepStrictEqual(heads1, heads2) assert.deepStrictEqual(s1, s2) }) }) describe('root object', () => { it('should handle single-property assignment', () => { s1 = Automerge.change(s1, 'set bar', doc => doc.foo = 'bar') s1 = Automerge.change(s1, 'set zap', doc => doc.zip = 'zap') assert.strictEqual(s1.foo, 'bar') assert.strictEqual(s1.zip, 'zap') assert.deepStrictEqual(s1, {foo: 'bar', zip: 'zap'}) }) it('should allow floating-point values', () => { s1 = Automerge.change(s1, doc => doc.number = 1589032171.1) assert.strictEqual(s1.number, 1589032171.1) }) it('should handle multi-property assignment', () => { s1 = Automerge.change(s1, 'multi-assign', doc => { Object.assign(doc, {foo: 'bar', answer: 42}) }) assert.strictEqual(s1.foo, 'bar') assert.strictEqual(s1.answer, 42) assert.deepStrictEqual(s1, {foo: 'bar', answer: 42}) }) it('should handle root property deletion', () => { s1 = Automerge.change(s1, 'set foo', doc => { doc.foo = 'bar'; doc.something = null }) s1 = Automerge.change(s1, 'del foo', doc => { delete doc.foo }) assert.strictEqual(s1.foo, undefined) assert.strictEqual(s1.something, null) assert.deepStrictEqual(s1, {something: null}) }) it('should follow JS delete behavior', () => { s1 = Automerge.change(s1, 'set foo', doc => { doc.foo = 'bar' }) let deleted s1 = Automerge.change(s1, 'del foo', doc => { deleted = delete doc.foo }) assert.strictEqual(deleted, true) let deleted2 assert.doesNotThrow(() => { s1 = Automerge.change(s1, 'del baz', doc => { deleted2 = delete doc.baz }) }) assert.strictEqual(deleted2, true) }) it('should allow the type of a property to be changed', () => { s1 = Automerge.change(s1, 'set number', doc => doc.prop = 123) assert.strictEqual(s1.prop, 123) s1 = Automerge.change(s1, 'set string', doc => doc.prop = '123') assert.strictEqual(s1.prop, '123') s1 = Automerge.change(s1, 'set null', doc => doc.prop = null) assert.strictEqual(s1.prop, null) s1 = Automerge.change(s1, 'set bool', doc => doc.prop = true) assert.strictEqual(s1.prop, true) }) it('should require property names to be valid', () => { assert.throws(() => { Automerge.change(s1, 'foo', doc => doc[''] = 'x') }, /must not be an empty string/) }) it('should not allow assignment of unsupported datatypes', () => { Automerge.change(s1, doc => { assert.throws(() => { doc.foo = undefined }, /Unsupported type of value: undefined/) assert.throws(() => { doc.foo = {prop: undefined} }, /Unsupported type of value: undefined/) assert.throws(() => { doc.foo = () => {} }, /Unsupported type of value: function/) assert.throws(() => { doc.foo = Symbol('foo') }, /Unsupported type of value: symbol/) }) }) }) describe('nested maps', () => { it('should assign an objectId to nested maps', () => { s1 = Automerge.change(s1, doc => { doc.nested = {} }) assert.strictEqual(OPID_PATTERN.test(Automerge.getObjectId(s1.nested)), true) assert.notEqual(Automerge.getObjectId(s1.nested), '_root') }) it('should handle assignment of a nested property', () => { s1 = Automerge.change(s1, 'first change', doc => { doc.nested = {} doc.nested.foo = 'bar' }) s1 = Automerge.change(s1, 'second change', doc => { doc.nested.one = 1 }) assert.deepStrictEqual(s1, {nested: {foo: 'bar', one: 1}}) assert.deepStrictEqual(s1.nested, {foo: 'bar', one: 1}) assert.strictEqual(s1.nested.foo, 'bar') assert.strictEqual(s1.nested.one, 1) }) it('should handle assignment of an object literal', () => { s1 = Automerge.change(s1, doc => { doc.textStyle = {bold: false, fontSize: 12} }) assert.deepStrictEqual(s1, {textStyle: {bold: false, fontSize: 12}}) assert.deepStrictEqual(s1.textStyle, {bold: false, fontSize: 12}) assert.strictEqual(s1.textStyle.bold, false) assert.strictEqual(s1.textStyle.fontSize, 12) }) it('should handle assignment of multiple nested properties', () => { s1 = Automerge.change(s1, doc => { doc.textStyle = {bold: false, fontSize: 12} Object.assign(doc.textStyle, {typeface: 'Optima', fontSize: 14}) }) assert.strictEqual(s1.textStyle.typeface, 'Optima') assert.strictEqual(s1.textStyle.bold, false) assert.strictEqual(s1.textStyle.fontSize, 14) assert.deepStrictEqual(s1.textStyle, {typeface: 'Optima', bold: false, fontSize: 14}) }) it('should handle arbitrary-depth nesting', () => { s1 = Automerge.change(s1, doc => { doc.a = {b: {c: {d: {e: {f: {g: 'h'}}}}}} }) s1 = Automerge.change(s1, doc => { doc.a.b.c.d.e.f.i = 'j' }) assert.deepStrictEqual(s1, {a: { b: { c: { d: { e: { f: { g: 'h', i: 'j'}}}}}}}) assert.strictEqual(s1.a.b.c.d.e.f.g, 'h') assert.strictEqual(s1.a.b.c.d.e.f.i, 'j') }) it('should allow an old object to be replaced with a new one', () => { s1 = Automerge.change(s1, 'change 1', doc => { doc.myPet = {species: 'dog', legs: 4, breed: 'dachshund'} }) s2 = Automerge.change(s1, 'change 2', doc => { doc.myPet = {species: 'koi', variety: '紅白', colors: {red: true, white: true, black: false}} }) assert.deepStrictEqual(s1.myPet, { species: 'dog', legs: 4, breed: 'dachshund' }) assert.strictEqual(s1.myPet.breed, 'dachshund') assert.deepStrictEqual(s2.myPet, { species: 'koi', variety: '紅白', colors: {red: true, white: true, black: false} }) assert.strictEqual(s2.myPet.breed, undefined) assert.strictEqual(s2.myPet.variety, '紅白') }) it('should allow fields to be changed between primitive and nested map', () => { s1 = Automerge.change(s1, doc => doc.color = '#ff7f00') assert.strictEqual(s1.color, '#ff7f00') s1 = Automerge.change(s1, doc => doc.color = {red: 255, green: 127, blue: 0}) assert.deepStrictEqual(s1.color, {red: 255, green: 127, blue: 0}) s1 = Automerge.change(s1, doc => doc.color = '#ff7f00') assert.strictEqual(s1.color, '#ff7f00') }) it('should not allow several references to the same map object', () => { s1 = Automerge.change(s1, doc => doc.object = {}) assert.throws(() => { Automerge.change(s1, doc => { doc.x = doc.object }) }, /Cannot create a reference to an existing document object/) assert.throws(() => { Automerge.change(s1, doc => { doc.x = s1.object }) }, /Cannot create a reference to an existing document object/) assert.throws(() => { Automerge.change(s1, doc => { doc.x = {}; doc.y = doc.x }) }, /Cannot create a reference to an existing document object/) }) it('should not allow object-copying idioms', () => { s1 = Automerge.change(s1, doc => { doc.items = [{id: 'id1', name: 'one'}, {id: 'id2', name: 'two'}] }) // People who have previously worked with immutable state in JavaScript may be tempted // to use idioms like this, which don't work well with Automerge -- see e.g. // https://github.com/automerge/automerge/issues/260 assert.throws(() => { Automerge.change(s1, doc => { doc.items = [...doc.items, {id: 'id3', name: 'three'}] }) }, /Cannot create a reference to an existing document object/) }) it('should handle deletion of properties within a map', () => { s1 = Automerge.change(s1, 'set style', doc => { doc.textStyle = {typeface: 'Optima', bold: false, fontSize: 12} }) s1 = Automerge.change(s1, 'non-bold', doc => delete doc.textStyle.bold) assert.strictEqual(s1.textStyle.bold, undefined) assert.deepStrictEqual(s1.textStyle, {typeface: 'Optima', fontSize: 12}) }) it('should handle deletion of references to a map', () => { s1 = Automerge.change(s1, 'make rich text doc', doc => { Object.assign(doc, {title: 'Hello', textStyle: {typeface: 'Optima', fontSize: 12}}) }) s1 = Automerge.change(s1, doc => delete doc.textStyle) assert.strictEqual(s1.textStyle, undefined) assert.deepStrictEqual(s1, {title: 'Hello'}) }) it('should validate field names', () => { s1 = Automerge.change(s1, doc => doc.nested = {}) assert.throws(() => { Automerge.change(s1, doc => doc.nested[''] = 'x') }, /must not be an empty string/) assert.throws(() => { Automerge.change(s1, doc => doc.nested = {'': 'x'}) }, /must not be an empty string/) }) }) describe('lists', () => { it('should allow elements to be inserted', () => { s1 = Automerge.change(s1, doc => doc.noodles = []) s1 = Automerge.change(s1, doc => doc.noodles.insertAt(0, 'udon', 'soba')) s1 = Automerge.change(s1, doc => doc.noodles.insertAt(1, 'ramen')) assert.deepStrictEqual(s1, {noodles: ['udon', 'ramen', 'soba']}) assert.deepStrictEqual(s1.noodles, ['udon', 'ramen', 'soba']) assert.strictEqual(s1.noodles[0], 'udon') assert.strictEqual(s1.noodles[1], 'ramen') assert.strictEqual(s1.noodles[2], 'soba') assert.strictEqual(s1.noodles.length, 3) }) it('should handle assignment of a list literal', () => { s1 = Automerge.change(s1, doc => doc.noodles = ['udon', 'ramen', 'soba']) assert.deepStrictEqual(s1, {noodles: ['udon', 'ramen', 'soba']}) assert.deepStrictEqual(s1.noodles, ['udon', 'ramen', 'soba']) assert.strictEqual(s1.noodles[0], 'udon') assert.strictEqual(s1.noodles[1], 'ramen') assert.strictEqual(s1.noodles[2], 'soba') assert.strictEqual(s1.noodles[3], undefined) assert.strictEqual(s1.noodles.length, 3) }) it('should only allow numeric indexes', () => { s1 = Automerge.change(s1, doc => doc.noodles = ['udon', 'ramen', 'soba']) s1 = Automerge.change(s1, doc => doc.noodles[1] = 'Ramen!') assert.strictEqual(s1.noodles[1], 'Ramen!') s1 = Automerge.change(s1, doc => doc.noodles['1'] = 'RAMEN!!!') assert.strictEqual(s1.noodles[1], 'RAMEN!!!') assert.throws(() => { Automerge.change(s1, doc => doc.noodles.favourite = 'udon') }, /list index must be a number/) assert.throws(() => { Automerge.change(s1, doc => doc.noodles[''] = 'udon') }, /list index must be a number/) assert.throws(() => { Automerge.change(s1, doc => doc.noodles['1e6'] = 'udon') }, /list index must be a number/) }) it('should handle deletion of list elements', () => { s1 = Automerge.change(s1, doc => doc.noodles = ['udon', 'ramen', 'soba']) s1 = Automerge.change(s1, doc => delete doc.noodles[1]) assert.deepStrictEqual(s1.noodles, ['udon', 'soba']) s1 = Automerge.change(s1, doc => doc.noodles.deleteAt(1)) assert.deepStrictEqual(s1.noodles, ['udon']) assert.strictEqual(s1.noodles[0], 'udon') assert.strictEqual(s1.noodles[1], undefined) assert.strictEqual(s1.noodles[2], undefined) assert.strictEqual(s1.noodles.length, 1) }) it('should handle assignment of individual list indexes', () => { s1 = Automerge.change(s1, doc => doc.japaneseFood = ['udon', 'ramen', 'soba']) s1 = Automerge.change(s1, doc => doc.japaneseFood[1] = 'sushi') assert.deepStrictEqual(s1.japaneseFood, ['udon', 'sushi', 'soba']) assert.strictEqual(s1.japaneseFood[0], 'udon') assert.strictEqual(s1.japaneseFood[1], 'sushi') assert.strictEqual(s1.japaneseFood[2], 'soba') assert.strictEqual(s1.japaneseFood[3], undefined) assert.strictEqual(s1.japaneseFood.length, 3) }) it('should treat out-by-one assignment as insertion', () => { s1 = Automerge.change(s1, doc => doc.japaneseFood = ['udon']) s1 = Automerge.change(s1, doc => doc.japaneseFood[1] = 'sushi') assert.deepStrictEqual(s1.japaneseFood, ['udon', 'sushi']) assert.strictEqual(s1.japaneseFood[0], 'udon') assert.strictEqual(s1.japaneseFood[1], 'sushi') assert.strictEqual(s1.japaneseFood[2], undefined) assert.strictEqual(s1.japaneseFood.length, 2) }) it('should allow bulk assignment of multiple list indexes', () => { s1 = Automerge.change(s1, doc => doc.noodles = ['udon', 'ramen', 'soba']) s1 = Automerge.change(s1, doc => Object.assign(doc.noodles, {0: 'うどん', 2: 'そば'})) assert.deepStrictEqual(s1.noodles, ['うどん', 'ramen', 'そば']) assert.strictEqual(s1.noodles[0], 'うどん') assert.strictEqual(s1.noodles[1], 'ramen') assert.strictEqual(s1.noodles[2], 'そば') assert.strictEqual(s1.noodles.length, 3) }) it('should handle nested objects', () => { s1 = Automerge.change(s1, doc => doc.noodles = [{type: 'ramen', dishes: ['tonkotsu', 'shoyu']}]) s1 = Automerge.change(s1, doc => doc.noodles.push({type: 'udon', dishes: ['tempura udon']})) s1 = Automerge.change(s1, doc => doc.noodles[0].dishes.push('miso')) assert.deepStrictEqual(s1, {noodles: [ {type: 'ramen', dishes: ['tonkotsu', 'shoyu', 'miso']}, {type: 'udon', dishes: ['tempura udon']} ]}) assert.deepStrictEqual(s1.noodles[0], { type: 'ramen', dishes: ['tonkotsu', 'shoyu', 'miso'] }) assert.deepStrictEqual(s1.noodles[1], { type: 'udon', dishes: ['tempura udon'] }) }) it('should handle nested lists', () => { s1 = Automerge.change(s1, doc => doc.noodleMatrix = [['ramen', 'tonkotsu', 'shoyu']]) s1 = Automerge.change(s1, doc => doc.noodleMatrix.push(['udon', 'tempura udon'])) s1 = Automerge.change(s1, doc => doc.noodleMatrix[0].push('miso')) assert.deepStrictEqual(s1.noodleMatrix, [['ramen', 'tonkotsu', 'shoyu', 'miso'], ['udon', 'tempura udon']]) assert.deepStrictEqual(s1.noodleMatrix[0], ['ramen', 'tonkotsu', 'shoyu', 'miso']) assert.deepStrictEqual(s1.noodleMatrix[1], ['udon', 'tempura udon']) }) it('should handle deep nesting', () => { s1 = Automerge.change(s1, doc => doc.nesting = { maps: { m1: { m2: { foo: "bar", baz: {} }, m2a: { } } }, lists: [ [ 1, 2, 3 ], [ [ 3, 4, 5, [6]], 7 ] ], mapsinlists: [ { foo: "bar" }, [ { bar: "baz" } ] ], listsinmaps: { foo: [1, 2, 3], bar: [ [ { baz: "123" } ] ] } }) s1 = Automerge.change(s1, doc => { doc.nesting.maps.m1a = "123" doc.nesting.maps.m1.m2.baz.xxx = "123" delete doc.nesting.maps.m1.m2a doc.nesting.lists.shift() doc.nesting.lists[0][0].pop() doc.nesting.lists[0][0].push(100) doc.nesting.mapsinlists[0].foo = "baz" doc.nesting.mapsinlists[1][0].foo = "bar" delete doc.nesting.mapsinlists[1] doc.nesting.listsinmaps.foo.push(4) doc.nesting.listsinmaps.bar[0][0].baz = "456" delete doc.nesting.listsinmaps.bar }) assert.deepStrictEqual(s1, { nesting: { maps: { m1: { m2: { foo: "bar", baz: { xxx: "123" } } }, m1a: "123" }, lists: [ [ [ 3, 4, 5, 100 ], 7 ] ], mapsinlists: [ { foo: "baz" } ], listsinmaps: { foo: [1, 2, 3, 4] } }}) }) it('should handle replacement of the entire list', () => { s1 = Automerge.change(s1, doc => doc.noodles = ['udon', 'soba', 'ramen']) s1 = Automerge.change(s1, doc => doc.japaneseNoodles = doc.noodles.slice()) s1 = Automerge.change(s1, doc => doc.noodles = ['wonton', 'pho']) assert.deepStrictEqual(s1, { noodles: ['wonton', 'pho'], japaneseNoodles: ['udon', 'soba', 'ramen'] }) assert.deepStrictEqual(s1.noodles, ['wonton', 'pho']) assert.strictEqual(s1.noodles[0], 'wonton') assert.strictEqual(s1.noodles[1], 'pho') assert.strictEqual(s1.noodles[2], undefined) assert.strictEqual(s1.noodles.length, 2) }) it('should allow assignment to change the type of a list element', () => { s1 = Automerge.change(s1, doc => doc.noodles = ['udon', 'soba', 'ramen']) assert.deepStrictEqual(s1.noodles, ['udon', 'soba', 'ramen']) s1 = Automerge.change(s1, doc => doc.noodles[1] = {type: 'soba', options: ['hot', 'cold']}) assert.deepStrictEqual(s1.noodles, ['udon', {type: 'soba', options: ['hot', 'cold']}, 'ramen']) s1 = Automerge.change(s1, doc => doc.noodles[1] = ['hot soba', 'cold soba']) assert.deepStrictEqual(s1.noodles, ['udon', ['hot soba', 'cold soba'], 'ramen']) s1 = Automerge.change(s1, doc => doc.noodles[1] = 'soba is the best') assert.deepStrictEqual(s1.noodles, ['udon', 'soba is the best', 'ramen']) }) it('should allow list creation and assignment in the same change callback', () => { s1 = Automerge.change(Automerge.init(), doc => { doc.letters = ['a', 'b', 'c'] doc.letters[1] = 'd' }) assert.strictEqual(s1.letters[1], 'd') }) it('should allow adding and removing list elements in the same change callback', () => { s1 = Automerge.change(Automerge.init(), doc => doc.noodles = []) s1 = Automerge.change(s1, doc => { doc.noodles.push('udon') doc.noodles.deleteAt(0) }) assert.deepStrictEqual(s1, {noodles: []}) // do the add-remove cycle twice, test for #151 (https://github.com/automerge/automerge/issues/151) s1 = Automerge.change(s1, doc => { doc.noodles.push('soba') doc.noodles.deleteAt(0) }) assert.deepStrictEqual(s1, {noodles: []}) }) it('should handle arbitrary-depth nesting', () => { s1 = Automerge.change(s1, doc => doc.maze = [[[[[[[['noodles', ['here']]]]]]]]]) s1 = Automerge.change(s1, doc => doc.maze[0][0][0][0][0][0][0][1].unshift('found')) assert.deepStrictEqual(s1.maze, [[[[[[[['noodles', ['found', 'here']]]]]]]]]) assert.deepStrictEqual(s1.maze[0][0][0][0][0][0][0][1][1], 'here') }) it('should not allow several references to the same list object', () => { s1 = Automerge.change(s1, doc => doc.list = []) assert.throws(() => { Automerge.change(s1, doc => { doc.x = doc.list }) }, /Cannot create a reference to an existing document object/) assert.throws(() => { Automerge.change(s1, doc => { doc.x = s1.list }) }, /Cannot create a reference to an existing document object/) assert.throws(() => { Automerge.change(s1, doc => { doc.x = []; doc.y = doc.x }) }, /Cannot create a reference to an existing document object/) }) it('concurrent edits insert in reverse actorid order if counters equal', () => { s1 = Automerge.init('aaaa') s2 = Automerge.init('bbbb') s1 = Automerge.change(s1, doc => doc.list = []) s2 = Automerge.merge(s2, s1) s1 = Automerge.change(s1, doc => doc.list.splice(0, 0, "2@aaaa")) s2 = Automerge.change(s2, doc => doc.list.splice(0, 0, "2@bbbb")) s2 = Automerge.merge(s2, s1) assert.deepStrictEqual(s2.list, ["2@bbbb", "2@aaaa"]) }) it('concurrent edits insert in reverse counter order if different', () => { s1 = Automerge.init('aaaa') s2 = Automerge.init('bbbb') s1 = Automerge.change(s1, doc => doc.list = []) s2 = Automerge.merge(s2, s1) s1 = Automerge.change(s1, doc => doc.list.splice(0, 0, "2@aaaa")) s2 = Automerge.change(s2, doc => doc.foo = "2@bbbb") s2 = Automerge.change(s2, doc => doc.list.splice(0, 0, "3@bbbb")) s2 = Automerge.merge(s2, s1) assert.deepStrictEqual(s2.list, ["3@bbbb", "2@aaaa"]) }) }) describe('numbers', () => { it('should default to int for positive numbers', () => { const s1 = Automerge.change(Automerge.init(), doc => doc.number = 1) const binChange = Automerge.getLastLocalChange(s1) const change = decodeChange(binChange) assert.deepStrictEqual(change.ops[0], { action: 'set', datatype: 'int', insert: false, key: 'number', obj: '_root', pred: [], value: 1 }) }) it('should default to int for negative numbers', () => { const s1 = Automerge.change(Automerge.init(), doc => doc.number = -1) const binChange = Automerge.getLastLocalChange(s1) const change = decodeChange(binChange) assert.deepStrictEqual(change.ops[0], { action: 'set', datatype: 'int', insert: false, key: 'number', obj: '_root', pred: [], value: -1 }) }) it('should default to float64 for floats', () => { const s1 = Automerge.change(Automerge.init(), doc => doc.number = 1.1) const binChange = Automerge.getLastLocalChange(s1) const change = decodeChange(binChange) assert.deepStrictEqual(change.ops[0], { action: 'set', datatype: 'float64', insert: false, key: 'number', obj: '_root', pred: [], value: 1.1 }) }) it('float64 can be specificed manually', () => { const s1 = Automerge.change(Automerge.init(), doc => doc.number = new Automerge.Float64(3)) const binChange = Automerge.getLastLocalChange(s1) const change = decodeChange(binChange) assert.deepStrictEqual(change.ops[0], { action: 'set', datatype: 'float64', insert: false, key: 'number', obj: '_root', pred: [], value: 3 }) }) it('int can be specificed manually', () => { const s1 = Automerge.change(Automerge.init(), doc => doc.number = new Automerge.Int(3)) const binChange = Automerge.getLastLocalChange(s1) const change = decodeChange(binChange) assert.deepStrictEqual(change.ops[0], { action: 'set', datatype: 'int', insert: false, key: 'number', obj: '_root', pred: [], value: 3 }) }) it('uint can be specificed manually', () => { const s1 = Automerge.change(Automerge.init(), doc => doc.number = new Automerge.Uint(3)) const binChange = Automerge.getLastLocalChange(s1) const change = decodeChange(binChange) assert.deepStrictEqual(change.ops[0], { action: 'set', datatype: 'uint', insert: false, key: 'number', obj: '_root', pred: [], value: 3 }) }) }) describe('counters', () => { it('should allow deleting counters from maps', () => { const s1 = Automerge.change(Automerge.init(), doc => doc.birds = {wrens: new Automerge.Counter(1)}) const s2 = Automerge.change(s1, doc => doc.birds.wrens.increment(2)) const s3 = Automerge.change(s2, doc => delete doc.birds.wrens) assert.deepStrictEqual(s2, {birds: {wrens: new Automerge.Counter(3)}}) assert.deepStrictEqual(s3, {birds: {}}) }) it('should not allow deleting counters from lists', () => { const s1 = Automerge.change(Automerge.init(), doc => doc.recordings = [new Automerge.Counter(1)]) const s2 = Automerge.change(s1, doc => doc.recordings[0].increment(2)) assert.deepStrictEqual(s2, {recordings: [new Automerge.Counter(3)]}) assert.throws(() => { Automerge.change(s2, doc => doc.recordings.deleteAt(0)) }, /Unsupported operation/) }) it('should allow putting multiple counters in a list', () => { const s1 = Automerge.from({ counters: [ new Automerge.Counter(1), new Automerge.Counter(2) ] }) assert.deepStrictEqual(s1, {counters: [ new Automerge.Counter(1), new Automerge.Counter(2) ] }) }) it('should allow putting counters in a list with non counters', () => { let date = new Date() const s1 = Automerge.from({ counters: [ new Automerge.Counter(1), -1, new Automerge.Counter(2), 2.2, true, date ] }) assert.deepStrictEqual(s1, {counters: [ new Automerge.Counter(1), -1, new Automerge.Counter(2), 2.2, true, date ] }) }) }) }) describe('concurrent use', () => { let s1, s2, s3 beforeEach(() => { s1 = Automerge.init() s2 = Automerge.init() s3 = Automerge.init() }) it('should merge concurrent updates of different properties', () => { s1 = Automerge.change(s1, doc => doc.foo = 'bar') s2 = Automerge.change(s2, doc => doc.hello = 'world') s3 = Automerge.merge(s1, s2) assert.strictEqual(s3.foo, 'bar') assert.strictEqual(s3.hello, 'world') assert.deepStrictEqual(s3, {foo: 'bar', hello: 'world'}) assert.strictEqual(Automerge.getConflicts(s3, 'foo'), undefined) assert.strictEqual(Automerge.getConflicts(s3, 'hello'), undefined) }) it('should add concurrent increments of the same property', () => { s1 = Automerge.change(s1, doc => doc.counter = new Automerge.Counter()) s2 = Automerge.merge(s2, s1) s1 = Automerge.change(s1, doc => doc.counter.increment()) s2 = Automerge.change(s2, doc => doc.counter.increment(2)) s3 = Automerge.merge(s1, s2) assert.strictEqual(s1.counter.value, 1) assert.strictEqual(s2.counter.value, 2) assert.strictEqual(s3.counter.value, 3) assert.strictEqual(Automerge.getConflicts(s3, 'counter'), undefined) }) it('should add increments only to the values they precede', () => { s1 = Automerge.change(s1, doc => doc.counter = new Automerge.Counter(0)) s1 = Automerge.change(s1, doc => doc.counter.increment()) s2 = Automerge.change(s2, doc => doc.counter = new Automerge.Counter(100)) s2 = Automerge.change(s2, doc => doc.counter.increment(3)) s3 = Automerge.merge(s1, s2) if (Automerge.getActorId(s1) > Automerge.getActorId(s2)) { assert.deepStrictEqual(s3, {counter: new Automerge.Counter(1)}) } else { assert.deepStrictEqual(s3, {counter: new Automerge.Counter(103)}) } assert.deepStrictEqual(Automerge.getConflicts(s3, 'counter'), { [`1@${Automerge.getActorId(s1)}`]: new Automerge.Counter(1), [`1@${Automerge.getActorId(s2)}`]: new Automerge.Counter(103) }) }) it('should detect concurrent updates of the same field', () => { s1 = Automerge.change(s1, doc => doc.field = 'one') s2 = Automerge.change(s2, doc => doc.field = 'two') s3 = Automerge.merge(s1, s2) if (Automerge.getActorId(s1) > Automerge.getActorId(s2)) { assert.deepStrictEqual(s3, {field: 'one'}) } else { assert.deepStrictEqual(s3, {field: 'two'}) } assert.deepStrictEqual(Automerge.getConflicts(s3, 'field'), { [`1@${Automerge.getActorId(s1)}`]: 'one', [`1@${Automerge.getActorId(s2)}`]: 'two' }) }) it('should detect concurrent updates of the same list element', () => { s1 = Automerge.change(s1, doc => doc.birds = ['finch']) s2 = Automerge.merge(s2, s1) s1 = Automerge.change(s1, doc => doc.birds[0] = 'greenfinch') s2 = Automerge.change(s2, doc => doc.birds[0] = 'goldfinch') s3 = Automerge.merge(s1, s2) if (Automerge.getActorId(s1) > Automerge.getActorId(s2)) { assert.deepStrictEqual(s3.birds, ['greenfinch']) } else { assert.deepStrictEqual(s3.birds, ['goldfinch']) } assert.deepStrictEqual(Automerge.getConflicts(s3.birds, 0), { [`3@${Automerge.getActorId(s1)}`]: 'greenfinch', [`3@${Automerge.getActorId(s2)}`]: 'goldfinch' }) }) it('should handle assignment conflicts of different types', () => { s1 = Automerge.change(s1, doc => doc.field = 'string') s2 = Automerge.change(s2, doc => doc.field = ['list']) s3 = Automerge.change(s3, doc => doc.field = {thing: 'map'}) s1 = Automerge.merge(Automerge.merge(s1, s2), s3) assertEqualsOneOf(s1.field, 'string', ['list'], {thing: 'map'}) assert.deepStrictEqual(Automerge.getConflicts(s1, 'field'), { [`1@${Automerge.getActorId(s1)}`]: 'string', [`1@${Automerge.getActorId(s2)}`]: ['list'], [`1@${Automerge.getActorId(s3)}`]: {thing: 'map'} }) }) it('should handle changes within a conflicting map field', () => { s1 = Automerge.change(s1, doc => doc.field = 'string') s2 = Automerge.change(s2, doc => doc.field = {}) s2 = Automerge.change(s2, doc => doc.field.innerKey = 42) s3 = Automerge.merge(s1, s2) assertEqualsOneOf(s3.field, 'string', {innerKey: 42}) assert.deepStrictEqual(Automerge.getConflicts(s3, 'field'), { [`1@${Automerge.getActorId(s1)}`]: 'string', [`1@${Automerge.getActorId(s2)}`]: {innerKey: 42} }) }) it('should handle changes within a conflicting list element', () => { s1 = Automerge.change(s1, doc => doc.list = ['hello']) s2 = Automerge.merge(s2, s1) s1 = Automerge.change(s1, doc => doc.list[0] = {map1: true}) s1 = Automerge.change(s1, doc => doc.list[0].key = 1) s2 = Automerge.change(s2, doc => doc.list[0] = {map2: true}) s2 = Automerge.change(s2, doc => doc.list[0].key = 2) s3 = Automerge.merge(s1, s2) if (Automerge.getActorId(s1) > Automerge.getActorId(s2)) { assert.deepStrictEqual(s3.list, [{map1: true, key: 1}]) } else { assert.deepStrictEqual(s3.list, [{map2: true, key: 2}]) } assert.deepStrictEqual(Automerge.getConflicts(s3.list, 0), { [`3@${Automerge.getActorId(s1)}`]: {map1: true, key: 1}, [`3@${Automerge.getActorId(s2)}`]: {map2: true, key: 2} }) }) it('should not merge concurrently assigned nested maps', () => { s1 = Automerge.change(s1, doc => doc.config = {background: 'blue'}) s2 = Automerge.change(s2, doc => doc.config = {logo_url: 'logo.png'}) s3 = Automerge.merge(s1, s2) assertEqualsOneOf(s3.config, {background: 'blue'}, {logo_url: 'logo.png'}) assert.deepStrictEqual(Automerge.getConflicts(s3, 'config'), { [`1@${Automerge.getActorId(s1)}`]: {background: 'blue'}, [`1@${Automerge.getActorId(s2)}`]: {logo_url: 'logo.png'} }) }) it('should clear conflicts after assigning a new value', () => { s1 = Automerge.change(s1, doc => doc.field = 'one') s2 = Automerge.change(s2, doc => doc.field = 'two') s3 = Automerge.merge(s1, s2) s3 = Automerge.change(s3, doc => doc.field = 'three') assert.deepStrictEqual(s3, {field: 'three'}) assert.strictEqual(Automerge.getConflicts(s3, 'field'), undefined) s2 = Automerge.merge(s2, s3) assert.deepStrictEqual(s2, {field: 'three'}) assert.strictEqual(Automerge.getConflicts(s2, 'field'), undefined) }) it('should handle concurrent insertions at different list positions', () => { s1 = Automerge.change(s1, doc => doc.list = ['one', 'three']) s2 = Automerge.merge(s2, s1) s1 = Automerge.change(s1, doc => doc.list.splice(1, 0, 'two')) s2 = Automerge.change(s2, doc => doc.list.push('four')) s3 = Automerge.merge(s1, s2) assert.deepStrictEqual(s3, {list: ['one', 'two', 'three', 'four']}) assert.strictEqual(Automerge.getConflicts(s3, 'list'), undefined) }) it('should handle concurrent insertions at the same list position', () => { s1 = Automerge.change(s1, doc => doc.birds = ['parakeet']) s2 = Automerge.merge(s2, s1) s1 = Automerge.change(s1, doc => doc.birds.push('starling')) s2 = Automerge.change(s2, doc => doc.birds.push('chaffinch')) s3 = Automerge.merge(s1, s2) assertEqualsOneOf(s3.birds, ['parakeet', 'starling', 'chaffinch'], ['parakeet', 'chaffinch', 'starling']) s2 = Automerge.merge(s2, s3) assert.deepStrictEqual(s2, s3) }) it('should handle concurrent assignment and deletion of a map entry', () => { // Add-wins semantics s1 = Automerge.change(s1, doc => doc.bestBird = 'robin') s2 = Automerge.merge(s2, s1) s1 = Automerge.change(s1, doc => delete doc.bestBird) s2 = Automerge.change(s2, doc => doc.bestBird = 'magpie') s3 = Automerge.merge(s1, s2) assert.deepStrictEqual(s1, {}) assert.deepStrictEqual(s2, {bestBird: 'magpie'}) assert.deepStrictEqual(s3, {bestBird: 'magpie'}) assert.strictEqual(Automerge.getConflicts(s3, 'bestBird'), undefined) }) it('should handle concurrent assignment and deletion of a list element', () => { // Concurrent assignment ressurects a deleted list element. Perhaps a little // surprising, but consistent with add-wins semantics of maps (see test above) s1 = Automerge.change(s1, doc => doc.birds = ['blackbird', 'thrush', 'goldfinch']) s2 = Automerge.merge(s2, s1) s1 = Automerge.change(s1, doc => doc.birds[1] = 'starling') s2 = Automerge.change(s2, doc => doc.birds.splice(1, 1)) s3 = Automerge.merge(s1, s2) assert.deepStrictEqual(s1.birds, ['blackbird', 'starling', 'goldfinch']) assert.deepStrictEqual(s2.birds, ['blackbird', 'goldfinch']) assert.deepStrictEqual(s3.birds, ['blackbird', 'starling', 'goldfinch']) }) it('should handle insertion after a deleted list element', () => { s1 = Automerge.change(s1, doc => doc.birds = ['blackbird', 'thrush', 'goldfinch']) s2 = Automerge.merge(s2, s1) s1 = Automerge.change(s1, doc => doc.birds.splice(1, 2)) s2 = Automerge.change(s2, doc => doc.birds.splice(2, 0, 'starling')) s3 = Automerge.merge(s1, s2) assert.deepStrictEqual(s3, {birds: ['blackbird', 'starling']}) assert.deepStrictEqual(Automerge.merge(s2, s3), {birds: ['blackbird', 'starling']}) }) it('should handle concurrent deletion of the same element', () => { s1 = Automerge.change(s1, doc => doc.birds = ['albatross', 'buzzard', 'cormorant']) s2 = Automerge.merge(s2, s1) s1 = Automerge.change(s1, doc => doc.birds.deleteAt(1)) // buzzard s2 = Automerge.change(s2, doc => doc.birds.deleteAt(1)) // buzzard s3 = Automerge.merge(s1, s2) assert.deepStrictEqual(s3.birds, ['albatross', 'cormorant']) }) it('should handle concurrent deletion of different elements', () => { s1 = Automerge.change(s1, doc => doc.birds = ['albatross', 'buzzard', 'cormorant']) s2 = Automerge.merge(s2, s1) s1 = Automerge.change(s1, doc => doc.birds.deleteAt(0)) // albatross s2 = Automerge.change(s2, doc => doc.birds.deleteAt(1)) // buzzard s3 = Automerge.merge(s1, s2) assert.deepStrictEqual(s3.birds, ['cormorant']) }) it('should handle concurrent updates at different levels of the tree', () => { // A delete higher up in the tree overrides an update in a subtree s1 = Automerge.change(s1, doc => doc.animals = {birds: {pink: 'flamingo', black: 'starling'}, mammals: ['badger']}) s2 = Automerge.merge(s2, s1) s1 = Automerge.change(s1, doc => doc.animals.birds.brown = 'sparrow') s2 = Automerge.change(s2, doc => delete doc.animals.birds) s3 = Automerge.merge(s1, s2) assert.deepStrictEqual(s1.animals, { birds: { pink: 'flamingo', brown: 'sparrow', black: 'starling' }, mammals: ['badger'] }) assert.deepStrictEqual(s2.animals, {mammals: ['badger']}) assert.deepStrictEqual(s3.animals, {mammals: ['badger']}) }) it('should handle updates of concurrently deleted objects', () => { s1 = Automerge.change(s1, doc => doc.birds = {blackbird: {feathers: 'black'}}) s2 = Automerge.merge(s2, s1) s1 = Automerge.change(s1, doc => delete doc.birds.blackbird) s2 = Automerge.change(s2, doc => doc.birds.blackbird.beak = 'orange') s3 = Automerge.merge(s1, s2) assert.deepStrictEqual(s1, {birds: {}}) }) it('should not interleave sequence insertions at the same position', () => { s1 = Automerge.change(s1, doc => doc.wisdom = []) s2 = Automerge.merge(s2, s1) s1 = Automerge.change(s1, doc => doc.wisdom.push('to', 'be', 'is', 'to', 'do')) s2 = Automerge.change(s2, doc => doc.wisdom.push('to', 'do', 'is', 'to', 'be')) s3 = Automerge.merge(s1, s2) assertEqualsOneOf(s3.wisdom, ['to', 'be', 'is', 'to', 'do', 'to', 'do', 'is', 'to', 'be'], ['to', 'do', 'is', 'to', 'be', 'to', 'be', 'is', 'to', 'do']) // In case you're wondering: http://quoteinvestigator.com/2013/09/16/do-be-do/ }) describe('multiple insertions at the same list position', () => { it('should handle insertion by greater actor ID', () => { s1 = Automerge.init('aaaa') s2 = Automerge.init('bbbb') s1 = Automerge.change(s1, doc => doc.list = ['two']) s2 = Automerge.merge(s2, s1) s2 = Automerge.change(s2, doc => doc.list.splice(0, 0, 'one')) assert.deepStrictEqual(s2.list, ['one', 'two']) }) it('should handle insertion by lesser actor ID', () => { s1 = Automerge.init('bbbb') s2 = Automerge.init('aaaa') s1 = Automerge.change(s1, doc => doc.list = ['two']) s2 = Automerge.merge(s2, s1) s2 = Automerge.change(s2, doc => doc.list.splice(0, 0, 'one')) assert.deepStrictEqual(s2.list, ['one', 'two']) }) it('should handle insertion regardless of actor ID', () => { s1 = Automerge.change(s1, doc => doc.list = ['two']) s2 = Automerge.merge(s2, s1) s2 = Automerge.change(s2, doc => doc.list.splice(0, 0, 'one')) assert.deepStrictEqual(s2.list, ['one', 'two']) }) it('should make insertion order consistent with causality', () => { s1 = Automerge.change(s1, doc => doc.list = ['four']) s2 = Automerge.merge(s2, s1) s2 = Automerge.change(s2, doc => doc.list.unshift('three')) s1 = Automerge.merge(s1, s2) s1 = Automerge.change(s1, doc => doc.list.unshift('two')) s2 = Automerge.merge(s2, s1) s2 = Automerge.change(s2, doc => doc.list.unshift('one')) assert.deepStrictEqual(s2.list, ['one', 'two', 'three', 'four']) }) }) }) describe('saving and loading', () => { it('should save and restore an empty document', () => { let s = Automerge.load(Automerge.save(Automerge.init())) assert.deepStrictEqual(s, {}) }) it('should generate a new random actor ID', () => { let s1 = Automerge.init() let s2 = Automerge.load(Automerge.save(s1)) assert.strictEqual(UUID_PATTERN.test(Automerge.getActorId(s1).toString()), true) assert.strictEqual(UUID_PATTERN.test(Automerge.getActorId(s2).toString()), true) assert.notEqual(Automerge.getActorId(s1), Automerge.getActorId(s2)) }) it('should allow a custom actor ID to be set', () => { let s = Automerge.load(Automerge.save(Automerge.init()), '333333') assert.strictEqual(Automerge.getActorId(s), '333333') }) it('should reconstitute complex datatypes', () => { let s1 = Automerge.change(Automerge.init(), doc => doc.todos = [{title: 'water plants', done: false}]) let s2 = Automerge.load(Automerge.save(s1)) assert.deepStrictEqual(s2, {todos: [{title: 'water plants', done: false}]}) }) it('should save and load maps with @ symbols in the keys', () => { let s1 = Automerge.change(Automerge.init(), doc => doc["123@4567"] = "hello") let s2 = Automerge.load(Automerge.save(s1)) assert.deepStrictEqual(s2, { "123@4567": "hello" }) }) it('should reconstitute conflicts', () => { let s1 = Automerge.change(Automerge.init('111111'), doc => doc.x = 3) let s2 = Automerge.change(Automerge.init('222222'), doc => doc.x = 5) s1 = Automerge.merge(s1, s2) let s3 = Automerge.load(Automerge.save(s1)) assert.strictEqual(s1.x, 5) assert.strictEqual(s3.x, 5) assert.deepStrictEqual(Automerge.getConflicts(s1, 'x'), {'1@111111': 3, '1@222222': 5}) assert.deepStrictEqual(Automerge.getConflicts(s3, 'x'), {'1@111111': 3, '1@222222': 5}) }) it('should reconstitute element ID counters', () => { const s1 = Automerge.init('01234567') const s2 = Automerge.change(s1, doc => doc.list = ['a']) const listId = Automerge.getObjectId(s2.list) const changes12 = Automerge.getAllChanges(s2).map(decodeChange) assert.deepStrictEqual(changes12, [{ hash: changes12[0].hash, actor: '01234567', seq: 1, startOp: 1, time: changes12[0].time, message: '', deps: [], ops: [ {obj: '_root', action: 'makeList', key: 'list', insert: false, pred: []}, {obj: listId, action: 'set', elemId: '_head', insert: true, value: 'a', pred: []} ] }]) const s3 = Automerge.change(s2, doc => doc.list.deleteAt(0)) const s4 = Automerge.load(Automerge.save(s3), '01234567') const s5 = Automerge.change(s4, doc => doc.list.push('b')) const changes45 = Automerge.getAllChanges(s5).map(decodeChange) assert.deepStrictEqual(s5, {list: ['b']}) assert.deepStrictEqual(changes45[2], { hash: changes45[2].hash, actor: '01234567', seq: 3, startOp: 4, time: changes45[2].time, message: '', deps: [changes45[1].hash], ops: [ {obj: listId, action: 'set', elemId: '_head', insert: true, value: 'b', pred: []} ] }) }) it('should allow a reloaded list to be mutated', () => { let doc = Automerge.change(Automerge.init(), doc => doc.foo = []) doc = Automerge.load(Automerge.save(doc)) doc = Automerge.change(doc, 'add', doc => doc.foo.push(1)) doc = Automerge.load(Automerge.save(doc)) assert.deepStrictEqual(doc.foo, [1]) }) it('should reload a document containing deflated columns', () => { // In this test, the keyCtr column is long enough for deflate compression to kick in, but the // keyStr column is short. Thus, the deflate bit gets set for keyCtr but not for keyStr. // When checking whether the columns appear in ascending order, we must ignore the deflate bit. let doc = Automerge.change(Automerge.init(), doc => { doc.list = [] for (let i = 0; i < 200; i++) doc.list.insertAt(Math.floor(Math.random() * i), 'a') }) Automerge.load(Automerge.save(doc)) let expected = [] for (let i = 0; i < 200; i++) expected.push('a') assert.deepStrictEqual(doc, {list: expected}) }) it('should call patchCallback if supplied', () => { const s1 = Automerge.change(Automerge.init(), doc => doc.birds = ['Goldfinch']) const s2 = Automerge.change(s1, doc => doc.birds.push('Chaffinch')) const callbacks = [], actor = Automerge.getActorId(s1) const reloaded = Automerge.load(Automerge.save(s2), { patchCallback(patch, before, after, local) { callbacks.push({patch, before, after, local}) } }) assert.strictEqual(callbacks.length, 1) assert.deepStrictEqual(callbacks[0].patch, { maxOp: 3, deps: [decodeChange(Automerge.getAllChanges(s2)[1]).hash], clock: {[actor]: 2}, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {birds: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'list', edits: [ {action: 'multi-insert', index: 0, elemId: `2@${actor}`, values: ['Goldfinch', 'Chaffinch']} ] }}}} }) assert.deepStrictEqual(callbacks[0].before, {}) assert.strictEqual(callbacks[0].after, reloaded) assert.strictEqual(callbacks[0].local, false) }) it('should reconstruct the original changes if needed', () => { let doc = Automerge.init() for (let i = 0; i < 10; i++) doc = Automerge.change(doc, doc => doc.x = i) doc = Automerge.load(Automerge.save(doc)) assert.strictEqual(Automerge.getAllChanges(doc).length, 10) }) it('should deduplicate changes after saving and reloading', () => { let initChange = Automerge.getLastLocalChange(Automerge.change(Automerge.init('0000'), { time: 0 }, (doc) => { doc.panels = [] })) let [s1] = Automerge.applyChanges(Automerge.init(), [initChange]) let [s2] = Automerge.applyChanges(Automerge.init(), [initChange]) s1 = Automerge.change(s1, doc => doc.panels.push({ id: 'panel1' })) s2 = Automerge.change(s2, doc => doc.panels.push({ id: 'panel2' })) s1 = Automerge.load(Automerge.save(s1)) let [s3] = Automerge.applyChanges(s1, Automerge.getAllChanges(s2)) assert.strictEqual(s3.panels.length, 2) }) }) describe('history API', () => { it('should return an empty history for an empty document', () => { assert.deepStrictEqual(Automerge.getHistory(Automerge.init()), []) }) it('should make past document states accessible', () => { let s = Automerge.init() s = Automerge.change(s, doc => doc.config = {background: 'blue'}) s = Automerge.change(s, doc => doc.birds = ['mallard']) s = Automerge.change(s, doc => doc.birds.unshift('oystercatcher')) assert.deepStrictEqual(Automerge.getHistory(s).map(state => state.snapshot), [ {config: {background: 'blue'}}, {config: {background: 'blue'}, birds: ['mallard']}, {config: {background: 'blue'}, birds: ['oystercatcher', 'mallard']} ]) }) it('should make change messages accessible', () => { let s = Automerge.init() s = Automerge.change(s, 'Empty Bookshelf', doc => doc.books = []) s = Automerge.change(s, 'Add Orwell', doc => doc.books.push('Nineteen Eighty-Four')) s = Automerge.change(s, 'Add Huxley', doc => doc.books.push('Brave New World')) assert.deepStrictEqual(s.books, ['Nineteen Eighty-Four', 'Brave New World']) assert.deepStrictEqual(Automerge.getHistory(s).map(state => state.change.message), ['Empty Bookshelf', 'Add Orwell', 'Add Huxley']) }) }) describe('changes API', () => { it('should return an empty list on an empty document', () => { let changes = Automerge.getAllChanges(Automerge.init()) assert.deepStrictEqual(changes, []) }) it('should return an empty list when nothing changed', () => { let s1 = Automerge.change(Automerge.init(), doc => doc.birds = ['Chaffinch']) assert.deepStrictEqual(Automerge.getChanges(s1, s1), []) }) it('should do nothing when applying an empty list of changes', () => { let s1 = Automerge.change(Automerge.init(), doc => doc.birds = ['Chaffinch']) assert.deepStrictEqual(Automerge.applyChanges(s1, [])[0], s1) }) it('should throw a useful error if the wrong type of argument is passed to applyChanges', () => { let s1 = Automerge.change(Automerge.init(), doc => doc.birds = ['Chaffinch']) let changes = Automerge.getAllChanges(s1) assert.throws(() => { Automerge.applyChanges(Automerge.init(), changes[0]) }, /applyChanges takes an array of Uint8Arrays/) assert.throws(() => { Automerge.applyChanges(Automerge.init(), changes[0].buffer) }, /applyChanges takes an array of Uint8Arrays/) assert.throws(() => { Automerge.applyChanges(Automerge.init(), ['this is a string']) }, /Not a byte array/) assert.throws(() => { Automerge.applyChanges(Automerge.init(), changes.map(change => change.buffer)) }, /Not a byte array/) }) it('should return all changes when compared to an empty document', () => { let s1 = Automerge.change(Automerge.init(), 'Add Chaffinch', doc => doc.birds = ['Chaffinch']) let s2 = Automerge.change(s1, 'Add Bullfinch', doc => doc.birds.push('Bullfinch')) let changes = Automerge.getChanges(Automerge.init(), s2) assert.strictEqual(changes.length, 2) }) it('should allow a document copy to be reconstructed from scratch', () => { let s1 = Automerge.change(Automerge.init(), 'Add Chaffinch', doc => doc.birds = ['Chaffinch']) let s2 = Automerge.change(s1, 'Add Bullfinch', doc => doc.birds.push('Bullfinch')) let changes = Automerge.getAllChanges(s2) let [s3] = Automerge.applyChanges(Automerge.init(), changes) assert.deepStrictEqual(s3.birds, ['Chaffinch', 'Bullfinch']) }) it('should return changes since the last given version', () => { let s1 = Automerge.change(Automerge.init(), 'Add Chaffinch', doc => doc.birds = ['Chaffinch']) let changes1 = Automerge.getAllChanges(s1) let s2 = Automerge.change(s1, 'Add Bullfinch', doc => doc.birds.push('Bullfinch')) let changes2 = Automerge.getChanges(s1, s2) assert.strictEqual(changes1.length, 1) // Add Chaffinch assert.strictEqual(changes2.length, 1) // Add Bullfinch }) it('should incrementally apply changes since the last given version', () => { let s1 = Automerge.change(Automerge.init(), 'Add Chaffinch', doc => doc.birds = ['Chaffinch']) let changes1 = Automerge.getAllChanges(s1) let s2 = Automerge.change(s1, 'Add Bullfinch', doc => doc.birds.push('Bullfinch')) let changes2 = Automerge.getChanges(s1, s2) let [s3] = Automerge.applyChanges(Automerge.init(), changes1) let [s4] = Automerge.applyChanges(s3, changes2) assert.deepStrictEqual(s3.birds, ['Chaffinch']) assert.deepStrictEqual(s4.birds, ['Chaffinch', 'Bullfinch']) }) it('should handle updates to a list element', () => { let s1 = Automerge.change(Automerge.init(), doc => doc.birds = ['Chaffinch', 'Bullfinch']) let s2 = Automerge.change(s1, doc => doc.birds[0] = 'Goldfinch') let [s3] = Automerge.applyChanges(Automerge.init(), Automerge.getAllChanges(s2)) assert.deepStrictEqual(s3.birds, ['Goldfinch', 'Bullfinch']) assert.strictEqual(Automerge.getConflicts(s3.birds, 0), undefined) }) it('should handle updates to a text object', () => { let s1 = Automerge.change(Automerge.init(), doc => doc.text = new Automerge.Text('ab')) let s2 = Automerge.change(s1, doc => doc.text.set(0, 'A')) let [s3] = Automerge.applyChanges(Automerge.init(), Automerge.getAllChanges(s2)) assert.deepStrictEqual([...s3.text], ['A', 'b']) }) it('should report missing dependencies', () => { let s1 = Automerge.change(Automerge.init(), doc => doc.birds = ['Chaffinch']) let s2 = Automerge.merge(Automerge.init(), s1) s2 = Automerge.change(s2, doc => doc.birds.push('Bullfinch')) let changes = Automerge.getAllChanges(s2) let [s3, patch] = Automerge.applyChanges(Automerge.init(), [changes[1]]) assert.deepStrictEqual(s3, {}) assert.deepStrictEqual(Automerge.Backend.getMissingDeps(Automerge.Frontend.getBackendState(s3)), decodeChange(changes[1]).deps) assert.strictEqual(patch.pendingChanges, 1) ;[s3, patch] = Automerge.applyChanges(s3, [changes[0]]) assert.deepStrictEqual(s3.birds, ['Chaffinch', 'Bullfinch']) assert.deepStrictEqual(Automerge.Backend.getMissingDeps(Automerge.Frontend.getBackendState(s3)), []) assert.strictEqual(patch.pendingChanges, 0) }) it('should allow changes to be applied in any order', () => { let s1 = Automerge.change(Automerge.init(), doc => doc.bird = 'Goldfinch') let s2 = Automerge.change(s1, doc => doc.bird = 'Chaffinch') let s3 = Automerge.change(s2, doc => doc.bird = 'Greenfinch') let changes = Automerge.getAllChanges(s3).reverse() let [s4] = Automerge.applyChanges(Automerge.init(), changes) assert.deepStrictEqual(s4, {bird: 'Greenfinch'}) }) it('should report missing dependencies with out-of-order applyChanges', () => { let s0 = Automerge.init() let s1 = Automerge.change(s0, doc => doc.test = ['a']) let changes01 = Automerge.getAllChanges(s1) let s2 = Automerge.change(s1, doc => doc.test = ['b']) let changes12 = Automerge.getChanges(s1, s2) let s3 = Automerge.change(s2, doc => doc.test = ['c']) let changes23 = Automerge.getChanges(s2, s3) let s4 = Automerge.init() let [s5] = Automerge.applyChanges(s4, changes23) let [s6, patch6] = Automerge.applyChanges(s5, changes12) assert.deepStrictEqual(Automerge.Backend.getMissingDeps(Automerge.Frontend.getBackendState(s6)), [decodeChange(changes01[0]).hash]) assert.strictEqual(patch6.pendingChanges, 2) }) it('should call patchCallback if supplied when applying changes', () => { const s1 = Automerge.change(Automerge.init(), doc => doc.birds = ['Goldfinch']) const callbacks = [], actor = Automerge.getActorId(s1) const before = Automerge.init() const [after, patch] = Automerge.applyChanges(before, Automerge.getAllChanges(s1), { patchCallback(patch, before, after, local) { callbacks.push({patch, before, after, local}) } }) assert.strictEqual(callbacks.length, 1) assert.deepStrictEqual(callbacks[0].patch, { maxOp: 2, deps: [decodeChange(Automerge.getAllChanges(s1)[0]).hash], clock: {[actor]: 1}, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {birds: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'list', edits: [ {action: 'insert', index: 0, elemId: `2@${actor}`, opId: `2@${actor}`, value: {type: 'value', value: 'Goldfinch'}} ] }}}} }) assert.strictEqual(callbacks[0].patch, patch) assert.strictEqual(callbacks[0].before, before) assert.strictEqual(callbacks[0].after, after) assert.strictEqual(callbacks[0].local, false) }) it('should merge multiple applied changes into one patch', () => { const s1 = Automerge.change(Automerge.init(), doc => doc.birds = ['Goldfinch']) const s2 = Automerge.change(s1, doc => doc.birds.push('Chaffinch')) const patches = [], actor = Automerge.getActorId(s2) Automerge.applyChanges(Automerge.init(), Automerge.getAllChanges(s2), {patchCallback: p => patches.push(p)}) assert.deepStrictEqual(patches, [{ maxOp: 3, deps: [decodeChange(Automerge.getAllChanges(s2)[1]).hash], clock: {[actor]: 2}, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {birds: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'list', edits: [ {action: 'multi-insert', index: 0, elemId: `2@${actor}`, values: ['Goldfinch', 'Chaffinch']} ] }}}} }]) }) it('should call a patchCallback registered on doc initialisation', () => { const s1 = Automerge.change(Automerge.init(), doc => doc.bird = 'Goldfinch') const patches = [], actor = Automerge.getActorId(s1) const before = Automerge.init({patchCallback: p => patches.push(p)}) Automerge.applyChanges(before, Automerge.getAllChanges(s1)) assert.deepStrictEqual(patches, [{ maxOp: 1, deps: [decodeChange(Automerge.getAllChanges(s1)[0]).hash], clock: {[actor]: 1}, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {bird: {[`1@${actor}`]: {type: 'value', value: 'Goldfinch'}}}} }]) }) }) }) ================================================ FILE: test/text_test.js ================================================ const assert = require('assert') const Automerge = process.env.TEST_DIST === '1' ? require('../dist/automerge') : require('../src/automerge') const { assertEqualsOneOf } = require('./helpers') function attributeStateToAttributes(accumulatedAttributes) { const attributes = {} Object.entries(accumulatedAttributes).forEach(([key, values]) => { if (values.length && values[0] !== null) { attributes[key] = values[0] } }) return attributes } function isEquivalent(a, b) { const aProps = Object.getOwnPropertyNames(a) const bProps = Object.getOwnPropertyNames(b) if (aProps.length != bProps.length) { return false } for (let i = 0; i < aProps.length; i++) { const propName = aProps[i] if (a[propName] !== b[propName]) { return false } } return true } function isControlMarker(pseudoCharacter) { return typeof pseudoCharacter === 'object' && pseudoCharacter.attributes } function opFrom(text, attributes) { let op = { insert: text } if (Object.keys(attributes).length > 0) { op.attributes = attributes } return op } function accumulateAttributes(span, accumulatedAttributes) { Object.entries(span).forEach(([key, value]) => { if (!accumulatedAttributes[key]) { accumulatedAttributes[key] = [] } if (value === null) { if (accumulatedAttributes[key].length === 0 || accumulatedAttributes[key] === null) { accumulatedAttributes[key].unshift(null) } else { accumulatedAttributes[key].shift() } } else { if (accumulatedAttributes[key][0] === null) { accumulatedAttributes[key].shift() } else { accumulatedAttributes[key].unshift(value) } } }) return accumulatedAttributes } function automergeTextToDeltaDoc(text) { let ops = [] let controlState = {} let currentString = "" let attributes = {} text.toSpans().forEach((span) => { if (isControlMarker(span)) { controlState = accumulateAttributes(span.attributes, controlState) } else { let next = attributeStateToAttributes(controlState) // if the next span has the same calculated attributes as the current span // don't bother outputting it as a separate span, just let it ride if (typeof span === 'string' && isEquivalent(next, attributes)) { currentString = currentString + span return } if (currentString) { ops.push(opFrom(currentString, attributes)) } // If we've got a string, we might be able to concatenate it to another // same-attributed-string, so remember it and go to the next iteration. if (typeof span === 'string') { currentString = span attributes = next } else { // otherwise we have an embed "character" and should output it immediately. // embeds are always one-"character" in length. ops.push(opFrom(span, next)) currentString = '' attributes = {} } } }) // at the end, flush any accumulated string out if (currentString) { ops.push(opFrom(currentString, attributes)) } return ops } function inverseAttributes(attributes) { let invertedAttributes = {} Object.keys(attributes).forEach((key) => { invertedAttributes[key] = null }) return invertedAttributes } function applyDeleteOp(text, offset, op) { let length = op.delete while (length > 0) { if (isControlMarker(text.get(offset))) { offset += 1 } else { // we need to not delete control characters, but we do delete embed characters text.deleteAt(offset, 1) length -= 1 } } return [text, offset] } function applyRetainOp(text, offset, op) { let length = op.retain if (op.attributes) { text.insertAt(offset, { attributes: op.attributes }) offset += 1 } while (length > 0) { const char = text.get(offset) offset += 1 if (!isControlMarker(char)) { length -= 1 } } if (op.attributes) { text.insertAt(offset, { attributes: inverseAttributes(op.attributes) }) offset += 1 } return [text, offset] } function applyInsertOp(text, offset, op) { let originalOffset = offset if (typeof op.insert === 'string') { text.insertAt(offset, ...op.insert.split('')) offset += op.insert.length } else { // we have an embed or something similar text.insertAt(offset, op.insert) offset += 1 } if (op.attributes) { text.insertAt(originalOffset, { attributes: op.attributes }) offset += 1 } if (op.attributes) { text.insertAt(offset, { attributes: inverseAttributes(op.attributes) }) offset += 1 } return [text, offset] } // XXX: uhhhhh, why can't I pass in text? function applyDeltaDocToAutomergeText(delta, doc) { let offset = 0 delta.forEach(op => { if (op.retain) { [, offset] = applyRetainOp(doc.text, offset, op) } else if (op.delete) { [, offset] = applyDeleteOp(doc.text, offset, op) } else if (op.insert) { [, offset] = applyInsertOp(doc.text, offset, op) } }) } describe('Automerge.Text', () => { let s1, s2 beforeEach(() => { s1 = Automerge.change(Automerge.init(), doc => doc.text = new Automerge.Text()) s2 = Automerge.merge(Automerge.init(), s1) }) it('should support insertion', () => { s1 = Automerge.change(s1, doc => doc.text.insertAt(0, 'a')) assert.strictEqual(s1.text.length, 1) assert.strictEqual(s1.text.get(0), 'a') assert.strictEqual(s1.text.toString(), 'a') assert.strictEqual(s1.text.getElemId(0), `2@${Automerge.getActorId(s1)}`) }) it('should support deletion', () => { s1 = Automerge.change(s1, doc => doc.text.insertAt(0, 'a', 'b', 'c')) s1 = Automerge.change(s1, doc => doc.text.deleteAt(1, 1)) assert.strictEqual(s1.text.length, 2) assert.strictEqual(s1.text.get(0), 'a') assert.strictEqual(s1.text.get(1), 'c') assert.strictEqual(s1.text.toString(), 'ac') }) it("should support implicit and explicit deletion", () => { s1 = Automerge.change(s1, doc => doc.text.insertAt(0, "a", "b", "c")) s1 = Automerge.change(s1, doc => doc.text.deleteAt(1)) s1 = Automerge.change(s1, doc => doc.text.deleteAt(1, 0)) assert.strictEqual(s1.text.length, 2) assert.strictEqual(s1.text.get(0), "a") assert.strictEqual(s1.text.get(1), "c") assert.strictEqual(s1.text.toString(), "ac") }) it('should handle concurrent insertion', () => { s1 = Automerge.change(s1, doc => doc.text.insertAt(0, 'a', 'b', 'c')) s2 = Automerge.change(s2, doc => doc.text.insertAt(0, 'x', 'y', 'z')) s1 = Automerge.merge(s1, s2) assert.strictEqual(s1.text.length, 6) assertEqualsOneOf(s1.text.toString(), 'abcxyz', 'xyzabc') assertEqualsOneOf(s1.text.join(''), 'abcxyz', 'xyzabc') }) it('should handle text and other ops in the same change', () => { s1 = Automerge.change(s1, doc => { doc.foo = 'bar' doc.text.insertAt(0, 'a') }) assert.strictEqual(s1.foo, 'bar') assert.strictEqual(s1.text.toString(), 'a') assert.strictEqual(s1.text.join(''), 'a') }) it('should serialize to JSON as a simple string', () => { s1 = Automerge.change(s1, doc => doc.text.insertAt(0, 'a', '"', 'b')) assert.strictEqual(JSON.stringify(s1), '{"text":"a\\"b"}') }) it('should allow modification before an object is assigned to a document', () => { s1 = Automerge.change(Automerge.init(), doc => { const text = new Automerge.Text() text.insertAt(0, 'a', 'b', 'c', 'd') text.deleteAt(2) doc.text = text assert.strictEqual(doc.text.toString(), 'abd') assert.strictEqual(doc.text.join(''), 'abd') }) assert.strictEqual(s1.text.toString(), 'abd') assert.strictEqual(s1.text.join(''), 'abd') }) it('should allow modification after an object is assigned to a document', () => { s1 = Automerge.change(Automerge.init(), doc => { const text = new Automerge.Text() doc.text = text doc.text.insertAt(0, 'a', 'b', 'c', 'd') doc.text.deleteAt(2) assert.strictEqual(doc.text.toString(), 'abd') assert.strictEqual(doc.text.join(''), 'abd') }) assert.strictEqual(s1.text.join(''), 'abd') }) it('should not allow modification outside of a change callback', () => { assert.throws(() => s1.text.insertAt(0, 'a'), /Text object cannot be modified outside of a change block/) }) describe('with initial value', () => { it('should accept a string as initial value', () => { let s1 = Automerge.change(Automerge.init(), doc => doc.text = new Automerge.Text('init')) assert.strictEqual(s1.text.length, 4) assert.strictEqual(s1.text.get(0), 'i') assert.strictEqual(s1.text.get(1), 'n') assert.strictEqual(s1.text.get(2), 'i') assert.strictEqual(s1.text.get(3), 't') assert.strictEqual(s1.text.toString(), 'init') }) it('should accept an array as initial value', () => { let s1 = Automerge.change(Automerge.init(), doc => doc.text = new Automerge.Text(['i', 'n', 'i', 't'])) assert.strictEqual(s1.text.length, 4) assert.strictEqual(s1.text.get(0), 'i') assert.strictEqual(s1.text.get(1), 'n') assert.strictEqual(s1.text.get(2), 'i') assert.strictEqual(s1.text.get(3), 't') assert.strictEqual(s1.text.toString(), 'init') }) it('should initialize text in Automerge.from()', () => { let s1 = Automerge.from({text: new Automerge.Text('init')}) assert.strictEqual(s1.text.length, 4) assert.strictEqual(s1.text.get(0), 'i') assert.strictEqual(s1.text.get(1), 'n') assert.strictEqual(s1.text.get(2), 'i') assert.strictEqual(s1.text.get(3), 't') assert.strictEqual(s1.text.toString(), 'init') }) it('should encode the initial value as a change', () => { const s1 = Automerge.from({text: new Automerge.Text('init')}) const changes = Automerge.getAllChanges(s1) assert.strictEqual(changes.length, 1) const [s2] = Automerge.applyChanges(Automerge.init(), changes) assert.strictEqual(s2.text instanceof Automerge.Text, true) assert.strictEqual(s2.text.toString(), 'init') assert.strictEqual(s2.text.join(''), 'init') }) it('should allow immediate access to the value', () => { Automerge.change(Automerge.init(), doc => { const text = new Automerge.Text('init') assert.strictEqual(text.length, 4) assert.strictEqual(text.get(0), 'i') assert.strictEqual(text.toString(), 'init') doc.text = text assert.strictEqual(doc.text.length, 4) assert.strictEqual(doc.text.get(0), 'i') assert.strictEqual(doc.text.toString(), 'init') }) }) it('should allow pre-assignment modification of the initial value', () => { let s1 = Automerge.change(Automerge.init(), doc => { const text = new Automerge.Text('init') text.deleteAt(3) assert.strictEqual(text.join(''), 'ini') doc.text = text assert.strictEqual(doc.text.join(''), 'ini') assert.strictEqual(doc.text.toString(), 'ini') }) assert.strictEqual(s1.text.toString(), 'ini') assert.strictEqual(s1.text.join(''), 'ini') }) it('should allow post-assignment modification of the initial value', () => { let s1 = Automerge.change(Automerge.init(), doc => { const text = new Automerge.Text('init') doc.text = text doc.text.deleteAt(0) doc.text.insertAt(0, 'I') assert.strictEqual(doc.text.join(''), 'Init') assert.strictEqual(doc.text.toString(), 'Init') }) assert.strictEqual(s1.text.join(''), 'Init') assert.strictEqual(s1.text.toString(), 'Init') }) }) describe('non-textual control characters', () => { let s1 beforeEach(() => { s1 = Automerge.change(Automerge.init(), doc => { doc.text = new Automerge.Text() doc.text.insertAt(0, 'a') doc.text.insertAt(1, { attribute: 'bold' }) }) }) it('should allow fetching non-textual characters', () => { assert.deepEqual(s1.text.get(1), { attribute: 'bold' }) assert.strictEqual(s1.text.getElemId(1), `3@${Automerge.getActorId(s1)}`) }) it('should include control characters in string length', () => { assert.strictEqual(s1.text.length, 2) assert.strictEqual(s1.text.get(0), 'a') }) it('should exclude control characters from toString()', () => { assert.strictEqual(s1.text.toString(), 'a') }) it('should allow control characters to be updated', () => { const s2 = Automerge.change(s1, doc => doc.text.get(1).attribute = 'italic') const s3 = Automerge.load(Automerge.save(s2)) assert.strictEqual(s1.text.get(1).attribute, 'bold') assert.strictEqual(s2.text.get(1).attribute, 'italic') assert.strictEqual(s3.text.get(1).attribute, 'italic') }) describe('spans interface to Text', () => { it('should return a simple string as a single span', () => { let s1 = Automerge.change(Automerge.init(), doc => { doc.text = new Automerge.Text('hello world') }) assert.deepEqual(s1.text.toSpans(), ['hello world']) }) it('should return an empty string as an empty array', () => { let s1 = Automerge.change(Automerge.init(), doc => { doc.text = new Automerge.Text() }) assert.deepEqual(s1.text.toSpans(), []) }) it('should split a span at a control character', () => { let s1 = Automerge.change(Automerge.init(), doc => { doc.text = new Automerge.Text('hello world') doc.text.insertAt(5, { attributes: { bold: true } }) }) assert.deepEqual(s1.text.toSpans(), ['hello', { attributes: { bold: true } }, ' world']) }) it('should allow consecutive control characters', () => { let s1 = Automerge.change(Automerge.init(), doc => { doc.text = new Automerge.Text('hello world') doc.text.insertAt(5, { attributes: { bold: true } }) doc.text.insertAt(6, { attributes: { italic: true } }) }) assert.deepEqual(s1.text.toSpans(), ['hello', { attributes: { bold: true } }, { attributes: { italic: true } }, ' world' ]) }) it('should allow non-consecutive control characters', () => { let s1 = Automerge.change(Automerge.init(), doc => { doc.text = new Automerge.Text('hello world') doc.text.insertAt(5, { attributes: { bold: true } }) doc.text.insertAt(12, { attributes: { italic: true } }) }) assert.deepEqual(s1.text.toSpans(), ['hello', { attributes: { bold: true } }, ' world', { attributes: { italic: true } } ]) }) it('should be convertable into a Quill delta', () => { let s1 = Automerge.change(Automerge.init(), doc => { doc.text = new Automerge.Text('Gandalf the Grey') doc.text.insertAt(0, { attributes: { bold: true } }) doc.text.insertAt(7 + 1, { attributes: { bold: null } }) doc.text.insertAt(12 + 2, { attributes: { color: '#cccccc' } }) }) let deltaDoc = automergeTextToDeltaDoc(s1.text) // From https://quilljs.com/docs/delta/ let expectedDoc = [ { insert: 'Gandalf', attributes: { bold: true } }, { insert: ' the ' }, { insert: 'Grey', attributes: { color: '#cccccc' } } ] assert.deepEqual(deltaDoc, expectedDoc) }) it('should support embeds', () => { let s1 = Automerge.change(Automerge.init(), doc => { doc.text = new Automerge.Text('') doc.text.insertAt(0, { attributes: { link: 'https://quilljs.com' } }) doc.text.insertAt(1, { image: 'https://quilljs.com/assets/images/icon.png' }) doc.text.insertAt(2, { attributes: { link: null } }) }) let deltaDoc = automergeTextToDeltaDoc(s1.text) // From https://quilljs.com/docs/delta/ let expectedDoc = [{ // An image link insert: { image: 'https://quilljs.com/assets/images/icon.png' }, attributes: { link: 'https://quilljs.com' } }] assert.deepEqual(deltaDoc, expectedDoc) }) it('should handle concurrent overlapping spans', () => { let s1 = Automerge.change(Automerge.init(), doc => { doc.text = new Automerge.Text('Gandalf the Grey') }) let s2 = Automerge.merge(Automerge.init(), s1) let s3 = Automerge.change(s1, doc => { doc.text.insertAt(8, { attributes: { bold: true } }) doc.text.insertAt(16 + 1, { attributes: { bold: null } }) }) let s4 = Automerge.change(s2, doc => { doc.text.insertAt(0, { attributes: { bold: true } }) doc.text.insertAt(11 + 1, { attributes: { bold: null } }) }) let merged = Automerge.merge(s3, s4) let deltaDoc = automergeTextToDeltaDoc(merged.text) // From https://quilljs.com/docs/delta/ let expectedDoc = [ { insert: 'Gandalf the Grey', attributes: { bold: true } }, ] assert.deepEqual(deltaDoc, expectedDoc) }) it('should handle debolding spans', () => { let s1 = Automerge.change(Automerge.init(), doc => { doc.text = new Automerge.Text('Gandalf the Grey') }) let s2 = Automerge.merge(Automerge.init(), s1) let s3 = Automerge.change(s1, doc => { doc.text.insertAt(0, { attributes: { bold: true } }) doc.text.insertAt(16 + 1, { attributes: { bold: null } }) }) let s4 = Automerge.change(s2, doc => { doc.text.insertAt(8, { attributes: { bold: null } }) doc.text.insertAt(11 + 1, { attributes: { bold: true } }) }) let merged = Automerge.merge(s3, s4) let deltaDoc = automergeTextToDeltaDoc(merged.text) // From https://quilljs.com/docs/delta/ let expectedDoc = [ { insert: 'Gandalf ', attributes: { bold: true } }, { insert: 'the' }, { insert: ' Grey', attributes: { bold: true } }, ] assert.deepEqual(deltaDoc, expectedDoc) }) // xxx: how would this work for colors? it('should handle destyling across destyled spans', () => { let s1 = Automerge.change(Automerge.init(), doc => { doc.text = new Automerge.Text('Gandalf the Grey') }) let s2 = Automerge.merge(Automerge.init(), s1) let s3 = Automerge.change(s1, doc => { doc.text.insertAt(0, { attributes: { bold: true } }) doc.text.insertAt(16 + 1, { attributes: { bold: null } }) }) let s4 = Automerge.change(s2, doc => { doc.text.insertAt(8, { attributes: { bold: null } }) doc.text.insertAt(11 + 1, { attributes: { bold: true } }) }) let merged = Automerge.merge(s3, s4) let final = Automerge.change(merged, doc => { doc.text.insertAt(3 + 1, { attributes: { bold: null } }) doc.text.insertAt(doc.text.length, { attributes: { bold: true } }) }) let deltaDoc = automergeTextToDeltaDoc(final.text) // From https://quilljs.com/docs/delta/ let expectedDoc = [ { insert: 'Gan', attributes: { bold: true } }, { insert: 'dalf the Grey' }, ] assert.deepEqual(deltaDoc, expectedDoc) }) it('should apply an insert', () => { let s1 = Automerge.change(Automerge.init(), doc => { doc.text = new Automerge.Text('Hello world') }) const delta = [ { retain: 6 }, { insert: 'reader' }, { delete: 5 } ] let s2 = Automerge.change(s1, doc => { applyDeltaDocToAutomergeText(delta, doc) }) assert.strictEqual(s2.text.join(''), 'Hello reader') }) it('should apply an insert with control characters', () => { let s1 = Automerge.change(Automerge.init(), doc => { doc.text = new Automerge.Text('Hello world') }) const delta = [ { retain: 6 }, { insert: 'reader', attributes: { bold: true } }, { delete: 5 }, { insert: '!' } ] let s2 = Automerge.change(s1, doc => { applyDeltaDocToAutomergeText(delta, doc) }) assert.strictEqual(s2.text.toString(), 'Hello reader!') assert.deepEqual(s2.text.toSpans(), [ "Hello ", { attributes: { bold: true } }, "reader", { attributes: { bold: null } }, "!" ]) }) it('should account for control characters in retain/delete lengths', () => { let s1 = Automerge.change(Automerge.init(), doc => { doc.text = new Automerge.Text('Hello world') doc.text.insertAt(4, { attributes: { color: '#ccc' } }) doc.text.insertAt(10, { attributes: { color: '#f00' } }) }) const delta = [ { retain: 6 }, { insert: 'reader', attributes: { bold: true } }, { delete: 5 }, { insert: '!' } ] let s2 = Automerge.change(s1, doc => { applyDeltaDocToAutomergeText(delta, doc) }) assert.strictEqual(s2.text.toString(), 'Hello reader!') assert.deepEqual(s2.text.toSpans(), [ "Hell", { attributes: { color: '#ccc'} }, "o ", { attributes: { bold: true } }, "reader", { attributes: { bold: null } }, { attributes: { color: '#f00'} }, "!" ]) }) it('should support embeds', () => { let s1 = Automerge.change(Automerge.init(), doc => { doc.text = new Automerge.Text('') }) let deltaDoc = [{ // An image link insert: { image: 'https://quilljs.com/assets/images/icon.png' }, attributes: { link: 'https://quilljs.com' } }] let s2 = Automerge.change(s1, doc => { applyDeltaDocToAutomergeText(deltaDoc, doc) }) assert.deepEqual(s2.text.toSpans(), [ { attributes: { link: 'https://quilljs.com' } }, { image: 'https://quilljs.com/assets/images/icon.png'}, { attributes: { link: null } }, ]) }) }) }) it('should support unicode when creating text', () => { s1 = Automerge.from({ text: new Automerge.Frontend.Text('🐦') }) assert.strictEqual(s1.text.get(0), '🐦') }) }) ================================================ FILE: test/typescript_test.ts ================================================ import * as assert from 'assert' import * as Automerge from 'automerge' import { Backend, Frontend, Counter, Doc } from 'automerge' const UUID_PATTERN = /^[0-9a-f]{32}$/ interface BirdList { birds: Automerge.List } interface NumberBox { number: number } describe('TypeScript support', () => { describe('Automerge.init()', () => { it('should allow a document to be `any`', () => { let s1 = Automerge.init() s1 = Automerge.change(s1, doc => (doc.key = 'value')) assert.strictEqual(s1.key, 'value') assert.strictEqual(s1.nonexistent, undefined) assert.deepStrictEqual(s1, { key: 'value' }) }) it('should allow a document type to be specified as a parameter to `init`', () => { let s1 = Automerge.init() // Note: Technically, `s1` is not really a `BirdList` yet but just an empty object. assert.equal(s1.hasOwnProperty('birds'), false) // Since we're pulling the wool over TypeScript's eyes, it can't give us compile-time protection // from something like this: // assert.equal(s1.birds.length, 0) // Runtime error: Cannot read property 'length' of undefined // Nevertheless this way seems more ergonomical (than having `init` return a type of `{}` or // `Partial`, for example) because it allows us to have a single type for the object // throughout its life, rather than having to recast it once its required fields have // been populated. s1 = Automerge.change(s1, doc => (doc.birds = ['goldfinch'])) assert.deepStrictEqual(s1.birds, ['goldfinch']) }) it('should allow a document type to be specified on the result of `init`', () => { // This is equivalent to passing the type parameter to `init`; note that the result is a // `Doc`, which is frozen let s1: Doc = Automerge.init() let s2 = Automerge.change(s1, doc => (doc.birds = ['goldfinch'])) assert.deepStrictEqual(s2.birds, ['goldfinch']) }) it('should allow a document to be initialized with `from`', () => { const s1 = Automerge.from({ birds: [] }) assert.strictEqual(s1.birds.length, 0) const s2 = Automerge.change(s1, doc => doc.birds.push('magpie')) assert.strictEqual(s2.birds[0], 'magpie') }) it('should allow passing options when initializing with `from`', () => { const actorId = '1234' const s1 = Automerge.from({ birds: [] }, actorId) assert.strictEqual(Automerge.getActorId(s1), '1234') const s2 = Automerge.from({ birds: [] }, { actorId }) assert.strictEqual(Automerge.getActorId(s2), '1234') }) it('should allow the actorId to be configured', () => { let s1 = Automerge.init('111111') assert.strictEqual(Automerge.getActorId(s1), '111111') let s2 = Automerge.init() assert.strictEqual(UUID_PATTERN.test(Automerge.getActorId(s2)), true) }) it('should allow the freeze option to be passed in', () => { let s1 = Automerge.init({ freeze: true }) let s2 = Automerge.change(s1, doc => (doc.birds = [])) assert.strictEqual(Object.isFrozen(s2), true) assert.strictEqual(Object.isFrozen(s2.birds), true) }) it('should allow a frontend to be `any`', () => { const s0 = Frontend.init() const [s1, req1] = Frontend.change(s0, doc => (doc.key = 'value')) assert.strictEqual(s1.key, 'value') assert.strictEqual(s1.nonexistent, undefined) assert.strictEqual(UUID_PATTERN.test(Frontend.getActorId(s1)), true) }) it('should allow a frontend type to be specified', () => { const s0 = Frontend.init() const [s1, req1] = Frontend.change(s0, doc => (doc.birds = ['goldfinch'])) assert.strictEqual(s1.birds[0], 'goldfinch') assert.deepStrictEqual(s1, { birds: ['goldfinch'] }) }) it('should allow a frontend actorId to be configured', () => { const s0 = Frontend.init('111111') assert.strictEqual(Frontend.getActorId(s0), '111111') }) it('should allow frontend actorId assignment to be deferred', () => { const s0 = Frontend.init({ deferActorId: true }) assert.strictEqual(Frontend.getActorId(s0), undefined) const s1 = Frontend.setActorId(s0, 'abcdef1234') const [s2, req] = Frontend.change(s1, doc => (doc.number = 15)) assert.deepStrictEqual(s2, { number: 15 }) }) it('should allow a frontend to be initialized with `from`', () => { const [s1, req1] = Frontend.from({ birds: [] }) assert.strictEqual(s1.birds.length, 0) const [s2, req2] = Frontend.change(s1, doc => doc.birds.push('magpie')) assert.strictEqual(s2.birds[0], 'magpie') }) it('should allow options to be passed to Frontend.from()', () => { const [s1, req1] = Frontend.from({ birds: []}, { actorId: '1234' }) assert.strictEqual(Frontend.getActorId(s1), '1234') assert.deepStrictEqual(s1, { birds: [] }) const [s2, req2] = Frontend.from({ birds: []}, '1234') assert.strictEqual(Frontend.getActorId(s2), '1234') }) it('should allow the length of the array to be increased', () => { let s1: Doc = Automerge.from({ birds: []}) let s2 = Automerge.change(s1, doc => doc.birds.length = 1) assert.deepStrictEqual(s2.birds, [null]) }) it('should allow the length of the array to be decreased', () => { let s1: Doc = Automerge.from({ birds: ['1234']}) let s2 = Automerge.change(s1, doc => doc.birds.length = 0) assert.deepStrictEqual(s2.birds, []) }) it('should throw error if length is invalid', () => { let s1: Doc = Automerge.from({ birds: ['1234']}) assert.throws(() => Automerge.change(s1, doc => { doc.birds.length = undefined }), "array length") assert.throws(() => Automerge.change(s1, doc => { doc.birds.length = NaN }), "array length") }) }) describe('saving and loading', () => { it('should allow an `any` type document to be loaded', () => { let s1 = Automerge.init() s1 = Automerge.change(s1, doc => (doc.key = 'value')) let s2: any = Automerge.load(Automerge.save(s1)) assert.strictEqual(s2.key, 'value') assert.deepStrictEqual(s2, { key: 'value' }) }) it('should allow a document of declared type to be loaded', () => { let s1 = Automerge.init() s1 = Automerge.change(s1, doc => (doc.birds = ['goldfinch'])) let s2 = Automerge.load(Automerge.save(s1)) assert.strictEqual(s2.birds[0], 'goldfinch') assert.deepStrictEqual(s2, { birds: ['goldfinch'] }) assert.strictEqual(UUID_PATTERN.test(Automerge.getActorId(s2)), true) }) it('should allow the actorId to be configured', () => { let s1 = Automerge.init() s1 = Automerge.change(s1, doc => (doc.birds = ['goldfinch'])) let s2 = Automerge.load(Automerge.save(s1), '111111') assert.strictEqual(Automerge.getActorId(s2), '111111') }) it('should allow the freeze option to be passed in', () => { let s1 = Automerge.init() s1 = Automerge.change(s1, doc => (doc.birds = ['goldfinch'])) let s2 = Automerge.load(Automerge.save(s1), { freeze: true }) assert.strictEqual(Object.isFrozen(s2), true) assert.strictEqual(Object.isFrozen(s2.birds), true) }) }) describe('making changes', () => { it('should accept an optional message', () => { let s1 = Automerge.init() s1 = Automerge.change(s1, 'hello', doc => (doc.birds = [])) assert.strictEqual(Automerge.getHistory(s1)[0].change.message, 'hello') }) it('should support list modifications', () => { let s1: Doc = Automerge.init() s1 = Automerge.change(s1, doc => (doc.birds = ['goldfinch'])) s1 = Automerge.change(s1, doc => { doc.birds.insertAt(1, 'greenfinch', 'bullfinch', 'chaffinch') doc.birds.deleteAt(0) doc.birds.deleteAt(0, 2) }) assert.deepStrictEqual(s1, { birds: ['chaffinch'] }) }) it('should allow empty changes', () => { let s1 = Automerge.init() s1 = Automerge.emptyChange(s1, 'my message') assert.strictEqual(Automerge.getHistory(s1)[0].change.message, 'my message') }) it('should allow inspection of conflicts', () => { let s1 = Automerge.init('111111') s1 = Automerge.change(s1, doc => (doc.number = 3)) let s2 = Automerge.init('222222') s2 = Automerge.change(s2, doc => (doc.number = 42)) let s3 = Automerge.merge(s1, s2) assert.strictEqual(s3.number, 42) assert.deepStrictEqual( Automerge.getConflicts(s3, 'number'), { '1@111111': 3, '1@222222': 42 }) }) it('should allow changes in the frontend', () => { const s0 = Frontend.init() const [s1, change1] = Frontend.change(s0, doc => (doc.birds = ['goldfinch'])) const [s2, change2] = Frontend.change(s1, doc => doc.birds.push('chaffinch')) assert.strictEqual(s2.birds[1], 'chaffinch') assert.deepStrictEqual(s2, { birds: ['goldfinch', 'chaffinch'] }) assert.strictEqual(change2.actor, Frontend.getActorId(s0)) assert.strictEqual(change2.seq, 2) assert.strictEqual(change2.time > 0, true) assert.strictEqual(change2.message, '') }) it('should accept a message in the frontend', () => { const s0 = Frontend.init() const [s1, req1] = Frontend.change(s0, 'test message', doc => (doc.number = 1)) assert.strictEqual(req1.message, 'test message') assert.strictEqual(req1.actor, Frontend.getActorId(s0)) assert.strictEqual(req1.ops.length, 1) }) it('should allow empty changes in the frontend', () => { const s0 = Frontend.init() const [s1, req1] = Frontend.emptyChange(s0, 'nothing happened') assert.strictEqual(req1.message, 'nothing happened') assert.strictEqual(req1.actor, Frontend.getActorId(s0)) assert.strictEqual(req1.ops.length, 0) }) it('should work with split frontend and backend', () => { const s0 = Frontend.init(), b0 = Backend.init() const [s1, change1] = Frontend.change(s0, doc => (doc.number = 1)) const [b1, patch1] = Backend.applyLocalChange(b0, change1) const s2 = Frontend.applyPatch(s1, patch1) assert.strictEqual(s2.number, 1) assert.strictEqual(patch1.actor, Automerge.getActorId(s0)) assert.strictEqual(patch1.seq, 1) assert.strictEqual(patch1.diffs.objectId, '_root') assert.strictEqual(patch1.diffs.type, 'map') assert.deepStrictEqual(Object.keys(patch1.diffs.props), ['number']) const value = patch1.diffs.props.number[`1@${Automerge.getActorId(s0)}`] assert.strictEqual((value as Automerge.ValueDiff).value, 1) }) }) describe('getting and applying changes', () => { it('should return an array of change objects', () => { let s1 = Automerge.init() s1 = Automerge.change(s1, doc => (doc.birds = ['goldfinch'])) let s2 = Automerge.change(s1, 'add chaffinch', doc => doc.birds.push('chaffinch')) const changes = Automerge.getChanges(s1, s2) assert.strictEqual(changes.length, 1) const change = Automerge.decodeChange(changes[0]) assert.strictEqual(change.message, 'add chaffinch') assert.strictEqual(change.actor, Automerge.getActorId(s2)) assert.strictEqual(change.seq, 2) }) it('should include operations in changes', () => { let s1 = Automerge.init() s1 = Automerge.change(s1, doc => (doc.number = 3)) const changes = Automerge.getAllChanges(s1) assert.strictEqual(changes.length, 1) const change = Automerge.decodeChange(changes[0]) assert.strictEqual(change.ops.length, 1) assert.strictEqual(change.ops[0].action, 'set') assert.strictEqual(change.ops[0].obj, '_root') assert.strictEqual(change.ops[0].key, 'number') assert.strictEqual(change.ops[0].value, 3) }) it('should allow changes to be re-applied', () => { let s1 = Automerge.init() s1 = Automerge.change(s1, doc => (doc.birds = [])) let s2 = Automerge.change(s1, doc => doc.birds.push('goldfinch')) const changes = Automerge.getAllChanges(s2) let [s3, patch] = Automerge.applyChanges(Automerge.init(), changes) assert.deepStrictEqual(s3.birds, ['goldfinch']) }) it('should allow concurrent changes to be merged', () => { let s1 = Automerge.init() s1 = Automerge.change(s1, doc => (doc.birds = ['goldfinch'])) let s2 = Automerge.merge(Automerge.init(), s1) s1 = Automerge.change(s1, doc => doc.birds.unshift('greenfinch')) s2 = Automerge.change(s2, doc => doc.birds.push('chaffinch')) let s3 = Automerge.merge(s1, s2) assert.deepStrictEqual(s3.birds, ['greenfinch', 'goldfinch', 'chaffinch']) }) }) describe('history inspection', () => { it('should inspect document history', () => { const s0 = Automerge.init() const s1 = Automerge.change(s0, 'one', doc => (doc.number = 1)) const s2 = Automerge.change(s1, 'two', doc => (doc.number = 2)) const history = Automerge.getHistory(s2) assert.strictEqual(history.length, 2) assert.strictEqual(history[0].change.message, 'one') assert.strictEqual(history[1].change.message, 'two') assert.strictEqual(history[0].snapshot.number, 1) assert.strictEqual(history[1].snapshot.number, 2) }) }) describe('state inspection', () => { it('should support looking up objects by ID', () => { const s0 = Automerge.init() const s1 = Automerge.change(s0, doc => (doc.birds = ['goldfinch'])) const obj = Automerge.getObjectId(s1.birds) assert.strictEqual(Automerge.getObjectById(s1, obj).length, 1) assert.strictEqual(Automerge.getObjectById(s1, obj), s1.birds) }) it('should allow looking up list element IDs', () => { const s0 = Automerge.init() const s1 = Automerge.change(s0, doc => (doc.birds = ['goldfinch'])) const elemIds = Automerge.Frontend.getElementIds(s1.birds) assert.deepStrictEqual(elemIds, [`2@${Automerge.getActorId(s1)}`]) }) }) describe('Automerge.Text', () => { interface TextDoc { text: Automerge.Text } let doc: Doc beforeEach(() => { doc = Automerge.change(Automerge.init(), doc => (doc.text = new Automerge.Text())) }) describe('insertAt', () => { it('should support inserting a single element', () => { doc = Automerge.change(doc, doc => doc.text.insertAt(0, 'abc')) assert.strictEqual(JSON.stringify(doc.text), '"abc"') }) it('should support inserting multiple elements', () => { doc = Automerge.change(doc, doc => doc.text.insertAt(0, 'a', 'b', 'c')) assert.strictEqual(JSON.stringify(doc.text), '"abc"') }) }) describe('deleteAt', () => { beforeEach(() => { doc = Automerge.change(doc, doc => doc.text.insertAt(0, 'a', 'b', 'c', 'd', 'e', 'f', 'g')) }) it('should support deleting a single element without specifying `numDelete`', () => { doc = Automerge.change(doc, doc => doc.text.deleteAt(2)) assert.strictEqual(JSON.stringify(doc.text), '"abdefg"') }) it('should support deleting multiple elements', () => { doc = Automerge.change(doc, doc => doc.text.deleteAt(3, 2)) assert.strictEqual(JSON.stringify(doc.text), '"abcfg"') }) }) describe('get', () => { it('should get the element at the given index', () => { doc = Automerge.change(doc, doc => doc.text.insertAt(0, 'a', 'b', 'cdefg', 'hi', 'jkl')) assert.strictEqual(doc.text.get(0), 'a') assert.strictEqual(doc.text.get(2), 'cdefg') }) }) describe('delegated read-only operations from `Array`', () => { const a = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i'] beforeEach(() => { doc = Automerge.change(doc, doc => doc.text.insertAt(0, ...a)) }) it('supports `indexOf`', () => assert.strictEqual(doc.text.indexOf('c'), 2)) it('supports `length`', () => assert.strictEqual(doc.text.length, 9)) it('supports `concat`', () => assert.strictEqual(doc.text.concat(['j']).length, 10)) it('supports `includes`', () => assert.strictEqual(doc.text.includes('q'), false)) }) describe('getElementIds', () => { it('should return the element ID of each character', () => { doc = Automerge.change(doc, doc => doc.text.insertAt(0, 'a', 'b')) const elemIds = Automerge.Frontend.getElementIds(doc.text) assert.deepStrictEqual(elemIds, [`2@${Automerge.getActorId(doc)}`, `3@${Automerge.getActorId(doc)}`]) }) }) }) describe('Automerge.Table', () => { interface Book { authors: string | string[] title: string isbn?: string } interface BookDb { books: Automerge.Table } // Example data const DDIA: Book = { authors: ['Kleppmann, Martin'], title: 'Designing Data-Intensive Applications', isbn: '1449373321', } const RSDP: Book = { authors: ['Cachin, Christian', 'Guerraoui, Rachid', 'Rodrigues, Luís'], title: 'Introduction to Reliable and Secure Distributed Programming', isbn: '3-642-15259-7', } let s1: Doc let id: Automerge.UUID let ddiaWithId: Book & Automerge.TableRow beforeEach(() => { s1 = Automerge.change(Automerge.init(), doc => { doc.books = new Automerge.Table() id = doc.books.add(DDIA) }) ddiaWithId = Object.assign({id}, DDIA) }) it('supports `byId`', () => assert.deepStrictEqual(s1.books.byId(id), ddiaWithId)) it('supports `count`', () => assert.strictEqual(s1.books.count, 1)) it('supports `ids`', () => assert.deepStrictEqual(s1.books.ids, [id])) it('supports iteration', () => assert.deepStrictEqual([...s1.books], [ddiaWithId])) it('allows adding row properties', () => { // Note that if we add columns and want to actually use them, we need to recast the table to a // new type e.g. without the `ts-ignore` flag, this would throw a type error: // @ts-ignore - Property 'publisher' does not exist on type book const p2 = s1.books.byId(id).publisher // So we need to create new types interface BookDeluxe extends Book { // ... existing properties, plus: publisher?: string } interface BookDeluxeDb { books: Automerge.Table } const s2 = s1 as Doc // Cast existing table to new type const s3 = Automerge.change( s2, doc => (doc.books.byId(id).publisher = "O'Reilly") ) // Now we're off to the races const p3 = s3.books.byId(id).publisher assert.strictEqual(p3, "O'Reilly") }) it('supports `remove`', () => { const s2 = Automerge.change(s1, doc => doc.books.remove(id)) assert.strictEqual(s2.books.count, 0) }) describe('supports `add`', () => { it('accepts value passed as object', () => { let bookId: string const s2 = Automerge.change(s1, doc => (bookId = doc.books.add(RSDP))) assert.deepStrictEqual(s2.books.byId(bookId), Object.assign({id: bookId}, RSDP)) assert.strictEqual(s2.books.byId(bookId).id, bookId) }) }) describe('standard array operations on rows', () => { it('returns a list of rows', () => assert.deepEqual(s1.books.rows, [ddiaWithId])) it('supports `filter`', () => assert.deepStrictEqual(s1.books.filter(book => book.authors.length === 1), [ddiaWithId])) it('supports `find`', () => { assert.deepStrictEqual(s1.books.find(book => book.isbn === '1449373321'), ddiaWithId)}) it('supports `map`', () => assert.deepStrictEqual(s1.books.map(book => book.title), [DDIA.title])) }) }) describe('Automerge.Counter', () => { interface CounterMap { [name: string]: Counter } interface CounterList { counts: Counter[] } interface BirdCounterMap { birds: CounterMap } it('should handle counters inside maps', () => { const doc1 = Automerge.change(Automerge.init(), doc => { doc.wrens = new Counter() }) assert.equal(doc1.wrens, 0) const doc2 = Automerge.change(doc1, doc => { doc.wrens.increment() }) assert.equal(doc2.wrens, 1) }) it('should handle counters inside lists', () => { const doc1 = Automerge.change(Automerge.init(), doc => { doc.counts = [new Counter(1)] }) assert.equal(doc1.counts[0], 1) const doc2 = Automerge.change(doc1, doc => { doc.counts[0].increment(2) }) assert.equal(doc2.counts[0].value, 3) }) describe('counter as numeric primitive', () => { let doc1: CounterMap beforeEach(() => { doc1 = Automerge.change(Automerge.init(), doc => { doc.birds = new Counter(3) }) }) it('is equal (==) but not strictly equal (===) to its numeric value', () => { assert.equal(doc1.birds, 3) assert.notStrictEqual(doc1.birds, 3) }) it('has to be explicitly cast to be used as a number', () => { let birdCount: number // This is valid javascript, but without the `ts-ignore` flag, it fails to compile: // @ts-ignore birdCount = doc1.birds // Type 'Counter' is not assignable to type 'number'.ts(2322) // This is because TypeScript doesn't know about the `.valueOf()` trick. // https://github.com/Microsoft/TypeScript/issues/2361 // If we want to treat a counter value as a number, we have to explicitly cast it to keep // TypeScript happy. // We can cast by putting a `+` in front of it: birdCount = +doc1.birds assert.equal(birdCount < 4, true) assert.equal(birdCount >= 0, true) // Or we can be explicit (have to cast as unknown, then number): birdCount = (doc1.birds as unknown) as number assert.equal(birdCount <= 2, false) assert.equal(birdCount + 10, 13) }) it('is converted to a string using its numeric value', () => { assert.equal(doc1.birds.toString(), '3') assert.equal(`I saw ${doc1.birds} birds`, 'I saw 3 birds') assert.equal(['I saw', doc1.birds, 'birds'].join(' '), 'I saw 3 birds') }) }) }) describe('Automerge.Observable', () => { interface TextDoc { text: Automerge.Text } it('should call a patchCallback when a document changes', () => { let callbackCalled = false, actor = '' let doc = Automerge.init({patchCallback: (patch, before, after, local, changes) => { callbackCalled = true assert.deepStrictEqual(patch.diffs.props.text[`1@${actor}`], { objectId: `1@${actor}`, type: 'text', edits: [ {action: 'insert', index: 0, elemId: `2@${actor}`, opId: `2@${actor}`, value: {type: 'value', value: 'a'}} ] }) assert.deepStrictEqual(before, {}) assert.strictEqual(after.text.toString(), 'a') assert.strictEqual(local, true) assert.strictEqual(changes.length, 1) assert.ok(changes[0] instanceof Uint8Array) }}) actor = Automerge.getActorId(doc) doc = Automerge.change(doc, doc => doc.text = new Automerge.Text('a')) assert.strictEqual(callbackCalled, true) }) it('should call an observer when a document changes', () => { let observable = new Automerge.Observable(), callbackCalled = false let doc = Automerge.from({text: new Automerge.Text()}, {observable}) let actor = Automerge.getActorId(doc) observable.observe(doc.text, (diff, before, after, local, changes) => { callbackCalled = true if (diff.type == 'text') { assert.deepStrictEqual(diff.edits, [ {action: 'insert', index: 0, elemId: `2@${actor}`, opId: `2@${actor}`, value: {type: 'value', value: 'a'}} ]) } assert.strictEqual(before.toString(), '') assert.strictEqual(after.toString(), 'a') assert.strictEqual(local, true) assert.strictEqual(changes.length, 1) assert.ok(changes[0] instanceof Uint8Array) }) doc = Automerge.change(doc, doc => doc.text.insertAt(0, 'a')) assert.strictEqual(callbackCalled, true) }) }) }) ================================================ FILE: test/uuid_test.js ================================================ const assert = require('assert') const Automerge = process.env.TEST_DIST === '1' ? require('../dist/automerge') : require('../src/automerge') const uuid = Automerge.uuid describe('uuid', () => { afterEach(() => { uuid.reset() }) describe('default implementation', () => { it('generates unique values', () => { assert.notEqual(uuid(), uuid()) }) }) describe('custom implementation', () => { let counter function customUuid() { return `custom-uuid-${counter++}` } before(() => uuid.setFactory(customUuid)) beforeEach(() => counter = 0) it('invokes the custom factory', () => { assert.equal(uuid(), 'custom-uuid-0') assert.equal(uuid(), 'custom-uuid-1') }) }) }) ================================================ FILE: test/wasm.js ================================================ /* eslint-disable no-unused-vars */ // This file is used for running the test suite against an alternative backend // implementation, such as the WebAssembly version compiled from Rust. // It needs to be loaded before the test suite files, which can be done with // `mocha --file test/wasm.js` (shortcut: `yarn testwasm`). // You need to set the environment variable WASM_BACKEND_PATH to the path where // the alternative backend module can be found; typically this is something // like `../automerge-rs/automerge-backend-wasm`. // Since this file relies on an environment variable and filesystem paths, it // currently only works in Node, not in a browser. if (!process.env.WASM_BACKEND_PATH) { throw new RangeError('Please set environment variable WASM_BACKEND_PATH to the path of the WebAssembly backend') } const assert = require('assert') const Automerge = process.env.TEST_DIST === '1' ? require('../dist/automerge') : require('../src/automerge') const jsBackend = require('../backend') const Frontend = require('../frontend') const { decodeChange } = require('../backend/columnar') const uuid = require('../src/uuid') const path = require('path') const wasmBackend = require(path.resolve(process.env.WASM_BACKEND_PATH)) Automerge.setDefaultBackend(wasmBackend) describe('JavaScript-WebAssembly interoperability', () => { describe('from JS to Wasm', () => { interopTests(jsBackend, wasmBackend) }) describe('from Wasm to JS', () => { interopTests(wasmBackend, jsBackend) }) }) function interopTests(sourceBackend, destBackend) { let source, dest beforeEach(() => { source = sourceBackend.init() dest = destBackend.init() }) it('should set a key in a map', () => { const actor = uuid() const [source1, p1, change1] = sourceBackend.applyLocalChange(source, { actor, seq: 1, time: 0, startOp: 1, deps: [], ops: [ {action: 'set', obj: '_root', key: 'bird', value: 'magpie', pred: []} ] }) const [dest1, patch] = destBackend.applyChanges(dest, [change1]) assert.deepStrictEqual(patch, { clock: {[actor]: 1}, deps: [decodeChange(change1).hash], maxOp: 1, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: { bird: {[`1@${actor}`]: {type: 'value', value: 'magpie'}} }} }) }) it('should delete a key from a map', () => { const actor = uuid() const [source1, p1, change1] = sourceBackend.applyLocalChange(source, { actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'set', obj: '_root', key: 'bird', value: 'magpie', pred: []} ] }) const [source2, p2, change2] = sourceBackend.applyLocalChange(source1, { actor, seq: 2, startOp: 2, time: 0, deps: [], ops: [ {action: 'del', obj: '_root', key: 'bird', pred: [`1@${actor}`]} ] }) const [dest1, patch1] = destBackend.applyChanges(dest, [change1]) const [dest2, patch2] = destBackend.applyChanges(dest1, [change2]) assert.deepStrictEqual(patch2, { clock: {[actor]: 2}, deps: [decodeChange(change2).hash], maxOp: 2, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {bird: {}}} }) }) it('should create nested maps', () => { const actor = uuid() const [source1, p1, change1] = sourceBackend.applyLocalChange(source, { actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeMap', obj: '_root', key: 'birds', pred: []}, {action: 'set', obj: `1@${actor}`, key: 'wrens', datatype: 'int', value: 3, pred: []} ] }) const [dest1, patch1] = destBackend.applyChanges(dest, [change1]) assert.deepStrictEqual(patch1, { clock: {[actor]: 1}, deps: [decodeChange(change1).hash], maxOp: 2, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {birds: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'map', props: {wrens: {[`2@${actor}`]: {type: 'value', datatype: 'int', value: 3}}} }}}} }) }) it('should create lists', () => { const actor = uuid() const [source1, p1, change1] = sourceBackend.applyLocalChange(source, { actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeList', obj: '_root', key: 'birds', pred: []}, {action: 'set', obj: `1@${actor}`, elemId: '_head', insert: true, value: 'chaffinch', pred: []} ] }) const [dest1, patch1] = destBackend.applyChanges(dest, [change1]) assert.deepStrictEqual(patch1, { clock: {[actor]: 1}, deps: [decodeChange(change1).hash], maxOp: 2, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {birds: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'list', edits: [ {action: 'insert', index: 0, elemId: `2@${actor}`, opId: `2@${actor}`, value: {type: 'value', value: 'chaffinch'}} ] }}}} }) }) it('should delete list elements', () => { const actor = uuid() const [source1, p1, change1] = sourceBackend.applyLocalChange(source, { actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeList', obj: '_root', key: 'birds', pred: []}, {action: 'set', obj: `1@${actor}`, elemId: '_head', insert: true, value: 'chaffinch', pred: []} ] }) const [source2, p2, change2] = sourceBackend.applyLocalChange(source1, { actor, seq: 2, startOp: 3, time: 0, deps: [], ops: [ {action: 'del', obj: `1@${actor}`, elemId: `2@${actor}`, pred: [`2@${actor}`]} ] }) const [dest1, patch1] = destBackend.applyChanges(dest, [change1]) const [dest2, patch2] = destBackend.applyChanges(dest1, [change2]) assert.deepStrictEqual(patch2, { clock: {[actor]: 2}, deps: [decodeChange(change2).hash], maxOp: 3, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {birds: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'list', edits: [{action: 'remove', index: 0, count: 1}] }}}} }) }) it('should support Text objects', () => { const actor = uuid() const [source1, p1, change1] = sourceBackend.applyLocalChange(source, { actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeText', obj: '_root', key: 'text', pred: []}, {action: 'set', obj: `1@${actor}`, elemId: '_head', insert: true, value: 'a', pred: []}, {action: 'set', obj: `1@${actor}`, elemId: `2@${actor}`, insert: true, value: 'b', pred: []}, {action: 'set', obj: `1@${actor}`, elemId: `3@${actor}`, insert: true, value: 'c', pred: []} ] }) const [dest1, patch1] = destBackend.applyChanges(dest, [change1]) assert.deepStrictEqual(patch1, { clock: {[actor]: 1}, deps: [decodeChange(change1).hash], maxOp: 4, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {text: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'text', edits: [ {action: 'multi-insert', index: 0, elemId: `2@${actor}`, values: ['a', 'b', 'c']}, ], }}}} }) }) it('should support Table objects', () => { const actor = uuid(), rowId = uuid() const [source1, p1, change1] = sourceBackend.applyLocalChange(source, { actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'makeTable', obj: '_root', key: 'birds', insert: false, pred: []}, {action: 'makeMap', obj: `1@${actor}`, key: rowId, insert: false, pred: []}, {action: 'set', obj: `2@${actor}`, key: 'species', insert: false, value: 'Chaffinch', pred: []}, {action: 'set', obj: `2@${actor}`, key: 'colour', insert: false, value: 'brown', pred: []} ] }) const [dest1, patch1] = destBackend.applyChanges(dest, [change1]) assert.deepStrictEqual(patch1, { clock: {[actor]: 1}, deps: [decodeChange(change1).hash], maxOp: 4, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: {birds: {[`1@${actor}`]: { objectId: `1@${actor}`, type: 'table', props: {[rowId]: {[`2@${actor}`]: { objectId: `2@${actor}`, type: 'map', props: { species: {[`3@${actor}`]: {type: 'value', value: 'Chaffinch'}}, colour: {[`4@${actor}`]: {type: 'value', value: 'brown'}} } }}} }}}} }) }) it('should support Counter objects', () => { const actor = uuid() const [source1, p1, change1] = sourceBackend.applyLocalChange(source, { actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'set', obj: '_root', key: 'counter', value: 1, datatype: 'counter', pred: []} ] }) const [source2, p2, change2] = sourceBackend.applyLocalChange(source1, { actor, seq: 2, startOp: 2, time: 0, deps: [], ops: [ {action: 'inc', obj: '_root', key: 'counter', value: 2, pred: [`1@${actor}`]} ] }) const [dest1, patch1] = destBackend.applyChanges(dest, [change1]) const [dest2, patch2] = destBackend.applyChanges(dest1, [change2]) assert.deepStrictEqual(patch2, { clock: {[actor]: 2}, deps: [decodeChange(change2).hash], maxOp: 2, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: { counter: {[`1@${actor}`]: {type: 'value', value: 3, datatype: 'counter'}} }} }) }) it('should support Date objects', () => { const actor = uuid(), now = new Date() const [source1, p1, change1] = sourceBackend.applyLocalChange(source, { actor, seq: 1, startOp: 1, time: 0, deps: [], ops: [ {action: 'set', obj: '_root', key: 'now', value: now.getTime(), datatype: 'timestamp', pred: []} ] }) const [dest1, patch1] = destBackend.applyChanges(dest, [change1]) assert.deepStrictEqual(patch1, { clock: {[actor]: 1}, deps: [decodeChange(change1).hash], maxOp: 1, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: { now: {[`1@${actor}`]: {type: 'value', value: now.getTime(), datatype: 'timestamp'}} }} }) }) it('should support DEFLATE-compressed changes', () => { let longString = '', actor = uuid() for (let i = 0; i < 1024; i++) longString += 'a' const [source1, p1, change1] = sourceBackend.applyLocalChange(source, { actor, seq: 1, time: 0, startOp: 1, deps: [], ops: [ {action: 'set', obj: '_root', key: 'longString', value: longString, pred: []} ] }) assert.ok(change1.byteLength < 100) const [dest1, patch1] = destBackend.applyChanges(dest, [change1]) assert.deepStrictEqual(patch1, { clock: {[actor]: 1}, deps: [decodeChange(change1).hash], maxOp: 1, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: { longString: {[`1@${actor}`]: {type: 'value', value: longString}} }} }) }) describe('save() and load()', () => { it('should work for a simple document', () => { const actor = uuid() const [source1, p1, change1] = sourceBackend.applyLocalChange(source, { actor, seq: 1, time: 0, startOp: 1, deps: [], ops: [ {action: 'set', obj: '_root', key: 'bird', value: 'magpie', pred: []} ] }) const dest1 = destBackend.load(sourceBackend.save(source1)) const patch = destBackend.getPatch(dest1) assert.deepStrictEqual(patch, { clock: {[actor]: 1}, deps: [decodeChange(change1).hash], maxOp: 1, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: { bird: {[`1@${actor}`]: {type: 'value', value: 'magpie'}} }} }) }) it('should allow DEFLATE-compressed columns', () => { let longString = '', actor = uuid() for (let i = 0; i < 1024; i++) longString += 'a' const [source1, p1, change1] = sourceBackend.applyLocalChange(source, { actor, seq: 1, time: 0, startOp: 1, deps: [], ops: [ {action: 'set', obj: '_root', key: 'longString', value: longString, pred: []} ] }) const compressedDoc = sourceBackend.save(source1) assert.ok(compressedDoc.byteLength < 200) const patch = destBackend.getPatch(destBackend.load(compressedDoc)) assert.deepStrictEqual(patch, { clock: {[actor]: 1}, deps: [decodeChange(change1).hash], maxOp: 1, pendingChanges: 0, diffs: {objectId: '_root', type: 'map', props: { longString: {[`1@${actor}`]: {type: 'value', value: longString}} }} }) }) // TODO need more tests for save() and load() }) } ================================================ FILE: tsconfig.json ================================================ { "compilerOptions": { "allowJs": false, "baseUrl": ".", "esModuleInterop": true, "lib": ["dom", "esnext.asynciterable", "es2017", "es2016", "es2015"], "module": "commonjs", "moduleResolution": "node", "paths": { "automerge": ["*"]}, "rootDir": "", "target": "es2016", "typeRoots": ["./@types", "./node_modules/@types"] }, "exclude": ["dist/**/*"] } ================================================ FILE: webpack.config.js ================================================ const path = require('path') module.exports = { entry: './src/automerge.js', mode: 'development', output: { filename: 'automerge.js', library: 'Automerge', libraryTarget: 'umd', path: path.resolve(__dirname, 'dist'), // https://github.com/webpack/webpack/issues/6525 globalObject: 'this', // https://github.com/webpack/webpack/issues/11660 chunkLoading: false, }, devtool: 'source-map', module: {rules: []}, target: "browserslist:web" }