Showing preview only (888K chars total). Download the full file or copy to clipboard to get everything.
Repository: automerge/automerge-classic
Branch: main
Commit: 0605308926a0
Files: 51
Total size: 859.2 KB
Directory structure:
gitextract_5mussk42/
├── .babelrc
├── .eslintrc.json
├── .github/
│ └── workflows/
│ └── automerge-ci.yml
├── .gitignore
├── .mocharc.yaml
├── @types/
│ └── automerge/
│ └── index.d.ts
├── CHANGELOG.md
├── LICENSE
├── README.md
├── backend/
│ ├── backend.js
│ ├── columnar.js
│ ├── encoding.js
│ ├── index.js
│ ├── new.js
│ ├── sync.js
│ └── util.js
├── frontend/
│ ├── apply_patch.js
│ ├── constants.js
│ ├── context.js
│ ├── counter.js
│ ├── index.js
│ ├── numbers.js
│ ├── observable.js
│ ├── proxies.js
│ ├── table.js
│ └── text.js
├── karma.conf.js
├── karma.sauce.js
├── package.json
├── src/
│ ├── automerge.js
│ ├── common.js
│ └── uuid.js
├── test/
│ ├── backend_test.js
│ ├── columnar_test.js
│ ├── context_test.js
│ ├── encoding_test.js
│ ├── frontend_test.js
│ ├── fuzz_test.js
│ ├── helpers.js
│ ├── new_backend_test.js
│ ├── observable_test.js
│ ├── proxies_test.js
│ ├── sync_test.js
│ ├── table_test.js
│ ├── test.js
│ ├── text_test.js
│ ├── typescript_test.ts
│ ├── uuid_test.js
│ └── wasm.js
├── tsconfig.json
└── webpack.config.js
================================================
FILE CONTENTS
================================================
================================================
FILE: .babelrc
================================================
{
"presets": [
[
"@babel/preset-env"
]
]
}
================================================
FILE: .eslintrc.json
================================================
{
"env": {
"browser": true,
"commonjs": true,
"es2015": true,
"node": true,
"mocha": true
},
"ignorePatterns": "dist/**",
"extends": ["eslint:recommended", "plugin:compat/recommended"],
"parserOptions": {
"ecmaVersion": 2015
},
"rules": {
"accessor-pairs": "error",
"array-bracket-newline": "off",
"array-bracket-spacing": "off",
"array-callback-return": "error",
"array-element-newline": "off",
"arrow-body-style": "error",
"arrow-parens": "off",
"arrow-spacing": [
"error",
{
"after": true,
"before": true
}
],
"block-scoped-var": "error",
"block-spacing": "error",
"brace-style": [
"error",
"1tbs", { "allowSingleLine": true }
],
"camelcase": "off",
"capitalized-comments": "off",
"class-methods-use-this": "off",
"comma-dangle": "off",
"comma-spacing": [
"error",
{
"after": true,
"before": false
}
],
"comma-style": [
"error",
"last"
],
"complexity": "off",
"computed-property-spacing": [
"off",
"never"
],
"consistent-return": "off",
"consistent-this": "error",
"curly": "off",
"default-case": "off",
"default-case-last": "error",
"default-param-last": "error",
"dot-location": [
"error",
"property"
],
"dot-notation": "error",
"eol-last": "error",
"eqeqeq": "off",
"func-call-spacing": "off",
"func-name-matching": "error",
"func-names": "off",
"func-style": [
"error",
"declaration"
],
"function-paren-newline": "off",
"generator-star-spacing": "error",
"grouped-accessor-pairs": "error",
"guard-for-in": "error",
"id-denylist": "error",
"id-length": "off",
"id-match": "error",
"implicit-arrow-linebreak": "off",
"indent": "off",
"init-declarations": "off",
"jsx-quotes": "error",
"key-spacing": "off",
"keyword-spacing": [
"error",
{
"after": true,
"before": true
}
],
"line-comment-position": "off",
"linebreak-style": "off",
"lines-around-comment": "off",
"lines-between-class-members": "error",
"max-classes-per-file": "off",
"max-depth": "off",
"max-len": "off",
"max-lines": "off",
"max-lines-per-function": "off",
"max-nested-callbacks": "error",
"max-params": "off",
"max-statements": "off",
"max-statements-per-line": "off",
"multiline-comment-style": [
"error",
"separate-lines"
],
"new-parens": "error",
"newline-per-chained-call": "off",
"no-alert": "error",
"no-array-constructor": "error",
"no-await-in-loop": "error",
"no-bitwise": "off",
"no-caller": "error",
"no-confusing-arrow": "error",
"no-console": "error",
"no-constant-condition": [
"error",
{
"checkLoops": false
}
],
"no-constructor-return": "error",
"no-continue": "off",
"no-div-regex": "error",
"no-duplicate-imports": "error",
"no-else-return": "off",
"no-empty-function": "off",
"no-eq-null": "error",
"no-eval": "error",
"no-extend-native": "error",
"no-extra-bind": "error",
"no-extra-label": "error",
"no-extra-parens": "off",
"no-floating-decimal": "error",
"no-implicit-coercion": "off",
"no-implicit-globals": "error",
"no-implied-eval": "error",
"no-inline-comments": "off",
"no-invalid-this": "error",
"no-iterator": "error",
"no-label-var": "error",
"no-labels": "error",
"no-lone-blocks": "error",
"no-lonely-if": "off",
"no-loop-func": "off",
"no-loss-of-precision": "error",
"no-magic-numbers": "off",
"no-mixed-operators": "off",
"no-multi-assign": "error",
"no-multi-spaces": "off",
"no-multi-str": "error",
"no-multiple-empty-lines": "error",
"no-negated-condition": "off",
"no-nested-ternary": "off",
"no-new": "error",
"no-new-func": "error",
"no-new-object": "error",
"no-new-wrappers": "error",
"no-nonoctal-decimal-escape": "error",
"no-octal-escape": "error",
"no-param-reassign": "off",
"no-plusplus": "off",
"no-promise-executor-return": "error",
"no-proto": "error",
"no-restricted-exports": "error",
"no-restricted-globals": "error",
"no-restricted-imports": "error",
"no-restricted-properties": "error",
"no-restricted-syntax": "error",
"no-return-assign": "off",
"no-return-await": "error",
"no-script-url": "error",
"no-self-compare": "error",
"no-sequences": "error",
"no-shadow": "off",
"no-tabs": "error",
"no-template-curly-in-string": "error",
"no-ternary": "off",
"no-throw-literal": "error",
"no-trailing-spaces": "error",
"no-undef-init": "off",
"no-undefined": "off",
"no-underscore-dangle": "off",
"no-unmodified-loop-condition": "error",
"no-unneeded-ternary": "error",
"no-unreachable-loop": "error",
"no-unsafe-optional-chaining": "error",
"no-unused-expressions": "error",
"no-unused-vars": ["error", { "args": "after-used" }],
"no-use-before-define": "off",
"no-useless-backreference": "error",
"no-useless-call": "error",
"no-useless-computed-key": "error",
"no-useless-concat": "error",
"no-useless-constructor": "error",
"no-useless-rename": "error",
"no-useless-return": "error",
"no-var": "error",
"no-void": "error",
"no-warning-comments": "off",
"no-whitespace-before-property": "off",
"nonblock-statement-body-position": "error",
"object-curly-newline": "error",
"object-curly-spacing": "off",
"object-property-newline": "off",
"object-shorthand": "error",
"one-var": "off",
"one-var-declaration-per-line": "off",
"operator-assignment": "off",
"operator-linebreak": "error",
"padded-blocks": "off",
"padding-line-between-statements": "error",
"prefer-arrow-callback": "off",
"prefer-const": "off",
"prefer-destructuring": "off",
"prefer-exponentiation-operator": "error",
"prefer-named-capture-group": "off",
"prefer-numeric-literals": "error",
"prefer-object-spread": "off",
"prefer-promise-reject-errors": "error",
"prefer-regex-literals": "error",
"prefer-rest-params": "error",
"prefer-spread": "error",
"prefer-template": "off",
"quote-props": "off",
"quotes": "off",
"radix": "error",
"require-atomic-updates": "error",
"require-await": "error",
"require-unicode-regexp": "off",
"rest-spread-spacing": "error",
"semi": "off",
"semi-spacing": [
"error",
{
"after": true,
"before": false
}
],
"semi-style": [
"error",
"first"
],
"sort-imports": "error",
"sort-keys": "off",
"sort-vars": "off",
"space-before-blocks": "error",
"space-before-function-paren": "off",
"space-in-parens": [
"error",
"never"
],
"space-infix-ops": "error",
"space-unary-ops": "error",
"spaced-comment": [
"error",
"always"
],
"strict": [
"error",
"never"
],
"switch-colon-spacing": "error",
"symbol-description": "error",
"template-curly-spacing": [
"error",
"never"
],
"template-tag-spacing": "error",
"unicode-bom": [
"error",
"never"
],
"vars-on-top": "error",
"wrap-iife": "error",
"wrap-regex": "off",
"yield-star-spacing": "error",
"yoda": [
"error",
"never"
]
}
}
================================================
FILE: .github/workflows/automerge-ci.yml
================================================
name: CI
on: [push, pull_request]
jobs:
node-build:
runs-on: ubuntu-latest
strategy:
matrix:
node-version: [12.x, 14.x, 16.x]
steps:
- name: Check out repo
uses: actions/checkout@v2
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v2
with:
node-version: ${{ matrix.node-version }}
cache: 'yarn'
- name: Install dependencies
run: yarn
- name: ESLint
run: yarn lint
- name: Test suite
run: yarn test
- name: Bundle
run: yarn build
- name: Test suite using bundle
run: TEST_DIST=1 yarn test
- name: Load bundled code
run: node -e "const Automerge = require(\"./dist/automerge\")"
# browsertest:
# runs-on: ubuntu-latest
# # Don't run this job when triggered from a forked repository, since the secrets
# # (Sauce Labs credentials) are not available in that context
# if: ${{ github.repository == 'automerge/automerge' }}
# steps:
# - uses: actions/checkout@v2
# - name: Use Node.js
# uses: actions/setup-node@v2
# with:
# node-version: 16.x
# cache: 'yarn'
# - name: Install dependencies
# run: yarn
# - name: Bundle
# run: yarn build
# - name: Sauce Connect
# uses: saucelabs/sauce-connect-action@v1
# with:
# username: ${{ secrets.SAUCE_USERNAME }}
# accessKey: ${{ secrets.SAUCE_ACCESS_KEY }}
# tunnelIdentifier: github-action-tunnel
# scVersion: 4.7.0
# - name: Run browser tests
# run: node_modules/.bin/karma start karma.sauce.js
# env:
# SAUCE_USERNAME: ${{secrets.SAUCE_USERNAME}}
# SAUCE_ACCESS_KEY: ${{secrets.SAUCE_ACCESS_KEY}}
npm-publish:
name: npm-publish
if: ${{ github.repository == 'automerge/automerge' && github.ref == 'refs/heads/main' }}
# needs: [ node-build, browsertest ]
needs: [ node-build ]
runs-on: ubuntu-latest
steps:
- name: Check out repo
uses: actions/checkout@v2
- name: Use Node.js
uses: actions/setup-node@v2
with:
node-version: 16
- name: Install dependencies
run: yarn install
- name: npm publish if version has been updated
uses: JS-DevTools/npm-publish@v1
with:
token: ${{ secrets.NPM_AUTH_TOKEN }}
check-version: true
================================================
FILE: .gitignore
================================================
/coverage
/dist
/node_modules
.nyc_output
.vscode
================================================
FILE: .mocharc.yaml
================================================
use_strict: true
require:
- ts-node/register
- tsconfig-paths/register
watch-files:
- 'src/*.js'
- 'frontend/*.js'
- 'backend/*.js'
- 'test/*.js'
- 'test/*.ts'
spec:
- 'test/*test*.js'
- 'test/*test*.ts'
================================================
FILE: @types/automerge/index.d.ts
================================================
declare module 'automerge' {
/**
* The return type of `Automerge.init<T>()`, `Automerge.change<T>()`, etc. where `T` is the
* original type. It is a recursively frozen version of the original type.
*/
type Doc<T> = FreezeObject<T>
type ChangeFn<T> = (doc: T) => void
// Automerge.* functions
function init<T>(options?: InitOptions<T>): Doc<T>
function from<T>(initialState: T | Doc<T>, options?: InitOptions<T>): Doc<T>
function clone<T>(doc: Doc<T>, options?: InitOptions<T>): Doc<T>
function free<T>(doc: Doc<T>): void
type InitOptions<T> =
| string // = actorId
| {
actorId?: string
deferActorId?: boolean
freeze?: boolean
patchCallback?: PatchCallback<T>
observable?: Observable
}
type ChangeOptions<T> =
| string // = message
| {
message?: string
time?: number
patchCallback?: PatchCallback<T>
}
type PatchCallback<T> = (patch: Patch, before: T, after: T, local: boolean, changes: BinaryChange[]) => void
type ObserverCallback<T> = (diff: MapDiff | ListDiff | ValueDiff, before: T, after: T, local: boolean, changes: BinaryChange[]) => void
class Observable {
observe<T>(object: T, callback: ObserverCallback<T>): void
}
function merge<T>(localdoc: Doc<T>, remotedoc: Doc<T>): Doc<T>
function change<T>(doc: Doc<T>, options: ChangeOptions<T>, callback: ChangeFn<T>): Doc<T>
function change<T>(doc: Doc<T>, callback: ChangeFn<T>): Doc<T>
function emptyChange<D extends Doc<any>>(doc: D, options?: ChangeOptions<D>): D
function applyChanges<T>(doc: Doc<T>, changes: BinaryChange[]): [Doc<T>, Patch]
function equals<T>(val1: T, val2: T): boolean
function encodeChange(change: Change): BinaryChange
function decodeChange(binaryChange: BinaryChange): Change
function getActorId<T>(doc: Doc<T>): string
function getAllChanges<T>(doc: Doc<T>): BinaryChange[]
function getChanges<T>(olddoc: Doc<T>, newdoc: Doc<T>): BinaryChange[]
function getConflicts<T>(doc: Doc<T>, key: keyof T): any
function getHistory<T>(doc: Doc<T>): State<T>[]
function getLastLocalChange<T>(doc: Doc<T>): BinaryChange
function getObjectById<T>(doc: Doc<T>, objectId: OpId): any
function getObjectId(object: any): OpId
function load<T>(data: BinaryDocument, options?: InitOptions<T>): Doc<T>
function save<T>(doc: Doc<T>): BinaryDocument
function generateSyncMessage<T>(doc: Doc<T>, syncState: SyncState): [SyncState, BinarySyncMessage?]
function receiveSyncMessage<T>(doc: Doc<T>, syncState: SyncState, message: BinarySyncMessage): [Doc<T>, SyncState, Patch?]
function initSyncState(): SyncState
// custom CRDT types
class TableRow {
readonly id: UUID
}
class Table<T> {
constructor()
add(item: T): UUID
byId(id: UUID): T & TableRow
count: number
ids: UUID[]
remove(id: UUID): void
rows: (T & TableRow)[]
}
class List<T> extends Array<T> {
insertAt?(index: number, ...args: T[]): List<T>
deleteAt?(index: number, numDelete?: number): List<T>
}
class Text extends List<string> {
constructor(text?: string | string[])
get(index: number): string
toSpans<T>(): (string | T)[]
}
// Note that until https://github.com/Microsoft/TypeScript/issues/2361 is addressed, we
// can't treat a Counter like a literal number without force-casting it as a number.
// This won't compile:
// `assert.strictEqual(c + 10, 13) // Operator '+' cannot be applied to types 'Counter' and '10'.ts(2365)`
// But this will:
// `assert.strictEqual(c as unknown as number + 10, 13)`
class Counter extends Number {
constructor(value?: number)
increment(delta?: number): void
decrement(delta?: number): void
toString(): string
valueOf(): number
value: number
}
class Int { constructor(value: number) }
class Uint { constructor(value: number) }
class Float64 { constructor(value: number) }
// Readonly variants
type ReadonlyTable<T> = ReadonlyArray<T> & Table<T>
type ReadonlyList<T> = ReadonlyArray<T> & List<T>
type ReadonlyText = ReadonlyList<string> & Text
// Front & back
namespace Frontend {
function applyPatch<T>(doc: Doc<T>, patch: Patch, backendState?: BackendState): Doc<T>
function change<T>(doc: Doc<T>, message: string | undefined, callback: ChangeFn<T>): [Doc<T>, Change]
function change<T>(doc: Doc<T>, callback: ChangeFn<T>): [Doc<T>, Change]
function emptyChange<T>(doc: Doc<T>, message?: string): [Doc<T>, Change]
function from<T>(initialState: T | Doc<T>, options?: InitOptions<T>): [Doc<T>, Change]
function getActorId<T>(doc: Doc<T>): string
function getBackendState<T>(doc: Doc<T>): BackendState
function getConflicts<T>(doc: Doc<T>, key: keyof T): any
function getElementIds(list: any): string[]
function getLastLocalChange<T>(doc: Doc<T>): BinaryChange
function getObjectById<T>(doc: Doc<T>, objectId: OpId): Doc<T>
function getObjectId<T>(doc: Doc<T>): OpId
function init<T>(options?: InitOptions<T>): Doc<T>
function setActorId<T>(doc: Doc<T>, actorId: string): Doc<T>
}
namespace Backend {
function applyChanges(state: BackendState, changes: BinaryChange[]): [BackendState, Patch]
function applyLocalChange(state: BackendState, change: Change): [BackendState, Patch, BinaryChange]
function clone(state: BackendState): BackendState
function free(state: BackendState): void
function getAllChanges(state: BackendState): BinaryChange[]
function getChangeByHash(state: BackendState, hash: Hash): BinaryChange
function getChanges(state: BackendState, haveDeps: Hash[]): BinaryChange[]
function getChangesAdded(state1: BackendState, state2: BackendState): BinaryChange[]
function getHeads(state: BackendState): Hash[]
function getMissingDeps(state: BackendState, heads?: Hash[]): Hash[]
function getPatch(state: BackendState): Patch
function init(): BackendState
function load(data: BinaryDocument): BackendState
function loadChanges(state: BackendState, changes: BinaryChange[]): BackendState
function save(state: BackendState): BinaryDocument
function generateSyncMessage(state: BackendState, syncState: SyncState): [SyncState, BinarySyncMessage?]
function receiveSyncMessage(state: BackendState, syncState: SyncState, message: BinarySyncMessage): [BackendState, SyncState, Patch?]
function encodeSyncMessage(message: SyncMessage): BinarySyncMessage
function decodeSyncMessage(bytes: BinarySyncMessage): SyncMessage
function initSyncState(): SyncState
function encodeSyncState(syncState: SyncState): BinarySyncState
function decodeSyncState(bytes: BinarySyncState): SyncState
}
// Internals
type Hash = string // 64-digit hex string
type OpId = string // of the form `${counter}@${actorId}`
type UUID = string
type UUIDGenerator = () => UUID
interface UUIDFactory extends UUIDGenerator {
setFactory: (generator: UUIDGenerator) => void
reset: () => void
}
const uuid: UUIDFactory
interface Clock {
[actorId: string]: number
}
interface State<T> {
change: Change
snapshot: T
}
interface BackendState {
// no public methods or properties
}
type BinaryChange = Uint8Array & { __binaryChange: true }
type BinaryDocument = Uint8Array & { __binaryDocument: true }
type BinarySyncState = Uint8Array & { __binarySyncState: true }
type BinarySyncMessage = Uint8Array & { __binarySyncMessage: true }
interface SyncState {
// no public methods or properties
}
interface SyncMessage {
heads: Hash[]
need: Hash[]
have: SyncHave[]
changes: BinaryChange[]
}
interface SyncHave {
lastSync: Hash[]
bloom: Uint8Array
}
interface Change {
message: string
actor: string
time: number
seq: number
startOp: number
hash?: Hash
deps: Hash[]
ops: Op[]
}
interface Op {
action: OpAction
obj: OpId
key: string | number
insert: boolean
elemId?: OpId
child?: OpId
value?: number | boolean | string | null
datatype?: DataType
pred?: OpId[]
values?: (number | boolean | string | null)[]
multiOp?: number
}
interface Patch {
actor?: string
seq?: number
pendingChanges: number
clock: Clock
deps: Hash[]
diffs: MapDiff
maxOp: number
}
// Describes changes to a map (in which case propName represents a key in the
// map) or a table object (in which case propName is the primary key of a row).
interface MapDiff {
objectId: OpId // ID of object being updated
type: 'map' | 'table' // type of object being updated
// For each key/property that is changing, props contains one entry
// (properties that are not changing are not listed). The nested object is
// empty if the property is being deleted, contains one opId if it is set to
// a single value, and contains multiple opIds if there is a conflict.
props: {[propName: string]: {[opId: string]: MapDiff | ListDiff | ValueDiff }}
}
// Describes changes to a list or Automerge.Text object, in which each element
// is identified by its index.
interface ListDiff {
objectId: OpId // ID of object being updated
type: 'list' | 'text' // type of objct being updated
// This array contains edits in the order they should be applied.
edits: (SingleInsertEdit | MultiInsertEdit | UpdateEdit | RemoveEdit)[]
}
// Describes the insertion of a single element into a list or text object.
// The element can be a nested object.
interface SingleInsertEdit {
action: 'insert'
index: number // the list index at which to insert the new element
elemId: OpId // the unique element ID of the new list element
opId: OpId // ID of the operation that assigned this value
value: MapDiff | ListDiff | ValueDiff
}
// Describes the insertion of a consecutive sequence of primitive values into
// a list or text object. In the case of text, the values are strings (each
// character as a separate string value). Each inserted value is given a
// consecutive element ID: starting with `elemId` for the first value, the
// subsequent values are given elemIds with the same actor ID and incrementing
// counters. To insert non-primitive values, use SingleInsertEdit.
interface MultiInsertEdit {
action: 'multi-insert'
index: number // the list index at which to insert the first value
elemId: OpId // the unique ID of the first inserted element
values: number[] | boolean[] | string[] | null[] // list of values to insert
datatype?: DataType // all values must be of the same datatype
}
// Describes the update of the value or nested object at a particular index
// of a list or text object. In the case where there are multiple conflicted
// values at the same list index, multiple UpdateEdits with the same index
// (but different opIds) appear in the edits array of ListDiff.
interface UpdateEdit {
action: 'update'
index: number // the list index to update
opId: OpId // ID of the operation that assigned this value
value: MapDiff | ListDiff | ValueDiff
}
// Describes the deletion of one or more consecutive elements from a list or
// text object.
interface RemoveEdit {
action: 'remove'
index: number // index of the first list element to remove
count: number // number of list elements to remove
}
// Describes a primitive value, optionally tagged with a datatype that
// indicates how the value should be interpreted.
interface ValueDiff {
type: 'value'
value: number | boolean | string | null
datatype?: DataType
}
type OpAction =
| 'del'
| 'inc'
| 'set'
| 'link'
| 'makeText'
| 'makeTable'
| 'makeList'
| 'makeMap'
type CollectionType =
| 'list' //..
| 'map'
| 'table'
| 'text'
type DataType =
| 'int'
| 'uint'
| 'float64'
| 'counter'
| 'timestamp'
// TYPE UTILITY FUNCTIONS
// Type utility function: Freeze
// Generates a readonly version of a given object, array, or map type applied recursively to the nested members of the root type.
// It's like TypeScript's `readonly`, but goes all the way down a tree.
// prettier-ignore
type Freeze<T> =
T extends Function ? T
: T extends Text ? ReadonlyText
: T extends Table<infer T> ? FreezeTable<T>
: T extends List<infer T> ? FreezeList<T>
: T extends Array<infer T> ? FreezeArray<T>
: T extends Map<infer K, infer V> ? FreezeMap<K, V>
: T extends string & infer O ? string & O
: FreezeObject<T>
interface FreezeTable<T> extends ReadonlyTable<Freeze<T>> {}
interface FreezeList<T> extends ReadonlyList<Freeze<T>> {}
interface FreezeArray<T> extends ReadonlyArray<Freeze<T>> {}
interface FreezeMap<K, V> extends ReadonlyMap<Freeze<K>, Freeze<V>> {}
type FreezeObject<T> = { readonly [P in keyof T]: Freeze<T[P]> }
}
================================================
FILE: CHANGELOG.md
================================================
# Changelog
Automerge adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html) for assigning
version numbers. However, any feature that is not documented or labelled as "experimental" may
change without warning in a minor release.
All notable changes to Automerge will be documented in this file, which
is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
## Performance branch (working towards Automerge 1.0 release candidate)
- **Changed**: The data format for storing/transmitting changes (as returned by
`Automerge.getChanges()`) and documents (as returned by `Automerge.save()`) have changed.
They now use a binary encoding that is much more compact than the old JSON format. This is a
breaking change, but we will provide an upgrade tool to migrate any existing Automerge docs
to the new format.
- **Changed**: `Automerge.applyChanges()` now returns a two-element array, where the first
element is the updated document, and the second element is a patch describing the changes that
have been made (including any conflicts that have arisen). This simplifies applications that
need to update some other state, such as a user interface, to reflect changes in a document.
- **Changed** [#339]: `Automerge.Connection`, `Automerge.DocSet`, and `Automerge.WatchableDoc`
have been removed, and replaced with a new Automerge sync protocol that is implemented by the
functions `Automerge.generateSyncMessage()` and `Automerge.receiveSyncMessage()`.
([@ept], [@pvh], [@orionz], [@alexjg], [@jeffa5])
- **Changed**: The frontend/backend protocol (change requests generated by `Frontend.change()`
and patches returned by `Backend.applyChanges()`) has changed. The new format will be
documented separately.
- **Removed**: The undo/redo feature is removed for now. The original implementation was a bit of
a hack, and we decided not to support the hack for 1.0. Instead, we will bring back a
better-designed version of this feature in the future.
- **Changed**: Actor IDs are now required to consist only of lowercase hexadecimal digits
(UUIDs can still be used, but the hyphens need to be removed).
- **Changed**: `Automerge.getConflicts()` now returns *all* conflicting values, including the
value chosen as default resolution.
- **Changed**: Multiple references to the same object in an Automerge document are no longer
allowed. In other words, the document is now required to be a tree, not a DAG.
- **Changed**: We no longer assume that the backend state is immutable, giving us greater freedom
to implement the backend in a way that maximises performance. (Frontend state and Automerge
documents remain immutable as before.) This restricts certain usage patterns: for example, if
you update a document, you cannot then take a reference to the document state before that
update and update or query it. If you want to be able to continue referencing an old document
state, you can copy it using `Automerge.clone()`.
- **Removed**: `Automerge.diff` and `Backend.merge`, because they depended on the assumption of
an immutable backend. We hope to bring back a better implementation of `Automerge.diff` in the
future.
- **Changed**: Dependencies between changes are now expressed by referencing the hashes of
dependencies, rather than their actorId and sequence number. APIs have changed accordingly:
`Backend.getMissingDeps` now returns a list of hashes rather than a vector clock; the second
argument of `Backend.getChanges()` is now also a list of hashes. This change also affects
network sync protocols, e.g. based on `Automerge.Connection`, which need to now exchange
hashes rather than vector clocks.
- **Removed**: `Automerge.getMissingDeps()`; use `Backend.getMissingDeps()` instead.
- **Removed**: `Backend.getMissingChanges()`; use `Backend.getChanges()` instead.
- **Removed**: `Backend.getChangesForActor()` since it does not fit with a hash chaining approach.
- **Added**: `Frontend.getLastLocalChange()` returns the binary encoding of the last change made
by the local actor, and `Backend.getHeads()` returns the latest change hashes in the current
document state.
- **Changed**: `Backend.applyLocalChange()` now returns an array of three values: the updated
backend state, the patch to apply to the frontend, and the binary encoding of the change.
- **Added** [#308]: Experimental `Automerge.Observable` API allows an application to receive
a callback whenever a document (or some object within a document) changes ([@ept])
## [Unreleased]
## [0.14.2] — 2021-01-12
- **Fixed** [#301]: Handling out-of-bounds argument in `Array.slice()` ([@pierreprinetti])
- **Fixed** [#261]: Support calling `Array.indexOf()` with an object ([@philschatz])
## [0.14.1] — 2020-05-25
- **Fixed** [#249]: Corrected TypeScript declaration for `Automerge.Table.rows` ([@lauritzsh])
- **Fixed** [#252]: Corrected TypeScript declaration for `WatchableDoc` ([@vincentcapicotto])
- **Fixed** [#258]: Changes whose dependencies are missing are now preserved when saving and
reloading a document ([@KarenSarmiento], [@ept])
- **Changed** [#260]: If you try to assign an object that is already in an Automerge document,
you now get a more descriptive error message ([@johannesjo], [@ept])
## [0.14.0] — 2020-03-25
- **Removed** [#236]: Undocumented `Automerge.Table` API that allowed rows to be added by
providing an array of values. Now rows must be given as an object ([@HerbCaudill])
- **Removed** [#241]: Constructor of `Automerge.Table` no longer takes an array of columns, and
the `columns` property of `Automerge.Table` is also removed ([@ept])
- **Changed** [#242]: Rows of `Automerge.Table` now automatically get an `id` property containing
the primary key of that row ([@ept])
- **Removed** [#243]: `Automerge.Table` objects no longer have a `set()` method. Use `add()` or
`remove()` instead ([@ept])
- **Removed** support for Node 8, which is no longer being maintained
- **Added** [#194], [#238]: `Automerge.Text` objects may now contain objects as well as strings;
new method `Text.toSpans()` that concatenates characters while leaving objects unchanged
([@pvh], [@ept], [@nornagon])
## [0.13.0] — 2020-02-24
- **Added** [#232]: New API `Automerge.getAllChanges()` returns all changes ([@ept])
- **Fixed** [#230]: `Text.deleteAt` allows zero characters to be deleted ([@skokenes])
- **Fixed** [#219]: `canUndo` is false immediately after `Automerge.from` ([@ept])
- **Fixed** [#215]: Adjust TypeScript definition of `Freeze<T>` ([@jeffpeterson])
## [0.12.1] — 2019-08-22
- **Fixed** [#184]: Corrected TypeScript type definition for `Automerge.DocSet` ([@HerbCaudill])
- **Fixed** [#174]: If `.filter()`, `.find()` or similar methods are used inside a change callback,
the objects they return can now be mutated ([@ept], [@airhorns])
- **Fixed** [#199]: `Automerge.Text.toString()` now returns the unadulterated text ([@Gozala])
- **Added** [#210]: New method `DocSet.removeDoc()` ([@brentkeller])
## [0.12.0] — 2019-08-07
- **Changed** [#183]: `Frontend.from()` now accepts initialization options ([@HerbCaudill], [@ept])
- **Changed** [#180]: Mutation methods on `Automerge.Text` are now available without having to
assign the object to a document ([@ept])
- **Added** [#181]: Can now specify an initial value when creating `Automerge.Text` objects
([@Gozala], [@ept])
- **Fixed** [#202]: Stack overflow error when making large changes ([@HerbCaudill], [@ept])
## [0.11.0] — 2019-07-13
- **Added** [#127]: New `Automerge.from` function creates a new document and initializes it
with an initial state given as an argument ([@HerbCaudill], [@ept])
- **Added** [#155]: Type definitions now allow TypeScript applications to use Automerge with
static type-checking ([@HerbCaudill], [@airhorns], [@aslakhellesoy], [@ept])
- **Changed** [#177]: Automerge documents are no longer made immutable with `Object.freeze`
by default, due to the performance cost. Use the `{freeze: true}` option to continue
using immutable objects. ([@izuchukwu], [@ept])
- **Fixed** [#165]: Undo/redo now work when using separate frontend and backend ([@ept])
## [0.10.1] — 2019-05-17
- **Fixed** [#151]: Exception "Duplicate list element ID" after a list element was added and
removed again in the same change callback ([@ept], [@minhhien1996])
- **Changed** [#163]: Calling `JSON.stringify` on an Automerge document containing
`Automerge.Text`, `Automerge.Table` or `Automerge.Counter` now serializes those objects in a
clean way, rather than dumping the object's internal properties ([@ept])
## [0.10.0] — 2019-02-04
- **Added** [#29]: New `Automerge.Table` datatype provides an unordered collection of records,
like a relational database ([@ept])
- **Added** [#139]: JavaScript Date objects are now supported in Automerge documents ([@ept])
- **Added** [#147]: New `Automerge.Counter` datatype provides a CRDT counter ([@ept])
- **Removed** [#148]: `Automerge.inspect` has been removed ([@ept])
- **Fixed** [#145]: Exception "Duplicate list element ID" after reloading document from disk
([@ept])
- **Changed** [#150]: Underscore-prefixed property names are now allowed in map objects;
`doc.object._objectId` is now `Automerge.getObjectId(doc.object)`,
`doc.object._conflicts.property` is now `Automerge.getConflicts(doc.object, 'property')`,
and `doc._actorId` is now `Automerge.getActorId(doc)`. ([@ept])
## [0.9.2] — 2018-11-05
- **Fixed** [#128]: Fixed crash when Text object was modified in the same change as
another object ([@CGNonofr])
- **Fixed** [#129]: Prevent application of duplicate requests in `applyLocalChange()` ([@ept])
- **Changed** [#130]: Frontend API no longer uses `Frontend.getRequests()`; instead, frontend
change functions now return request objects directly ([@ept])
## [0.9.1] — 2018-09-27
- **Changed** [#126]: Backend no longer needs to know the actorId of the local node ([@ept])
- **Changed** [#126]: Frontend can now be initialized without actorId, as long as you call
`setActorId` before you make the first change ([@ept])
- **Changed** [#120]: Undo and redo must now be initiated by the frontend, not the backend ([@ept])
- **Fixed** [#120]: Fixed bug that would cause sequence numbers to be reused in some concurrent
executions ([@ept])
- **Fixed** [#125]: Exceptions now throw Error objects rather than plain strings ([@wincent])
## [0.9.0] — 2018-09-18
- **Added** [#112]: Added `Automerge.undo()` and `Automerge.redo()` ([@ept])
- **Added** [#118]: Introduced new Frontend and Backend APIs, and refactored existing APIs to use
them; this allows some of the work to be moved to a background thread, and provides better
modularisation ([@ept])
- **Removed** Removed the experimental Immutable.js-compatible API (`Automerge.initImmutable()`),
a casualty of the refactoring in [#118] ([@ept])
## [0.8.0] — 2018-08-02
- **Added** [#106]: New `doc._get(UUID)` method allows looking up an object by its `_objectId`
inside an `Automerge.change()` callback ([@mattkrick])
- **Added** [#109]: Export `OpSet.getMissingChanges` on the Automerge object ([@mattkrick])
- **Added** [#111]: New `Automerge.emptyChange()` allows a "change" record to be created without
actually changing the document ([@ept])
- **Changed** [#110]: Require that the change message in `Automerge.change()` must be a string
([@ept])
- **Changed** [#111]: If `Automerge.change()` does not modify the document, the function now
returns the original document object untouched ([@ept])
## [0.7.11] — 2018-06-26
- **Fixed** [#97]: `delete` operator no longer throws an exception if the property doesn't exist
([@salzhrani], [@EthanRBrown])
- **Fixed** [#104]: Fix an error when loading the webpack-packaged version of Automerge in Node.js
([@ept])
## [0.7.10] — 2018-06-12
- **Added** [#93]: Allow the UUID implementation to be replaced for testing purposes ([@kpruden])
- **Added** [#74]: Automerge.diff() now includes the path from the root to the modified object
([@ept])
## [0.7.9] — 2018-05-25
- **Fixed** [#90]: Compatibility with Node 10 ([@aslakhellesoy])
## [0.7.8] — 2018-05-15
- **Fixed** [#91]: Improve performance of changes that modify many list or map elements ([@ept])
## [0.7.7] — 2018-04-24
- **Changed** [#87]: Remove babel-polyfill from transpiled library ([@EthanRBrown])
## 0.7.4, 0.7.5, [0.7.6] — 2018-04-19
- Version bump to fix a build tooling issue
## [0.7.3] — 2018-04-19
- **Changed** [#85]: Publish Babel-transpiled code to npm to improve compatibility ([@EthanRBrown])
## [0.7.2] — 2018-04-17
- **Changed** [#83]: Changed `_objectId` property on Automerge map objects to be non-enumerable
([@EthanRBrown], [@ept])
- **Changed** [#84]: Changed `_conflicts`, `_state`, and `_actorId` to be non-enumerable
properties ([@ept])
- **Fixed** [#77]: Fixed exception when a list element is inserted and updated in the same change
callback ([@mmcgrana], [@ept])
- **Fixed** [#78]: Better error message when trying to use an unsupported datatype ([@ept])
## [0.7.1] — 2018-02-26
- **Fixed** [#69]: `Automerge.load` generates random actorId if none specified ([@saranrapjs])
- **Fixed** [#64]: `Automerge.applyChanges()` allows changes to be applied out-of-order
([@jimpick], [@ept])
## [0.7.0] — 2018-01-15
- **Added** [#62]: Initial support for Immutable.js API compatibility (read-only for now)
([@ept], [@jeffpeterson])
- **Added** [#45]: Added experimental APIs `Automerge.getMissingDeps`,
`Automerge.getChangesForActor`, and `Automerge.WatchableDoc` to support integration with dat
hypercore ([@pvh], [@ept])
- **Added** [#46]: Automerge list objects now also have a `_conflicts` property that records
concurrent assignments to the same list index, just like map objects have had all along ([@ept])
- **Changed** [#60]: `splice` in an `Automerge.change()` callback returns an array of deleted
elements (to match behaviour of
[`Array#splice`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/splice)).
([@aslakhellesoy])
- **Fixed** [#57]: Tests now work on operating systems with case-sensitive filesystems ([@mmmm1998])
## [0.6.0] — 2017-12-13
- **Added** [#44]: New APIs `Automerge.getChanges` and `Automerge.applyChanges` to provide more
flexibility for network protocol layer ([@ept])
- **Added** [#41]: New `Automerge.Text` datatype, which is more efficient than a list for
character-by-character editing of text ([@ept])
- **Added** [#40]: Lists are now backed by a new indexed skip list data structure, which is
faster ([@ept])
- **Changed** [#38]: To save memory, `Automerge.getHistory` now builds snapshots of past states
only when requested, rather than remembering them by default ([@ept])
## [0.5.0] — 2017-09-19
- **Added** [#37]: Added `Automerge.diff` to find the differences between to Automerge documents
([@ept])
- **Added** [#37]: Added support for incremental cache maintenance, bringing a 20x speedup for a
1,000-element list ([@ept])
- **Added** [#36]: Added `Automerge.Connection` and `Automerge.DocSet` classes to support
peer-to-peer network protocols ([@ept], [@pvh])
- **Changed**: Renamed `Automerge.changeset` to `Automerge.change` ([@ept])
## [0.4.3] — 2017-08-16
- **Fixed** [#34]: Fixed a bug that caused list elements to sometimes disappear
([@aslakhellesoy], [@ept])
- **Fixed** [#32]: Fixed a test failure in recent Node.js versions ([@aslakhellesoy])
## [0.4.2] — 2017-06-29
- **Added**: Set up Karma to run tests in web browsers ([@ept])
- **Added**: Set up Webpack to produce bundled JavaScript file for web browsers ([@ept])
## [0.4.1] — 2017-06-26
- **Changed**: `Automerge.getHistory` API now uses the object cache, which should be faster ([@ept])
## [0.4.0] — 2017-06-23
- **Changed**: Automerge documents are now just regular JavaScript objects, and Proxy is used only
within `Automerge.changeset` callbacks. Previously everything used Proxy. ([@ept])
- **Changed**: [#30]: Made `_objectId` an enumerable property, so that it is visible by default
([@ept])
- **Changed**: Support all standard JavaScript array methods and iterators on list proxy object
([@ept])
## [0.3.0] — 2017-06-13
- First public release.
[Unreleased]: https://github.com/automerge/automerge/compare/v0.14.2...HEAD
[0.14.2]: https://github.com/automerge/automerge/compare/v0.14.1...v0.14.2
[0.14.1]: https://github.com/automerge/automerge/compare/v0.14.0...v0.14.1
[0.14.0]: https://github.com/automerge/automerge/compare/v0.13.1...v0.14.0
[0.13.0]: https://github.com/automerge/automerge/compare/v0.12.1...v0.13.0
[0.12.1]: https://github.com/automerge/automerge/compare/v0.12.0...v0.12.1
[0.12.0]: https://github.com/automerge/automerge/compare/v0.11.0...v0.12.0
[0.11.0]: https://github.com/automerge/automerge/compare/v0.10.1...v0.11.0
[0.10.1]: https://github.com/automerge/automerge/compare/v0.10.0...v0.10.1
[0.10.0]: https://github.com/automerge/automerge/compare/v0.9.2...v0.10.0
[0.9.2]: https://github.com/automerge/automerge/compare/v0.9.1...v0.9.2
[0.9.1]: https://github.com/automerge/automerge/compare/v0.9.0...v0.9.1
[0.9.0]: https://github.com/automerge/automerge/compare/v0.8.0...v0.9.0
[0.8.0]: https://github.com/automerge/automerge/compare/v0.7.11...v0.8.0
[0.7.11]: https://github.com/automerge/automerge/compare/v0.7.10...v0.7.11
[0.7.10]: https://github.com/automerge/automerge/compare/v0.7.9...v0.7.10
[0.7.9]: https://github.com/automerge/automerge/compare/v0.7.8...v0.7.9
[0.7.8]: https://github.com/automerge/automerge/compare/v0.7.7...v0.7.8
[0.7.7]: https://github.com/automerge/automerge/compare/v0.7.6...v0.7.7
[0.7.6]: https://github.com/automerge/automerge/compare/v0.7.3...v0.7.6
[0.7.3]: https://github.com/automerge/automerge/compare/v0.7.2...v0.7.3
[0.7.2]: https://github.com/automerge/automerge/compare/v0.7.1...v0.7.2
[0.7.1]: https://github.com/automerge/automerge/compare/v0.7.0...v0.7.1
[0.7.0]: https://github.com/automerge/automerge/compare/v0.6.0...v0.7.0
[0.6.0]: https://github.com/automerge/automerge/compare/v0.5.0...v0.6.0
[0.5.0]: https://github.com/automerge/automerge/compare/v0.4.3...v0.5.0
[0.4.3]: https://github.com/automerge/automerge/compare/v0.4.2...v0.4.3
[0.4.2]: https://github.com/automerge/automerge/compare/v0.4.1...v0.4.2
[0.4.1]: https://github.com/automerge/automerge/compare/v0.4.0...v0.4.2
[0.4.0]: https://github.com/automerge/automerge/compare/v0.3.0...v0.4.0
[0.3.0]: https://github.com/automerge/automerge/compare/v0.2.0...v0.3.0
[#339]: https://github.com/automerge/automerge/pull/339
[#308]: https://github.com/automerge/automerge/pull/308
[#301]: https://github.com/automerge/automerge/pull/301
[#261]: https://github.com/automerge/automerge/pull/261
[#260]: https://github.com/automerge/automerge/issues/260
[#258]: https://github.com/automerge/automerge/issues/258
[#252]: https://github.com/automerge/automerge/pull/252
[#249]: https://github.com/automerge/automerge/pull/249
[#243]: https://github.com/automerge/automerge/pull/243
[#242]: https://github.com/automerge/automerge/pull/242
[#241]: https://github.com/automerge/automerge/pull/241
[#238]: https://github.com/automerge/automerge/pull/238
[#236]: https://github.com/automerge/automerge/pull/236
[#232]: https://github.com/automerge/automerge/pull/232
[#230]: https://github.com/automerge/automerge/issues/230
[#219]: https://github.com/automerge/automerge/issues/219
[#210]: https://github.com/automerge/automerge/pull/210
[#202]: https://github.com/automerge/automerge/issues/202
[#199]: https://github.com/automerge/automerge/pull/199
[#194]: https://github.com/automerge/automerge/issues/194
[#184]: https://github.com/automerge/automerge/pull/184
[#183]: https://github.com/automerge/automerge/pull/183
[#181]: https://github.com/automerge/automerge/pull/181
[#180]: https://github.com/automerge/automerge/issues/180
[#177]: https://github.com/automerge/automerge/issues/177
[#174]: https://github.com/automerge/automerge/issues/174
[#165]: https://github.com/automerge/automerge/pull/165
[#163]: https://github.com/automerge/automerge/pull/163
[#155]: https://github.com/automerge/automerge/pull/155
[#151]: https://github.com/automerge/automerge/issues/151
[#150]: https://github.com/automerge/automerge/pull/150
[#148]: https://github.com/automerge/automerge/pull/148
[#147]: https://github.com/automerge/automerge/pull/147
[#145]: https://github.com/automerge/automerge/issues/145
[#139]: https://github.com/automerge/automerge/pull/139
[#130]: https://github.com/automerge/automerge/pull/130
[#129]: https://github.com/automerge/automerge/pull/129
[#128]: https://github.com/automerge/automerge/pull/128
[#127]: https://github.com/automerge/automerge/issues/127
[#126]: https://github.com/automerge/automerge/pull/126
[#125]: https://github.com/automerge/automerge/pull/125
[#120]: https://github.com/automerge/automerge/pull/120
[#118]: https://github.com/automerge/automerge/pull/118
[#112]: https://github.com/automerge/automerge/pull/112
[#111]: https://github.com/automerge/automerge/pull/111
[#110]: https://github.com/automerge/automerge/pull/110
[#109]: https://github.com/automerge/automerge/pull/109
[#106]: https://github.com/automerge/automerge/issues/106
[#104]: https://github.com/automerge/automerge/issues/104
[#97]: https://github.com/automerge/automerge/issues/97
[#93]: https://github.com/automerge/automerge/pull/93
[#91]: https://github.com/automerge/automerge/pull/91
[#90]: https://github.com/automerge/automerge/pull/90
[#87]: https://github.com/automerge/automerge/pull/87
[#85]: https://github.com/automerge/automerge/pull/85
[#84]: https://github.com/automerge/automerge/pull/84
[#83]: https://github.com/automerge/automerge/pull/83
[#78]: https://github.com/automerge/automerge/issues/78
[#77]: https://github.com/automerge/automerge/pull/77
[#74]: https://github.com/automerge/automerge/pull/74
[#69]: https://github.com/automerge/automerge/pull/69
[#64]: https://github.com/automerge/automerge/pull/64
[#62]: https://github.com/automerge/automerge/pull/62
[#60]: https://github.com/automerge/automerge/pull/60
[#57]: https://github.com/automerge/automerge/pull/57
[#46]: https://github.com/automerge/automerge/issues/46
[#45]: https://github.com/automerge/automerge/pull/45
[#44]: https://github.com/automerge/automerge/pull/44
[#41]: https://github.com/automerge/automerge/pull/41
[#40]: https://github.com/automerge/automerge/pull/40
[#38]: https://github.com/automerge/automerge/issues/38
[#37]: https://github.com/automerge/automerge/pull/37
[#36]: https://github.com/automerge/automerge/pull/36
[#34]: https://github.com/automerge/automerge/pull/34
[#32]: https://github.com/automerge/automerge/pull/32
[#30]: https://github.com/automerge/automerge/pull/30
[#29]: https://github.com/automerge/automerge/issues/29
[@airhorns]: https://github.com/airhorns
[@alexjg]: https://github.com/alexjg
[@aslakhellesoy]: https://github.com/aslakhellesoy
[@brentkeller]: https://github.com/brentkeller
[@CGNonofr]: https://github.com/CGNonofr
[@EthanRBrown]: https://github.com/EthanRBrown
[@Gozala]: https://github.com/Gozala
[@HerbCaudill]: https://github.com/HerbCaudill
[@izuchukwu]: https://github.com/izuchukwu
[@jeffa5]: https://github.com/jeffa5
[@jeffpeterson]: https://github.com/jeffpeterson
[@jimpick]: https://github.com/jimpick
[@johannesjo]: https://github.com/johannesjo
[@ept]: https://github.com/ept
[@KarenSarmiento]: https://github.com/KarenSarmiento
[@kpruden]: https://github.com/kpruden
[@lauritzsh]: https://github.com/lauritzsh
[@mattkrick]: https://github.com/mattkrick
[@minhhien1996]: https://github.com/minhhien1996
[@mmcgrana]: https://github.com/mmcgrana
[@mmmm1998]: https://github.com/mmmm1998
[@nornagon]: https://github.com/nornagon
[@orionz]: https://github.com/orionz
[@pierreprinetti]: https://github.com/pierreprinetti
[@philschatz]: https://github.com/philschatz
[@pvh]: https://github.com/pvh
[@salzhrani]: https://github.com/salzhrani
[@saranrapjs]: https://github.com/saranrapjs
[@skokenes]: https://github.com/skokenes
[@vincentcapicotto]: https://github.com/vincentcapicotto
[@wincent]: https://github.com/wincent
================================================
FILE: LICENSE
================================================
Copyright (c) 2017-present Martin Kleppmann, Ink & Switch LLC, and the Automerge contributors
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
================================================
FILE: README.md
================================================
<img src='./img/sign.svg' width='500' alt='Automerge logo' />
## Deprecation Notice
Automerge now has a shiny new implementation at https://github.com/automerge/automerge. This repository is the original pure javascript implementation. All development effort has shifted to the new implementation which is written in Rust and so can easily be ported to other platforms.
## Original Readme
💬 [Join the Automerge Slack community](https://join.slack.com/t/automerge/shared_invite/zt-e4p3760n-kKh7r3KRH1YwwNfiZM8ktw)
[](https://github.com/automerge/automerge/actions/workflows/automerge-ci.yml)
[](https://app.saucelabs.com/open_sauce/user/automerge/builds)
Automerge is a library of data structures for building collaborative applications in JavaScript.
Please see [automerge.org](http://automerge.org/) for documentation.
For a set of extensible examples in TypeScript, see [automerge-repo](https://github.com/automerge/automerge-repo)
## Setup
If you're using npm, `npm install automerge`. If you're using yarn, `yarn add automerge`. Then you
can import it with `require('automerge')` as in [the example below](#usage) (or
`import * as Automerge from 'automerge'` if using ES2015 or TypeScript).
Otherwise, clone this repository, and then you can use the following commands:
- `yarn install` — installs dependencies.
- `yarn test` — runs the test suite in Node.
- `yarn run browsertest` — runs the test suite in web browsers.
- `yarn build` — creates a bundled JS file `dist/automerge.js` for web browsers. It includes the
dependencies and is set up so that you can load through a script tag.
## Meta
Copyright 2017–2021, the Automerge contributors. Released under the terms of the
MIT license (see `LICENSE`).
================================================
FILE: backend/backend.js
================================================
const { encodeChange } = require('./columnar')
const { BackendDoc } = require('./new')
const { backendState } = require('./util')
/**
* Returns an empty node state.
*/
function init() {
return {state: new BackendDoc(), heads: []}
}
function clone(backend) {
return {state: backendState(backend).clone(), heads: backend.heads}
}
function free(backend) {
backend.state = null
backend.frozen = true
}
/**
* Applies a list of `changes` from remote nodes to the node state `backend`.
* Returns a two-element array `[state, patch]` where `state` is the updated
* node state, and `patch` describes the modifications that need to be made
* to the document objects to reflect these changes.
*/
function applyChanges(backend, changes) {
const state = backendState(backend)
const patch = state.applyChanges(changes)
backend.frozen = true
return [{state, heads: state.heads}, patch]
}
function hashByActor(state, actorId, index) {
if (state.hashesByActor[actorId] && state.hashesByActor[actorId][index]) {
return state.hashesByActor[actorId][index]
}
if (!state.haveHashGraph) {
state.computeHashGraph()
if (state.hashesByActor[actorId] && state.hashesByActor[actorId][index]) {
return state.hashesByActor[actorId][index]
}
}
throw new RangeError(`Unknown change: actorId = ${actorId}, seq = ${index + 1}`)
}
/**
* Takes a single change request `request` made by the local user, and applies
* it to the node state `backend`. Returns a three-element array `[backend, patch, binaryChange]`
* where `backend` is the updated node state,`patch` confirms the
* modifications to the document objects, and `binaryChange` is a binary-encoded form of
* the change submitted.
*/
function applyLocalChange(backend, change) {
const state = backendState(backend)
if (change.seq <= state.clock[change.actor] || 0) {
throw new RangeError('Change request has already been applied')
}
// Add the local actor's last change hash to deps. We do this because when frontend
// and backend are on separate threads, the frontend may fire off several local
// changes in sequence before getting a response from the backend; since the binary
// encoding and hashing is done by the backend, the frontend does not know the hash
// of its own last change in this case. Rather than handle this situation as a
// special case, we say that the frontend includes only specifies other actors'
// deps in changes it generates, and the dependency from the local actor's last
// change is always added here in the backend.
//
// Strictly speaking, we should check whether the local actor's last change is
// indirectly reachable through a different actor's change; in that case, it is not
// necessary to add this dependency. However, it doesn't do any harm either (only
// using a few extra bytes of storage).
if (change.seq > 1) {
const lastHash = hashByActor(state, change.actor, change.seq - 2)
if (!lastHash) {
throw new RangeError(`Cannot find hash of localChange before seq=${change.seq}`)
}
let deps = {[lastHash]: true}
for (let hash of change.deps) deps[hash] = true
change.deps = Object.keys(deps).sort()
}
const binaryChange = encodeChange(change)
const patch = state.applyChanges([binaryChange], true)
backend.frozen = true
// On the patch we send out, omit the last local change hash
const lastHash = hashByActor(state, change.actor, change.seq - 1)
patch.deps = patch.deps.filter(head => head !== lastHash)
return [{state, heads: state.heads}, patch, binaryChange]
}
/**
* Returns the state of the document serialised to an Uint8Array.
*/
function save(backend) {
return backendState(backend).save()
}
/**
* Loads the document and/or changes contained in an Uint8Array, and returns a
* backend initialised with this state.
*/
function load(data) {
const state = new BackendDoc(data)
return {state, heads: state.heads}
}
/**
* Applies a list of `changes` to the node state `backend`, and returns the updated
* state with those changes incorporated. Unlike `applyChanges()`, this function
* does not produce a patch describing the incremental modifications, making it
* a little faster when loading a document from disk. When all the changes have
* been loaded, you can use `getPatch()` to construct the latest document state.
*/
function loadChanges(backend, changes) {
const state = backendState(backend)
state.applyChanges(changes)
backend.frozen = true
return {state, heads: state.heads}
}
/**
* Returns a patch that, when applied to an empty document, constructs the
* document tree in the state described by the node state `backend`.
*/
function getPatch(backend) {
return backendState(backend).getPatch()
}
/**
* Returns an array of hashes of the current "head" changes (i.e. those changes
* that no other change depends on).
*/
function getHeads(backend) {
return backend.heads
}
/**
* Returns the full history of changes that have been applied to a document.
*/
function getAllChanges(backend) {
return getChanges(backend, [])
}
/**
* Returns all changes that are newer than or concurrent to the changes
* identified by the hashes in `haveDeps`. If `haveDeps` is an empty array, all
* changes are returned. Throws an exception if any of the given hashes is unknown.
*/
function getChanges(backend, haveDeps) {
if (!Array.isArray(haveDeps)) {
throw new TypeError('Pass an array of hashes to Backend.getChanges()')
}
return backendState(backend).getChanges(haveDeps)
}
/**
* Returns all changes that are present in `backend2` but not in `backend1`.
* Intended for use in situations where the two backends are for different actors.
* To get the changes added between an older and a newer document state of the same
* actor, use `getChanges()` instead. `getChangesAdded()` throws an exception if
* one of the backend states is frozen (i.e. if it is not the latest state of that
* backend instance; this distinction matters when the backend is mutable).
*/
function getChangesAdded(backend1, backend2) {
return backendState(backend2).getChangesAdded(backendState(backend1))
}
/**
* If the backend has applied a change with the given `hash` (given as a
* hexadecimal string), returns that change (as a byte array). Returns undefined
* if no change with that hash has been applied. A change with missing
* dependencies does not count as having been applied.
*/
function getChangeByHash(backend, hash) {
return backendState(backend).getChangeByHash(hash)
}
/**
* Returns the hashes of any missing dependencies, i.e. where we have applied a
* change that has a dependency on a change we have not seen.
*
* If the argument `heads` is given (an array of hexadecimal strings representing
* hashes as returned by `getHeads()`), this function also ensures that all of
* those hashes resolve to either a change that has been applied to the document,
* or that has been enqueued for later application once missing dependencies have
* arrived. Any missing heads hashes are included in the returned array.
*/
function getMissingDeps(backend, heads = []) {
return backendState(backend).getMissingDeps(heads)
}
module.exports = {
init, clone, free, applyChanges, applyLocalChange, save, load, loadChanges, getPatch,
getHeads, getAllChanges, getChanges, getChangesAdded, getChangeByHash, getMissingDeps
}
================================================
FILE: backend/columnar.js
================================================
const pako = require('pako')
const { copyObject, parseOpId, equalBytes } = require('../src/common')
const {
utf8ToString, hexStringToBytes, bytesToHexString,
Encoder, Decoder, RLEEncoder, RLEDecoder, DeltaEncoder, DeltaDecoder, BooleanEncoder, BooleanDecoder
} = require('./encoding')
// Maybe we should be using the platform's built-in hash implementation?
// Node has the crypto module: https://nodejs.org/api/crypto.html and browsers have
// https://developer.mozilla.org/en-US/docs/Web/API/SubtleCrypto/digest
// However, the WebCrypto API is asynchronous (returns promises), which would
// force all our APIs to become asynchronous as well, which would be annoying.
//
// I think on balance, it's safe enough to use a random library off npm:
// - We only need one hash function (not a full suite of crypto algorithms);
// - SHA256 is quite simple and has fairly few opportunities for subtle bugs
// (compared to asymmetric cryptography anyway);
// - It does not need a secure source of random bits and does not need to be
// constant-time;
// - I have reviewed the source code and it seems pretty reasonable.
const { Hash } = require('fast-sha256')
// These bytes don't mean anything, they were generated randomly
const MAGIC_BYTES = new Uint8Array([0x85, 0x6f, 0x4a, 0x83])
const CHUNK_TYPE_DOCUMENT = 0
const CHUNK_TYPE_CHANGE = 1
const CHUNK_TYPE_DEFLATE = 2 // like CHUNK_TYPE_CHANGE but with DEFLATE compression
// Minimum number of bytes in a value before we enable DEFLATE compression (there is no point
// compressing very short values since compression may actually make them bigger)
const DEFLATE_MIN_SIZE = 256
// The least-significant 3 bits of a columnId indicate its datatype
const COLUMN_TYPE = {
GROUP_CARD: 0, ACTOR_ID: 1, INT_RLE: 2, INT_DELTA: 3, BOOLEAN: 4,
STRING_RLE: 5, VALUE_LEN: 6, VALUE_RAW: 7
}
// The 4th-least-significant bit of a columnId is set if the column is DEFLATE-compressed
const COLUMN_TYPE_DEFLATE = 8
// In the values in a column of type VALUE_LEN, the bottom four bits indicate the type of the value,
// one of the following types in VALUE_TYPE. The higher bits indicate the length of the value in the
// associated VALUE_RAW column (in bytes).
const VALUE_TYPE = {
NULL: 0, FALSE: 1, TRUE: 2, LEB128_UINT: 3, LEB128_INT: 4, IEEE754: 5,
UTF8: 6, BYTES: 7, COUNTER: 8, TIMESTAMP: 9, MIN_UNKNOWN: 10, MAX_UNKNOWN: 15
}
// make* actions must be at even-numbered indexes in this list
const ACTIONS = ['makeMap', 'set', 'makeList', 'del', 'makeText', 'inc', 'makeTable', 'link']
const OBJECT_TYPE = {makeMap: 'map', makeList: 'list', makeText: 'text', makeTable: 'table'}
const COMMON_COLUMNS = [
{columnName: 'objActor', columnId: 0 << 4 | COLUMN_TYPE.ACTOR_ID},
{columnName: 'objCtr', columnId: 0 << 4 | COLUMN_TYPE.INT_RLE},
{columnName: 'keyActor', columnId: 1 << 4 | COLUMN_TYPE.ACTOR_ID},
{columnName: 'keyCtr', columnId: 1 << 4 | COLUMN_TYPE.INT_DELTA},
{columnName: 'keyStr', columnId: 1 << 4 | COLUMN_TYPE.STRING_RLE},
{columnName: 'idActor', columnId: 2 << 4 | COLUMN_TYPE.ACTOR_ID},
{columnName: 'idCtr', columnId: 2 << 4 | COLUMN_TYPE.INT_DELTA},
{columnName: 'insert', columnId: 3 << 4 | COLUMN_TYPE.BOOLEAN},
{columnName: 'action', columnId: 4 << 4 | COLUMN_TYPE.INT_RLE},
{columnName: 'valLen', columnId: 5 << 4 | COLUMN_TYPE.VALUE_LEN},
{columnName: 'valRaw', columnId: 5 << 4 | COLUMN_TYPE.VALUE_RAW},
{columnName: 'chldActor', columnId: 6 << 4 | COLUMN_TYPE.ACTOR_ID},
{columnName: 'chldCtr', columnId: 6 << 4 | COLUMN_TYPE.INT_DELTA}
]
const CHANGE_COLUMNS = COMMON_COLUMNS.concat([
{columnName: 'predNum', columnId: 7 << 4 | COLUMN_TYPE.GROUP_CARD},
{columnName: 'predActor', columnId: 7 << 4 | COLUMN_TYPE.ACTOR_ID},
{columnName: 'predCtr', columnId: 7 << 4 | COLUMN_TYPE.INT_DELTA}
])
const DOC_OPS_COLUMNS = COMMON_COLUMNS.concat([
{columnName: 'succNum', columnId: 8 << 4 | COLUMN_TYPE.GROUP_CARD},
{columnName: 'succActor', columnId: 8 << 4 | COLUMN_TYPE.ACTOR_ID},
{columnName: 'succCtr', columnId: 8 << 4 | COLUMN_TYPE.INT_DELTA}
])
const DOCUMENT_COLUMNS = [
{columnName: 'actor', columnId: 0 << 4 | COLUMN_TYPE.ACTOR_ID},
{columnName: 'seq', columnId: 0 << 4 | COLUMN_TYPE.INT_DELTA},
{columnName: 'maxOp', columnId: 1 << 4 | COLUMN_TYPE.INT_DELTA},
{columnName: 'time', columnId: 2 << 4 | COLUMN_TYPE.INT_DELTA},
{columnName: 'message', columnId: 3 << 4 | COLUMN_TYPE.STRING_RLE},
{columnName: 'depsNum', columnId: 4 << 4 | COLUMN_TYPE.GROUP_CARD},
{columnName: 'depsIndex', columnId: 4 << 4 | COLUMN_TYPE.INT_DELTA},
{columnName: 'extraLen', columnId: 5 << 4 | COLUMN_TYPE.VALUE_LEN},
{columnName: 'extraRaw', columnId: 5 << 4 | COLUMN_TYPE.VALUE_RAW}
]
/**
* Maps an opId of the form {counter: 12345, actorId: 'someActorId'} to the form
* {counter: 12345, actorNum: 123, actorId: 'someActorId'}, where the actorNum
* is the index into the `actorIds` array.
*/
function actorIdToActorNum(opId, actorIds) {
if (!opId || !opId.actorId) return opId
const counter = opId.counter
const actorNum = actorIds.indexOf(opId.actorId)
if (actorNum < 0) throw new RangeError('missing actorId') // should not happen
return {counter, actorNum, actorId: opId.actorId}
}
/**
* Comparison function to pass to Array.sort(), which compares two opIds in the
* form produced by `actorIdToActorNum` so that they are sorted in increasing
* Lamport timestamp order (sorted first by counter, then by actorId).
*/
function compareParsedOpIds(id1, id2) {
if (id1.counter < id2.counter) return -1
if (id1.counter > id2.counter) return +1
if (id1.actorId < id2.actorId) return -1
if (id1.actorId > id2.actorId) return +1
return 0
}
/**
* Takes `changes`, an array of changes (represented as JS objects). Returns an
* object `{changes, actorIds}`, where `changes` is a copy of the argument in
* which all string opIds have been replaced with `{counter, actorNum}` objects,
* and where `actorIds` is a lexicographically sorted array of actor IDs occurring
* in any of the operations. `actorNum` is an index into that array of actorIds.
* If `single` is true, the actorId of the author of the change is moved to the
* beginning of the array of actorIds, so that `actorNum` is zero when referencing
* the author of the change itself. This special-casing is omitted if `single` is
* false.
*/
function parseAllOpIds(changes, single) {
const actors = {}, newChanges = []
for (let change of changes) {
change = copyObject(change)
actors[change.actor] = true
change.ops = expandMultiOps(change.ops, change.startOp, change.actor)
change.ops = change.ops.map(op => {
op = copyObject(op)
if (op.obj !== '_root') op.obj = parseOpId(op.obj)
if (op.elemId && op.elemId !== '_head') op.elemId = parseOpId(op.elemId)
if (op.child) op.child = parseOpId(op.child)
if (op.pred) op.pred = op.pred.map(parseOpId)
if (op.obj.actorId) actors[op.obj.actorId] = true
if (op.elemId && op.elemId.actorId) actors[op.elemId.actorId] = true
if (op.child && op.child.actorId) actors[op.child.actorId] = true
for (let pred of op.pred) actors[pred.actorId] = true
return op
})
newChanges.push(change)
}
let actorIds = Object.keys(actors).sort()
if (single) {
actorIds = [changes[0].actor].concat(actorIds.filter(actor => actor !== changes[0].actor))
}
for (let change of newChanges) {
change.actorNum = actorIds.indexOf(change.actor)
for (let i = 0; i < change.ops.length; i++) {
let op = change.ops[i]
op.id = {counter: change.startOp + i, actorNum: change.actorNum, actorId: change.actor}
op.obj = actorIdToActorNum(op.obj, actorIds)
op.elemId = actorIdToActorNum(op.elemId, actorIds)
op.child = actorIdToActorNum(op.child, actorIds)
op.pred = op.pred.map(pred => actorIdToActorNum(pred, actorIds))
}
}
return {changes: newChanges, actorIds}
}
/**
* Encodes the `obj` property of operation `op` into the two columns
* `objActor` and `objCtr`.
*/
function encodeObjectId(op, columns) {
if (op.obj === '_root') {
columns.objActor.appendValue(null)
columns.objCtr.appendValue(null)
} else if (op.obj.actorNum >= 0 && op.obj.counter > 0) {
columns.objActor.appendValue(op.obj.actorNum)
columns.objCtr.appendValue(op.obj.counter)
} else {
throw new RangeError(`Unexpected objectId reference: ${JSON.stringify(op.obj)}`)
}
}
/**
* Encodes the `key` and `elemId` properties of operation `op` into the three
* columns `keyActor`, `keyCtr`, and `keyStr`.
*/
function encodeOperationKey(op, columns) {
if (op.key) {
columns.keyActor.appendValue(null)
columns.keyCtr.appendValue(null)
columns.keyStr.appendValue(op.key)
} else if (op.elemId === '_head' && op.insert) {
columns.keyActor.appendValue(null)
columns.keyCtr.appendValue(0)
columns.keyStr.appendValue(null)
} else if (op.elemId && op.elemId.actorNum >= 0 && op.elemId.counter > 0) {
columns.keyActor.appendValue(op.elemId.actorNum)
columns.keyCtr.appendValue(op.elemId.counter)
columns.keyStr.appendValue(null)
} else {
throw new RangeError(`Unexpected operation key: ${JSON.stringify(op)}`)
}
}
/**
* Encodes the `action` property of operation `op` into the `action` column.
*/
function encodeOperationAction(op, columns) {
const actionCode = ACTIONS.indexOf(op.action)
if (actionCode >= 0) {
columns.action.appendValue(actionCode)
} else if (typeof op.action === 'number') {
columns.action.appendValue(op.action)
} else {
throw new RangeError(`Unexpected operation action: ${op.action}`)
}
}
/**
* Given the datatype for a number, determine the typeTag and the value to encode
* otherwise guess
*/
function getNumberTypeAndValue(op) {
switch (op.datatype) {
case "counter":
return [ VALUE_TYPE.COUNTER, op.value ]
case "timestamp":
return [ VALUE_TYPE.TIMESTAMP, op.value ]
case "uint":
return [ VALUE_TYPE.LEB128_UINT, op.value ]
case "int":
return [ VALUE_TYPE.LEB128_INT, op.value ]
case "float64": {
const buf64 = new ArrayBuffer(8), view64 = new DataView(buf64)
view64.setFloat64(0, op.value, true)
return [ VALUE_TYPE.IEEE754, new Uint8Array(buf64) ]
}
default:
// increment operators get resolved here ...
if (Number.isInteger(op.value) && op.value <= Number.MAX_SAFE_INTEGER && op.value >= Number.MIN_SAFE_INTEGER) {
return [ VALUE_TYPE.LEB128_INT, op.value ]
} else {
const buf64 = new ArrayBuffer(8), view64 = new DataView(buf64)
view64.setFloat64(0, op.value, true)
return [ VALUE_TYPE.IEEE754, new Uint8Array(buf64) ]
}
}
}
/**
* Encodes the `value` property of operation `op` into the two columns
* `valLen` and `valRaw`.
*/
function encodeValue(op, columns) {
if ((op.action !== 'set' && op.action !== 'inc') || op.value === null) {
columns.valLen.appendValue(VALUE_TYPE.NULL)
} else if (op.value === false) {
columns.valLen.appendValue(VALUE_TYPE.FALSE)
} else if (op.value === true) {
columns.valLen.appendValue(VALUE_TYPE.TRUE)
} else if (typeof op.value === 'string') {
const numBytes = columns.valRaw.appendRawString(op.value)
columns.valLen.appendValue(numBytes << 4 | VALUE_TYPE.UTF8)
} else if (ArrayBuffer.isView(op.value)) {
const numBytes = columns.valRaw.appendRawBytes(new Uint8Array(op.value.buffer))
columns.valLen.appendValue(numBytes << 4 | VALUE_TYPE.BYTES)
} else if (typeof op.value === 'number') {
let [typeTag, value] = getNumberTypeAndValue(op)
let numBytes
if (typeTag === VALUE_TYPE.LEB128_UINT) {
numBytes = columns.valRaw.appendUint53(value)
} else if (typeTag === VALUE_TYPE.IEEE754) {
numBytes = columns.valRaw.appendRawBytes(value)
} else {
numBytes = columns.valRaw.appendInt53(value)
}
columns.valLen.appendValue(numBytes << 4 | typeTag)
} else if (typeof op.datatype === 'number' && op.datatype >= VALUE_TYPE.MIN_UNKNOWN &&
op.datatype <= VALUE_TYPE.MAX_UNKNOWN && op.value instanceof Uint8Array) {
const numBytes = columns.valRaw.appendRawBytes(op.value)
columns.valLen.appendValue(numBytes << 4 | op.datatype)
} else if (op.datatype) {
throw new RangeError(`Unknown datatype ${op.datatype} for value ${op.value}`)
} else {
throw new RangeError(`Unsupported value in operation: ${op.value}`)
}
}
/**
* Given `sizeTag` (an unsigned integer read from a VALUE_LEN column) and `bytes` (a Uint8Array
* read from a VALUE_RAW column, with length `sizeTag >> 4`), this function returns an object of the
* form `{value: value, datatype: datatypeTag}` where `value` is a JavaScript primitive datatype
* corresponding to the value, and `datatypeTag` is a datatype annotation such as 'counter'.
*/
function decodeValue(sizeTag, bytes) {
if (sizeTag === VALUE_TYPE.NULL) {
return {value: null}
} else if (sizeTag === VALUE_TYPE.FALSE) {
return {value: false}
} else if (sizeTag === VALUE_TYPE.TRUE) {
return {value: true}
} else if (sizeTag % 16 === VALUE_TYPE.UTF8) {
return {value: utf8ToString(bytes)}
} else {
if (sizeTag % 16 === VALUE_TYPE.LEB128_UINT) {
return {value: new Decoder(bytes).readUint53(), datatype: "uint"}
} else if (sizeTag % 16 === VALUE_TYPE.LEB128_INT) {
return {value: new Decoder(bytes).readInt53(), datatype: "int"}
} else if (sizeTag % 16 === VALUE_TYPE.IEEE754) {
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength)
if (bytes.byteLength === 8) {
return {value: view.getFloat64(0, true), datatype: "float64"}
} else {
throw new RangeError(`Invalid length for floating point number: ${bytes.byteLength}`)
}
} else if (sizeTag % 16 === VALUE_TYPE.COUNTER) {
return {value: new Decoder(bytes).readInt53(), datatype: 'counter'}
} else if (sizeTag % 16 === VALUE_TYPE.TIMESTAMP) {
return {value: new Decoder(bytes).readInt53(), datatype: 'timestamp'}
} else {
return {value: bytes, datatype: sizeTag % 16}
}
}
}
/**
* Reads one value from the column `columns[colIndex]` and interprets it based
* on the column type. `actorIds` is a list of actors that appear in the change;
* `actorIds[0]` is the actorId of the change's author. Mutates the `result`
* object with the value, and returns the number of columns processed (this is 2
* in the case of a pair of VALUE_LEN and VALUE_RAW columns, which are processed
* in one go).
*/
function decodeValueColumns(columns, colIndex, actorIds, result) {
const { columnId, columnName, decoder } = columns[colIndex]
if (columnId % 8 === COLUMN_TYPE.VALUE_LEN && colIndex + 1 < columns.length &&
columns[colIndex + 1].columnId === columnId + 1) {
const sizeTag = decoder.readValue()
const rawValue = columns[colIndex + 1].decoder.readRawBytes(sizeTag >> 4)
const { value, datatype } = decodeValue(sizeTag, rawValue)
result[columnName] = value
if (datatype) result[columnName + '_datatype'] = datatype
return 2
} else if (columnId % 8 === COLUMN_TYPE.ACTOR_ID) {
const actorNum = decoder.readValue()
if (actorNum === null) {
result[columnName] = null
} else {
if (!actorIds[actorNum]) throw new RangeError(`No actor index ${actorNum}`)
result[columnName] = actorIds[actorNum]
}
} else {
result[columnName] = decoder.readValue()
}
return 1
}
/**
* Encodes an array of operations in a set of columns. The operations need to
* be parsed with `parseAllOpIds()` beforehand. If `forDocument` is true, we use
* the column structure of a whole document, otherwise we use the column
* structure for an individual change. Returns an array of
* `{columnId, columnName, encoder}` objects.
*/
function encodeOps(ops, forDocument) {
const columns = {
objActor : new RLEEncoder('uint'),
objCtr : new RLEEncoder('uint'),
keyActor : new RLEEncoder('uint'),
keyCtr : new DeltaEncoder(),
keyStr : new RLEEncoder('utf8'),
insert : new BooleanEncoder(),
action : new RLEEncoder('uint'),
valLen : new RLEEncoder('uint'),
valRaw : new Encoder(),
chldActor : new RLEEncoder('uint'),
chldCtr : new DeltaEncoder()
}
if (forDocument) {
columns.idActor = new RLEEncoder('uint')
columns.idCtr = new DeltaEncoder()
columns.succNum = new RLEEncoder('uint')
columns.succActor = new RLEEncoder('uint')
columns.succCtr = new DeltaEncoder()
} else {
columns.predNum = new RLEEncoder('uint')
columns.predCtr = new DeltaEncoder()
columns.predActor = new RLEEncoder('uint')
}
for (let op of ops) {
encodeObjectId(op, columns)
encodeOperationKey(op, columns)
columns.insert.appendValue(!!op.insert)
encodeOperationAction(op, columns)
encodeValue(op, columns)
if (op.child && op.child.counter) {
columns.chldActor.appendValue(op.child.actorNum)
columns.chldCtr.appendValue(op.child.counter)
} else {
columns.chldActor.appendValue(null)
columns.chldCtr.appendValue(null)
}
if (forDocument) {
columns.idActor.appendValue(op.id.actorNum)
columns.idCtr.appendValue(op.id.counter)
columns.succNum.appendValue(op.succ.length)
op.succ.sort(compareParsedOpIds)
for (let i = 0; i < op.succ.length; i++) {
columns.succActor.appendValue(op.succ[i].actorNum)
columns.succCtr.appendValue(op.succ[i].counter)
}
} else {
columns.predNum.appendValue(op.pred.length)
op.pred.sort(compareParsedOpIds)
for (let i = 0; i < op.pred.length; i++) {
columns.predActor.appendValue(op.pred[i].actorNum)
columns.predCtr.appendValue(op.pred[i].counter)
}
}
}
let columnList = []
for (let {columnName, columnId} of forDocument ? DOC_OPS_COLUMNS : CHANGE_COLUMNS) {
if (columns[columnName]) columnList.push({columnId, columnName, encoder: columns[columnName]})
}
return columnList.sort((a, b) => a.columnId - b.columnId)
}
function validDatatype(value, datatype) {
if (datatype === undefined) {
return (typeof value === 'string' || typeof value === 'boolean' || value === null)
} else {
return typeof value === 'number'
}
}
function expandMultiOps(ops, startOp, actor) {
let opNum = startOp
let expandedOps = []
for (const op of ops) {
if (op.action === 'set' && op.values && op.insert) {
if (op.pred.length !== 0) throw new RangeError('multi-insert pred must be empty')
let lastElemId = op.elemId
const datatype = op.datatype
for (const value of op.values) {
if (!validDatatype(value, datatype)) throw new RangeError(`Decode failed: bad value/datatype association (${value},${datatype})`)
expandedOps.push({action: 'set', obj: op.obj, elemId: lastElemId, datatype, value, pred: [], insert: true})
lastElemId = `${opNum}@${actor}`
opNum += 1
}
} else if (op.action === 'del' && op.multiOp > 1) {
if (op.pred.length !== 1) throw new RangeError('multiOp deletion must have exactly one pred')
const startElemId = parseOpId(op.elemId), startPred = parseOpId(op.pred[0])
for (let i = 0; i < op.multiOp; i++) {
const elemId = `${startElemId.counter + i}@${startElemId.actorId}`
const pred = [`${startPred.counter + i}@${startPred.actorId}`]
expandedOps.push({action: 'del', obj: op.obj, elemId, pred})
opNum += 1
}
} else {
expandedOps.push(op)
opNum += 1
}
}
return expandedOps
}
/**
* Takes a change as decoded by `decodeColumns`, and changes it into the form
* expected by the rest of the backend. If `forDocument` is true, we use the op
* structure of a whole document, otherwise we use the op structure for an
* individual change.
*/
function decodeOps(ops, forDocument) {
const newOps = []
for (let op of ops) {
const obj = (op.objCtr === null) ? '_root' : `${op.objCtr}@${op.objActor}`
const elemId = op.keyStr ? undefined : (op.keyCtr === 0 ? '_head' : `${op.keyCtr}@${op.keyActor}`)
const action = ACTIONS[op.action] || op.action
const newOp = elemId ? {obj, elemId, action} : {obj, key: op.keyStr, action}
newOp.insert = !!op.insert
if (ACTIONS[op.action] === 'set' || ACTIONS[op.action] === 'inc') {
newOp.value = op.valLen
if (op.valLen_datatype) newOp.datatype = op.valLen_datatype
}
if (!!op.chldCtr !== !!op.chldActor) {
throw new RangeError(`Mismatched child columns: ${op.chldCtr} and ${op.chldActor}`)
}
if (op.chldCtr !== null) newOp.child = `${op.chldCtr}@${op.chldActor}`
if (forDocument) {
newOp.id = `${op.idCtr}@${op.idActor}`
newOp.succ = op.succNum.map(succ => `${succ.succCtr}@${succ.succActor}`)
checkSortedOpIds(op.succNum.map(succ => ({counter: succ.succCtr, actorId: succ.succActor})))
} else {
newOp.pred = op.predNum.map(pred => `${pred.predCtr}@${pred.predActor}`)
checkSortedOpIds(op.predNum.map(pred => ({counter: pred.predCtr, actorId: pred.predActor})))
}
newOps.push(newOp)
}
return newOps
}
/**
* Throws an exception if the opIds in the given array are not in sorted order.
*/
function checkSortedOpIds(opIds) {
let last = null
for (let opId of opIds) {
if (last && compareParsedOpIds(last, opId) !== -1) {
throw new RangeError('operation IDs are not in ascending order')
}
last = opId
}
}
function encoderByColumnId(columnId) {
if ((columnId & 7) === COLUMN_TYPE.INT_DELTA) {
return new DeltaEncoder()
} else if ((columnId & 7) === COLUMN_TYPE.BOOLEAN) {
return new BooleanEncoder()
} else if ((columnId & 7) === COLUMN_TYPE.STRING_RLE) {
return new RLEEncoder('utf8')
} else if ((columnId & 7) === COLUMN_TYPE.VALUE_RAW) {
return new Encoder()
} else {
return new RLEEncoder('uint')
}
}
function decoderByColumnId(columnId, buffer) {
if ((columnId & 7) === COLUMN_TYPE.INT_DELTA) {
return new DeltaDecoder(buffer)
} else if ((columnId & 7) === COLUMN_TYPE.BOOLEAN) {
return new BooleanDecoder(buffer)
} else if ((columnId & 7) === COLUMN_TYPE.STRING_RLE) {
return new RLEDecoder('utf8', buffer)
} else if ((columnId & 7) === COLUMN_TYPE.VALUE_RAW) {
return new Decoder(buffer)
} else {
return new RLEDecoder('uint', buffer)
}
}
function makeDecoders(columns, columnSpec) {
const emptyBuf = new Uint8Array(0)
let decoders = [], columnIndex = 0, specIndex = 0
while (columnIndex < columns.length || specIndex < columnSpec.length) {
if (columnIndex === columns.length ||
(specIndex < columnSpec.length && columnSpec[specIndex].columnId < columns[columnIndex].columnId)) {
const {columnId, columnName} = columnSpec[specIndex]
decoders.push({columnId, columnName, decoder: decoderByColumnId(columnId, emptyBuf)})
specIndex++
} else if (specIndex === columnSpec.length || columns[columnIndex].columnId < columnSpec[specIndex].columnId) {
const {columnId, buffer} = columns[columnIndex]
decoders.push({columnId, decoder: decoderByColumnId(columnId, buffer)})
columnIndex++
} else { // columns[columnIndex].columnId === columnSpec[specIndex].columnId
const {columnId, buffer} = columns[columnIndex], {columnName} = columnSpec[specIndex]
decoders.push({columnId, columnName, decoder: decoderByColumnId(columnId, buffer)})
columnIndex++
specIndex++
}
}
return decoders
}
function decodeColumns(columns, actorIds, columnSpec) {
columns = makeDecoders(columns, columnSpec)
let parsedRows = []
while (columns.some(col => !col.decoder.done)) {
let row = {}, col = 0
while (col < columns.length) {
const columnId = columns[col].columnId
let groupId = columnId >> 4, groupCols = 1
while (col + groupCols < columns.length && columns[col + groupCols].columnId >> 4 === groupId) {
groupCols++
}
if (columnId % 8 === COLUMN_TYPE.GROUP_CARD) {
const values = [], count = columns[col].decoder.readValue()
for (let i = 0; i < count; i++) {
let value = {}
for (let colOffset = 1; colOffset < groupCols; colOffset++) {
decodeValueColumns(columns, col + colOffset, actorIds, value)
}
values.push(value)
}
row[columns[col].columnName] = values
col += groupCols
} else {
col += decodeValueColumns(columns, col, actorIds, row)
}
}
parsedRows.push(row)
}
return parsedRows
}
function decodeColumnInfo(decoder) {
// A number that is all 1 bits except for the bit that indicates whether a column is
// deflate-compressed. We ignore this bit when checking whether columns are sorted by ID.
const COLUMN_ID_MASK = (-1 ^ COLUMN_TYPE_DEFLATE) >>> 0
let lastColumnId = -1, columns = [], numColumns = decoder.readUint53()
for (let i = 0; i < numColumns; i++) {
const columnId = decoder.readUint53(), bufferLen = decoder.readUint53()
if ((columnId & COLUMN_ID_MASK) <= (lastColumnId & COLUMN_ID_MASK)) {
throw new RangeError('Columns must be in ascending order')
}
lastColumnId = columnId
columns.push({columnId, bufferLen})
}
return columns
}
function encodeColumnInfo(encoder, columns) {
const nonEmptyColumns = columns.filter(column => column.encoder.buffer.byteLength > 0)
encoder.appendUint53(nonEmptyColumns.length)
for (let column of nonEmptyColumns) {
encoder.appendUint53(column.columnId)
encoder.appendUint53(column.encoder.buffer.byteLength)
}
}
function decodeChangeHeader(decoder) {
const numDeps = decoder.readUint53(), deps = []
for (let i = 0; i < numDeps; i++) {
deps.push(bytesToHexString(decoder.readRawBytes(32)))
}
let change = {
actor: decoder.readHexString(),
seq: decoder.readUint53(),
startOp: decoder.readUint53(),
time: decoder.readInt53(),
message: decoder.readPrefixedString(),
deps
}
const actorIds = [change.actor], numActorIds = decoder.readUint53()
for (let i = 0; i < numActorIds; i++) actorIds.push(decoder.readHexString())
change.actorIds = actorIds
return change
}
/**
* Assembles a chunk of encoded data containing a checksum, headers, and a
* series of encoded columns. Calls `encodeHeaderCallback` with an encoder that
* should be used to add the headers. The columns should be given as `columns`.
*/
function encodeContainer(chunkType, encodeContentsCallback) {
const CHECKSUM_SIZE = 4 // checksum is first 4 bytes of SHA-256 hash of the rest of the data
const HEADER_SPACE = MAGIC_BYTES.byteLength + CHECKSUM_SIZE + 1 + 5 // 1 byte type + 5 bytes length
const body = new Encoder()
// Make space for the header at the beginning of the body buffer. We will
// copy the header in here later. This is cheaper than copying the body since
// the body is likely to be much larger than the header.
body.appendRawBytes(new Uint8Array(HEADER_SPACE))
encodeContentsCallback(body)
const bodyBuf = body.buffer
const header = new Encoder()
header.appendByte(chunkType)
header.appendUint53(bodyBuf.byteLength - HEADER_SPACE)
// Compute the hash over chunkType, length, and body
const headerBuf = header.buffer
const sha256 = new Hash()
sha256.update(headerBuf)
sha256.update(bodyBuf.subarray(HEADER_SPACE))
const hash = sha256.digest(), checksum = hash.subarray(0, CHECKSUM_SIZE)
// Copy header into the body buffer so that they are contiguous
bodyBuf.set(MAGIC_BYTES, HEADER_SPACE - headerBuf.byteLength - CHECKSUM_SIZE - MAGIC_BYTES.byteLength)
bodyBuf.set(checksum, HEADER_SPACE - headerBuf.byteLength - CHECKSUM_SIZE)
bodyBuf.set(headerBuf, HEADER_SPACE - headerBuf.byteLength)
return {hash, bytes: bodyBuf.subarray(HEADER_SPACE - headerBuf.byteLength - CHECKSUM_SIZE - MAGIC_BYTES.byteLength)}
}
function decodeContainerHeader(decoder, computeHash) {
if (!equalBytes(decoder.readRawBytes(MAGIC_BYTES.byteLength), MAGIC_BYTES)) {
throw new RangeError('Data does not begin with magic bytes 85 6f 4a 83')
}
const expectedHash = decoder.readRawBytes(4)
const hashStartOffset = decoder.offset
const chunkType = decoder.readByte()
const chunkLength = decoder.readUint53()
const header = {chunkType, chunkLength, chunkData: decoder.readRawBytes(chunkLength)}
if (computeHash) {
const sha256 = new Hash()
sha256.update(decoder.buf.subarray(hashStartOffset, decoder.offset))
const binaryHash = sha256.digest()
if (!equalBytes(binaryHash.subarray(0, 4), expectedHash)) {
throw new RangeError('checksum does not match data')
}
header.hash = bytesToHexString(binaryHash)
}
return header
}
function encodeChange(changeObj) {
const { changes, actorIds } = parseAllOpIds([changeObj], true)
const change = changes[0]
const { hash, bytes } = encodeContainer(CHUNK_TYPE_CHANGE, encoder => {
if (!Array.isArray(change.deps)) throw new TypeError('deps is not an array')
encoder.appendUint53(change.deps.length)
for (let hash of change.deps.slice().sort()) {
encoder.appendRawBytes(hexStringToBytes(hash))
}
encoder.appendHexString(change.actor)
encoder.appendUint53(change.seq)
encoder.appendUint53(change.startOp)
encoder.appendInt53(change.time)
encoder.appendPrefixedString(change.message || '')
encoder.appendUint53(actorIds.length - 1)
for (let actor of actorIds.slice(1)) encoder.appendHexString(actor)
const columns = encodeOps(change.ops, false)
encodeColumnInfo(encoder, columns)
for (let column of columns) encoder.appendRawBytes(column.encoder.buffer)
if (change.extraBytes) encoder.appendRawBytes(change.extraBytes)
})
const hexHash = bytesToHexString(hash)
if (changeObj.hash && changeObj.hash !== hexHash) {
throw new RangeError(`Change hash does not match encoding: ${changeObj.hash} != ${hexHash}`)
}
return (bytes.byteLength >= DEFLATE_MIN_SIZE) ? deflateChange(bytes) : bytes
}
function decodeChangeColumns(buffer) {
if (buffer[8] === CHUNK_TYPE_DEFLATE) buffer = inflateChange(buffer)
const decoder = new Decoder(buffer)
const header = decodeContainerHeader(decoder, true)
const chunkDecoder = new Decoder(header.chunkData)
if (!decoder.done) throw new RangeError('Encoded change has trailing data')
if (header.chunkType !== CHUNK_TYPE_CHANGE) throw new RangeError(`Unexpected chunk type: ${header.chunkType}`)
const change = decodeChangeHeader(chunkDecoder)
const columns = decodeColumnInfo(chunkDecoder)
for (let i = 0; i < columns.length; i++) {
if ((columns[i].columnId & COLUMN_TYPE_DEFLATE) !== 0) {
throw new RangeError('change must not contain deflated columns')
}
columns[i].buffer = chunkDecoder.readRawBytes(columns[i].bufferLen)
}
if (!chunkDecoder.done) {
const restLen = chunkDecoder.buf.byteLength - chunkDecoder.offset
change.extraBytes = chunkDecoder.readRawBytes(restLen)
}
change.columns = columns
change.hash = header.hash
return change
}
/**
* Decodes one change in binary format into its JS object representation.
*/
function decodeChange(buffer) {
const change = decodeChangeColumns(buffer)
change.ops = decodeOps(decodeColumns(change.columns, change.actorIds, CHANGE_COLUMNS), false)
delete change.actorIds
delete change.columns
return change
}
/**
* Decodes the header fields of a change in binary format, but does not decode
* the operations. Saves work when we only need to inspect the headers. Only
* computes the hash of the change if `computeHash` is true.
*/
function decodeChangeMeta(buffer, computeHash) {
if (buffer[8] === CHUNK_TYPE_DEFLATE) buffer = inflateChange(buffer)
const header = decodeContainerHeader(new Decoder(buffer), computeHash)
if (header.chunkType !== CHUNK_TYPE_CHANGE) {
throw new RangeError('Buffer chunk type is not a change')
}
const meta = decodeChangeHeader(new Decoder(header.chunkData))
meta.change = buffer
if (computeHash) meta.hash = header.hash
return meta
}
/**
* Compresses a binary change using DEFLATE.
*/
function deflateChange(buffer) {
const header = decodeContainerHeader(new Decoder(buffer), false)
if (header.chunkType !== CHUNK_TYPE_CHANGE) throw new RangeError(`Unexpected chunk type: ${header.chunkType}`)
const compressed = pako.deflateRaw(header.chunkData)
const encoder = new Encoder()
encoder.appendRawBytes(buffer.subarray(0, 8)) // copy MAGIC_BYTES and checksum
encoder.appendByte(CHUNK_TYPE_DEFLATE)
encoder.appendUint53(compressed.byteLength)
encoder.appendRawBytes(compressed)
return encoder.buffer
}
/**
* Decompresses a binary change that has been compressed with DEFLATE.
*/
function inflateChange(buffer) {
const header = decodeContainerHeader(new Decoder(buffer), false)
if (header.chunkType !== CHUNK_TYPE_DEFLATE) throw new RangeError(`Unexpected chunk type: ${header.chunkType}`)
const decompressed = pako.inflateRaw(header.chunkData)
const encoder = new Encoder()
encoder.appendRawBytes(buffer.subarray(0, 8)) // copy MAGIC_BYTES and checksum
encoder.appendByte(CHUNK_TYPE_CHANGE)
encoder.appendUint53(decompressed.byteLength)
encoder.appendRawBytes(decompressed)
return encoder.buffer
}
/**
* Takes an Uint8Array that may contain multiple concatenated changes, and
* returns an array of subarrays, each subarray containing one change.
*/
function splitContainers(buffer) {
let decoder = new Decoder(buffer), chunks = [], startOffset = 0
while (!decoder.done) {
decodeContainerHeader(decoder, false)
chunks.push(buffer.subarray(startOffset, decoder.offset))
startOffset = decoder.offset
}
return chunks
}
/**
* Decodes a list of changes from the binary format into JS objects.
* `binaryChanges` is an array of `Uint8Array` objects.
*/
function decodeChanges(binaryChanges) {
let decoded = []
for (let binaryChange of binaryChanges) {
for (let chunk of splitContainers(binaryChange)) {
if (chunk[8] === CHUNK_TYPE_DOCUMENT) {
decoded = decoded.concat(decodeDocument(chunk))
} else if (chunk[8] === CHUNK_TYPE_CHANGE || chunk[8] === CHUNK_TYPE_DEFLATE) {
decoded.push(decodeChange(chunk))
} else {
// ignoring chunk of unknown type
}
}
}
return decoded
}
function sortOpIds(a, b) {
if (a === b) return 0
if (a === '_root') return -1
if (b === '_root') return +1
const a_ = parseOpId(a), b_ = parseOpId(b)
if (a_.counter < b_.counter) return -1
if (a_.counter > b_.counter) return +1
if (a_.actorId < b_.actorId) return -1
if (a_.actorId > b_.actorId) return +1
return 0
}
/**
* Takes a set of operations `ops` loaded from an encoded document, and
* reconstructs the changes that they originally came from.
* Does not return anything, only mutates `changes`.
*/
function groupChangeOps(changes, ops) {
let changesByActor = {} // map from actorId to array of changes by that actor
for (let change of changes) {
change.ops = []
if (!changesByActor[change.actor]) changesByActor[change.actor] = []
if (change.seq !== changesByActor[change.actor].length + 1) {
throw new RangeError(`Expected seq = ${changesByActor[change.actor].length + 1}, got ${change.seq}`)
}
if (change.seq > 1 && changesByActor[change.actor][change.seq - 2].maxOp > change.maxOp) {
throw new RangeError('maxOp must increase monotonically per actor')
}
changesByActor[change.actor].push(change)
}
let opsById = {}
for (let op of ops) {
if (op.action === 'del') throw new RangeError('document should not contain del operations')
op.pred = opsById[op.id] ? opsById[op.id].pred : []
opsById[op.id] = op
for (let succ of op.succ) {
if (!opsById[succ]) {
if (op.elemId) {
const elemId = op.insert ? op.id : op.elemId
opsById[succ] = {id: succ, action: 'del', obj: op.obj, elemId, pred: []}
} else {
opsById[succ] = {id: succ, action: 'del', obj: op.obj, key: op.key, pred: []}
}
}
opsById[succ].pred.push(op.id)
}
delete op.succ
}
for (let op of Object.values(opsById)) {
if (op.action === 'del') ops.push(op)
}
for (let op of ops) {
const { counter, actorId } = parseOpId(op.id)
const actorChanges = changesByActor[actorId]
// Binary search to find the change that should contain this operation
let left = 0, right = actorChanges.length
while (left < right) {
const index = Math.floor((left + right) / 2)
if (actorChanges[index].maxOp < counter) {
left = index + 1
} else {
right = index
}
}
if (left >= actorChanges.length) {
throw new RangeError(`Operation ID ${op.id} outside of allowed range`)
}
actorChanges[left].ops.push(op)
}
for (let change of changes) {
change.ops.sort((op1, op2) => sortOpIds(op1.id, op2.id))
change.startOp = change.maxOp - change.ops.length + 1
delete change.maxOp
for (let i = 0; i < change.ops.length; i++) {
const op = change.ops[i], expectedId = `${change.startOp + i}@${change.actor}`
if (op.id !== expectedId) {
throw new RangeError(`Expected opId ${expectedId}, got ${op.id}`)
}
delete op.id
}
}
}
function decodeDocumentChanges(changes, expectedHeads) {
let heads = {} // change hashes that are not a dependency of any other change
for (let i = 0; i < changes.length; i++) {
let change = changes[i]
change.deps = []
for (let index of change.depsNum.map(d => d.depsIndex)) {
if (!changes[index] || !changes[index].hash) {
throw new RangeError(`No hash for index ${index} while processing index ${i}`)
}
const hash = changes[index].hash
change.deps.push(hash)
if (heads[hash]) delete heads[hash]
}
change.deps.sort()
delete change.depsNum
if (change.extraLen_datatype !== VALUE_TYPE.BYTES) {
throw new RangeError(`Bad datatype for extra bytes: ${VALUE_TYPE.BYTES}`)
}
change.extraBytes = change.extraLen
delete change.extraLen_datatype
// Encoding and decoding again to compute the hash of the change
changes[i] = decodeChange(encodeChange(change))
heads[changes[i].hash] = true
}
const actualHeads = Object.keys(heads).sort()
let headsEqual = (actualHeads.length === expectedHeads.length), i = 0
while (headsEqual && i < actualHeads.length) {
headsEqual = (actualHeads[i] === expectedHeads[i])
i++
}
if (!headsEqual) {
throw new RangeError(`Mismatched heads hashes: expected ${expectedHeads.join(', ')}, got ${actualHeads.join(', ')}`)
}
}
function encodeDocumentHeader(doc) {
const { changesColumns, opsColumns, actorIds, heads, headsIndexes, extraBytes } = doc
for (let column of changesColumns) deflateColumn(column)
for (let column of opsColumns) deflateColumn(column)
return encodeContainer(CHUNK_TYPE_DOCUMENT, encoder => {
encoder.appendUint53(actorIds.length)
for (let actor of actorIds) {
encoder.appendHexString(actor)
}
encoder.appendUint53(heads.length)
for (let head of heads.sort()) {
encoder.appendRawBytes(hexStringToBytes(head))
}
encodeColumnInfo(encoder, changesColumns)
encodeColumnInfo(encoder, opsColumns)
for (let column of changesColumns) encoder.appendRawBytes(column.encoder.buffer)
for (let column of opsColumns) encoder.appendRawBytes(column.encoder.buffer)
for (let index of headsIndexes) encoder.appendUint53(index)
if (extraBytes) encoder.appendRawBytes(extraBytes)
}).bytes
}
function decodeDocumentHeader(buffer) {
const documentDecoder = new Decoder(buffer)
const header = decodeContainerHeader(documentDecoder, true)
const decoder = new Decoder(header.chunkData)
if (!documentDecoder.done) throw new RangeError('Encoded document has trailing data')
if (header.chunkType !== CHUNK_TYPE_DOCUMENT) throw new RangeError(`Unexpected chunk type: ${header.chunkType}`)
const actorIds = [], numActors = decoder.readUint53()
for (let i = 0; i < numActors; i++) {
actorIds.push(decoder.readHexString())
}
const heads = [], headsIndexes = [], numHeads = decoder.readUint53()
for (let i = 0; i < numHeads; i++) {
heads.push(bytesToHexString(decoder.readRawBytes(32)))
}
const changesColumns = decodeColumnInfo(decoder)
const opsColumns = decodeColumnInfo(decoder)
for (let i = 0; i < changesColumns.length; i++) {
changesColumns[i].buffer = decoder.readRawBytes(changesColumns[i].bufferLen)
inflateColumn(changesColumns[i])
}
for (let i = 0; i < opsColumns.length; i++) {
opsColumns[i].buffer = decoder.readRawBytes(opsColumns[i].bufferLen)
inflateColumn(opsColumns[i])
}
if (!decoder.done) {
for (let i = 0; i < numHeads; i++) headsIndexes.push(decoder.readUint53())
}
const extraBytes = decoder.readRawBytes(decoder.buf.byteLength - decoder.offset)
return { changesColumns, opsColumns, actorIds, heads, headsIndexes, extraBytes }
}
function decodeDocument(buffer) {
const { changesColumns, opsColumns, actorIds, heads } = decodeDocumentHeader(buffer)
const changes = decodeColumns(changesColumns, actorIds, DOCUMENT_COLUMNS)
const ops = decodeOps(decodeColumns(opsColumns, actorIds, DOC_OPS_COLUMNS), true)
groupChangeOps(changes, ops)
decodeDocumentChanges(changes, heads)
return changes
}
/**
* DEFLATE-compresses the given column if it is large enough to make the compression worthwhile.
*/
function deflateColumn(column) {
if (column.encoder.buffer.byteLength >= DEFLATE_MIN_SIZE) {
column.encoder = {buffer: pako.deflateRaw(column.encoder.buffer)}
column.columnId |= COLUMN_TYPE_DEFLATE
}
}
/**
* Decompresses the given column if it is DEFLATE-compressed.
*/
function inflateColumn(column) {
if ((column.columnId & COLUMN_TYPE_DEFLATE) !== 0) {
column.buffer = pako.inflateRaw(column.buffer)
column.columnId ^= COLUMN_TYPE_DEFLATE
}
}
module.exports = {
COLUMN_TYPE, VALUE_TYPE, ACTIONS, OBJECT_TYPE, DOC_OPS_COLUMNS, CHANGE_COLUMNS, DOCUMENT_COLUMNS,
encoderByColumnId, decoderByColumnId, makeDecoders, decodeValue,
splitContainers, encodeChange, decodeChangeColumns, decodeChange, decodeChangeMeta, decodeChanges,
encodeDocumentHeader, decodeDocumentHeader, decodeDocument
}
================================================
FILE: backend/encoding.js
================================================
/**
* UTF-8 decoding and encoding using API that is supported in Node >= 12 and modern browsers:
* https://developer.mozilla.org/en-US/docs/Web/API/TextEncoder/encode
* https://developer.mozilla.org/en-US/docs/Web/API/TextDecoder/decode
* If you're running in an environment where it's not available, please use a polyfill, such as:
* https://github.com/anonyco/FastestSmallestTextEncoderDecoder
*/
const utf8encoder = new TextEncoder()
const utf8decoder = new TextDecoder('utf-8')
function stringToUtf8(string) {
return utf8encoder.encode(string)
}
function utf8ToString(buffer) {
return utf8decoder.decode(buffer)
}
/**
* Converts a string consisting of hexadecimal digits into an Uint8Array.
*/
function hexStringToBytes(value) {
if (typeof value !== 'string') {
throw new TypeError('value is not a string')
}
if (!/^([0-9a-f][0-9a-f])*$/.test(value)) {
throw new RangeError('value is not hexadecimal')
}
if (value === '') {
return new Uint8Array(0)
} else {
return new Uint8Array(value.match(/../g).map(b => parseInt(b, 16)))
}
}
const NIBBLE_TO_HEX = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f']
const BYTE_TO_HEX = new Array(256)
for (let i = 0; i < 256; i++) {
BYTE_TO_HEX[i] = `${NIBBLE_TO_HEX[(i >>> 4) & 0xf]}${NIBBLE_TO_HEX[i & 0xf]}`;
}
/**
* Converts a Uint8Array into the equivalent hexadecimal string.
*/
function bytesToHexString(bytes) {
let hex = '', len = bytes.byteLength
for (let i = 0; i < len; i++) {
hex += BYTE_TO_HEX[bytes[i]]
}
return hex
}
/**
* Wrapper around an Uint8Array that allows values to be appended to the buffer,
* and that automatically grows the buffer when space runs out.
*/
class Encoder {
constructor() {
this.buf = new Uint8Array(16)
this.offset = 0
}
/**
* Returns the byte array containing the encoded data.
*/
get buffer() {
this.finish()
return this.buf.subarray(0, this.offset)
}
/**
* Reallocates the encoder's buffer to be bigger.
*/
grow(minSize = 0) {
let newSize = this.buf.byteLength * 4
while (newSize < minSize) newSize *= 2
const newBuf = new Uint8Array(newSize)
newBuf.set(this.buf, 0)
this.buf = newBuf
return this
}
/**
* Appends one byte (0 to 255) to the buffer.
*/
appendByte(value) {
if (this.offset >= this.buf.byteLength) this.grow()
this.buf[this.offset] = value
this.offset += 1
}
/**
* Encodes a 32-bit nonnegative integer in a variable number of bytes using
* the LEB128 encoding scheme (https://en.wikipedia.org/wiki/LEB128) and
* appends it to the buffer. Returns the number of bytes written.
*/
appendUint32(value) {
if (!Number.isInteger(value)) throw new RangeError('value is not an integer')
if (value < 0 || value > 0xffffffff) throw new RangeError('number out of range')
const numBytes = Math.max(1, Math.ceil((32 - Math.clz32(value)) / 7))
if (this.offset + numBytes > this.buf.byteLength) this.grow()
for (let i = 0; i < numBytes; i++) {
this.buf[this.offset + i] = (value & 0x7f) | (i === numBytes - 1 ? 0x00 : 0x80)
value >>>= 7 // zero-filling right shift
}
this.offset += numBytes
return numBytes
}
/**
* Encodes a 32-bit signed integer in a variable number of bytes using the
* LEB128 encoding scheme (https://en.wikipedia.org/wiki/LEB128) and appends
* it to the buffer. Returns the number of bytes written.
*/
appendInt32(value) {
if (!Number.isInteger(value)) throw new RangeError('value is not an integer')
if (value < -0x80000000 || value > 0x7fffffff) throw new RangeError('number out of range')
const numBytes = Math.ceil((33 - Math.clz32(value >= 0 ? value : -value - 1)) / 7)
if (this.offset + numBytes > this.buf.byteLength) this.grow()
for (let i = 0; i < numBytes; i++) {
this.buf[this.offset + i] = (value & 0x7f) | (i === numBytes - 1 ? 0x00 : 0x80)
value >>= 7 // sign-propagating right shift
}
this.offset += numBytes
return numBytes
}
/**
* Encodes a nonnegative integer in a variable number of bytes using the LEB128
* encoding scheme, up to the maximum size of integers supported by JavaScript
* (53 bits).
*/
appendUint53(value) {
if (!Number.isInteger(value)) throw new RangeError('value is not an integer')
if (value < 0 || value > Number.MAX_SAFE_INTEGER) {
throw new RangeError('number out of range')
}
const high32 = Math.floor(value / 0x100000000)
const low32 = (value & 0xffffffff) >>> 0 // right shift to interpret as unsigned
return this.appendUint64(high32, low32)
}
/**
* Encodes a signed integer in a variable number of bytes using the LEB128
* encoding scheme, up to the maximum size of integers supported by JavaScript
* (53 bits).
*/
appendInt53(value) {
if (!Number.isInteger(value)) throw new RangeError('value is not an integer')
if (value < Number.MIN_SAFE_INTEGER || value > Number.MAX_SAFE_INTEGER) {
throw new RangeError('number out of range')
}
const high32 = Math.floor(value / 0x100000000)
const low32 = (value & 0xffffffff) >>> 0 // right shift to interpret as unsigned
return this.appendInt64(high32, low32)
}
/**
* Encodes a 64-bit nonnegative integer in a variable number of bytes using
* the LEB128 encoding scheme, and appends it to the buffer. The number is
* given as two 32-bit halves since JavaScript cannot accurately represent
* integers with more than 53 bits in a single variable.
*/
appendUint64(high32, low32) {
if (!Number.isInteger(high32) || !Number.isInteger(low32)) {
throw new RangeError('value is not an integer')
}
if (high32 < 0 || high32 > 0xffffffff || low32 < 0 || low32 > 0xffffffff) {
throw new RangeError('number out of range')
}
if (high32 === 0) return this.appendUint32(low32)
const numBytes = Math.ceil((64 - Math.clz32(high32)) / 7)
if (this.offset + numBytes > this.buf.byteLength) this.grow()
for (let i = 0; i < 4; i++) {
this.buf[this.offset + i] = (low32 & 0x7f) | 0x80
low32 >>>= 7 // zero-filling right shift
}
this.buf[this.offset + 4] = (low32 & 0x0f) | ((high32 & 0x07) << 4) | (numBytes === 5 ? 0x00 : 0x80)
high32 >>>= 3
for (let i = 5; i < numBytes; i++) {
this.buf[this.offset + i] = (high32 & 0x7f) | (i === numBytes - 1 ? 0x00 : 0x80)
high32 >>>= 7
}
this.offset += numBytes
return numBytes
}
/**
* Encodes a 64-bit signed integer in a variable number of bytes using the
* LEB128 encoding scheme, and appends it to the buffer. The number is given
* as two 32-bit halves since JavaScript cannot accurately represent integers
* with more than 53 bits in a single variable. The sign of the 64-bit
* number is determined by the sign of the `high32` half; the sign of the
* `low32` half is ignored.
*/
appendInt64(high32, low32) {
if (!Number.isInteger(high32) || !Number.isInteger(low32)) {
throw new RangeError('value is not an integer')
}
if (high32 < -0x80000000 || high32 > 0x7fffffff || low32 < -0x80000000 || low32 > 0xffffffff) {
throw new RangeError('number out of range')
}
low32 >>>= 0 // interpret as unsigned
if (high32 === 0 && low32 <= 0x7fffffff) return this.appendInt32(low32)
if (high32 === -1 && low32 >= 0x80000000) return this.appendInt32(low32 - 0x100000000)
const numBytes = Math.ceil((65 - Math.clz32(high32 >= 0 ? high32 : -high32 - 1)) / 7)
if (this.offset + numBytes > this.buf.byteLength) this.grow()
for (let i = 0; i < 4; i++) {
this.buf[this.offset + i] = (low32 & 0x7f) | 0x80
low32 >>>= 7 // zero-filling right shift
}
this.buf[this.offset + 4] = (low32 & 0x0f) | ((high32 & 0x07) << 4) | (numBytes === 5 ? 0x00 : 0x80)
high32 >>= 3 // sign-propagating right shift
for (let i = 5; i < numBytes; i++) {
this.buf[this.offset + i] = (high32 & 0x7f) | (i === numBytes - 1 ? 0x00 : 0x80)
high32 >>= 7
}
this.offset += numBytes
return numBytes
}
/**
* Appends the contents of byte buffer `data` to the buffer. Returns the
* number of bytes appended.
*/
appendRawBytes(data) {
if (this.offset + data.byteLength > this.buf.byteLength) {
this.grow(this.offset + data.byteLength)
}
this.buf.set(data, this.offset)
this.offset += data.byteLength
return data.byteLength
}
/**
* Appends a UTF-8 string to the buffer, without any metadata. Returns the
* number of bytes appended.
*/
appendRawString(value) {
if (typeof value !== 'string') throw new TypeError('value is not a string')
return this.appendRawBytes(stringToUtf8(value))
}
/**
* Appends the contents of byte buffer `data` to the buffer, prefixed with the
* number of bytes in the buffer (as a LEB128-encoded unsigned integer).
*/
appendPrefixedBytes(data) {
this.appendUint53(data.byteLength)
this.appendRawBytes(data)
return this
}
/**
* Appends a UTF-8 string to the buffer, prefixed with its length in bytes
* (where the length is encoded as an unsigned LEB128 integer).
*/
appendPrefixedString(value) {
if (typeof value !== 'string') throw new TypeError('value is not a string')
this.appendPrefixedBytes(stringToUtf8(value))
return this
}
/**
* Takes a value, which must be a string consisting only of hexadecimal
* digits, maps it to a byte array, and appends it to the buffer, prefixed
* with its length in bytes.
*/
appendHexString(value) {
this.appendPrefixedBytes(hexStringToBytes(value))
return this
}
/**
* Flushes any unwritten data to the buffer. Call this before reading from
* the buffer constructed by this Encoder.
*/
finish() {
}
}
/**
* Counterpart to Encoder. Wraps a Uint8Array buffer with a cursor indicating
* the current decoding position, and allows values to be incrementally read by
* decoding the bytes at the current position.
*/
class Decoder {
constructor(buffer) {
if (!(buffer instanceof Uint8Array)) {
throw new TypeError(`Not a byte array: ${buffer}`)
}
this.buf = buffer
this.offset = 0
}
/**
* Returns false if there is still data to be read at the current decoding
* position, and true if we are at the end of the buffer.
*/
get done() {
return this.offset === this.buf.byteLength
}
/**
* Resets the cursor position, so that the next read goes back to the
* beginning of the buffer.
*/
reset() {
this.offset = 0
}
/**
* Moves the current decoding position forward by the specified number of
* bytes, without decoding anything.
*/
skip(bytes) {
if (this.offset + bytes > this.buf.byteLength) {
throw new RangeError('cannot skip beyond end of buffer')
}
this.offset += bytes
}
/**
* Reads one byte (0 to 255) from the buffer.
*/
readByte() {
this.offset += 1
return this.buf[this.offset - 1]
}
/**
* Reads a LEB128-encoded unsigned integer from the current position in the buffer.
* Throws an exception if the value doesn't fit in a 32-bit unsigned int.
*/
readUint32() {
let result = 0, shift = 0
while (this.offset < this.buf.byteLength) {
const nextByte = this.buf[this.offset]
if (shift === 28 && (nextByte & 0xf0) !== 0) { // more than 5 bytes, or value > 0xffffffff
throw new RangeError('number out of range')
}
result = (result | (nextByte & 0x7f) << shift) >>> 0 // right shift to interpret value as unsigned
shift += 7
this.offset++
if ((nextByte & 0x80) === 0) return result
}
throw new RangeError('buffer ended with incomplete number')
}
/**
* Reads a LEB128-encoded signed integer from the current position in the buffer.
* Throws an exception if the value doesn't fit in a 32-bit signed int.
*/
readInt32() {
let result = 0, shift = 0
while (this.offset < this.buf.byteLength) {
const nextByte = this.buf[this.offset]
if ((shift === 28 && (nextByte & 0x80) !== 0) || // more than 5 bytes
(shift === 28 && (nextByte & 0x40) === 0 && (nextByte & 0x38) !== 0) || // positive int > 0x7fffffff
(shift === 28 && (nextByte & 0x40) !== 0 && (nextByte & 0x38) !== 0x38)) { // negative int < -0x80000000
throw new RangeError('number out of range')
}
result |= (nextByte & 0x7f) << shift
shift += 7
this.offset++
if ((nextByte & 0x80) === 0) {
if ((nextByte & 0x40) === 0 || shift > 28) {
return result // positive, or negative value that doesn't need sign-extending
} else {
return result | (-1 << shift) // sign-extend negative integer
}
}
}
throw new RangeError('buffer ended with incomplete number')
}
/**
* Reads a LEB128-encoded unsigned integer from the current position in the
* buffer. Allows any integer that can be safely represented by JavaScript
* (up to 2^53 - 1), and throws an exception outside of that range.
*/
readUint53() {
const { low32, high32 } = this.readUint64()
if (high32 < 0 || high32 > 0x1fffff) {
throw new RangeError('number out of range')
}
return high32 * 0x100000000 + low32
}
/**
* Reads a LEB128-encoded signed integer from the current position in the
* buffer. Allows any integer that can be safely represented by JavaScript
* (between -(2^53 - 1) and 2^53 - 1), throws an exception outside of that range.
*/
readInt53() {
const { low32, high32 } = this.readInt64()
if (high32 < -0x200000 || (high32 === -0x200000 && low32 === 0) || high32 > 0x1fffff) {
throw new RangeError('number out of range')
}
return high32 * 0x100000000 + low32
}
/**
* Reads a LEB128-encoded unsigned integer from the current position in the
* buffer. Throws an exception if the value doesn't fit in a 64-bit unsigned
* int. Returns the number in two 32-bit halves, as an object of the form
* `{high32, low32}`.
*/
readUint64() {
let low32 = 0, high32 = 0, shift = 0
while (this.offset < this.buf.byteLength && shift <= 28) {
const nextByte = this.buf[this.offset]
low32 = (low32 | (nextByte & 0x7f) << shift) >>> 0 // right shift to interpret value as unsigned
if (shift === 28) {
high32 = (nextByte & 0x70) >>> 4
}
shift += 7
this.offset++
if ((nextByte & 0x80) === 0) return { high32, low32 }
}
shift = 3
while (this.offset < this.buf.byteLength) {
const nextByte = this.buf[this.offset]
if (shift === 31 && (nextByte & 0xfe) !== 0) { // more than 10 bytes, or value > 2^64 - 1
throw new RangeError('number out of range')
}
high32 = (high32 | (nextByte & 0x7f) << shift) >>> 0
shift += 7
this.offset++
if ((nextByte & 0x80) === 0) return { high32, low32 }
}
throw new RangeError('buffer ended with incomplete number')
}
/**
* Reads a LEB128-encoded signed integer from the current position in the
* buffer. Throws an exception if the value doesn't fit in a 64-bit signed
* int. Returns the number in two 32-bit halves, as an object of the form
* `{high32, low32}`. The `low32` half is always non-negative, and the
* sign of the `high32` half indicates the sign of the 64-bit number.
*/
readInt64() {
let low32 = 0, high32 = 0, shift = 0
while (this.offset < this.buf.byteLength && shift <= 28) {
const nextByte = this.buf[this.offset]
low32 = (low32 | (nextByte & 0x7f) << shift) >>> 0 // right shift to interpret value as unsigned
if (shift === 28) {
high32 = (nextByte & 0x70) >>> 4
}
shift += 7
this.offset++
if ((nextByte & 0x80) === 0) {
if ((nextByte & 0x40) !== 0) { // sign-extend negative integer
if (shift < 32) low32 = (low32 | (-1 << shift)) >>> 0
high32 |= -1 << Math.max(shift - 32, 0)
}
return { high32, low32 }
}
}
shift = 3
while (this.offset < this.buf.byteLength) {
const nextByte = this.buf[this.offset]
// On the 10th byte there are only two valid values: all 7 value bits zero
// (if the value is positive) or all 7 bits one (if the value is negative)
if (shift === 31 && nextByte !== 0 && nextByte !== 0x7f) {
throw new RangeError('number out of range')
}
high32 |= (nextByte & 0x7f) << shift
shift += 7
this.offset++
if ((nextByte & 0x80) === 0) {
if ((nextByte & 0x40) !== 0 && shift < 32) { // sign-extend negative integer
high32 |= -1 << shift
}
return { high32, low32 }
}
}
throw new RangeError('buffer ended with incomplete number')
}
/**
* Extracts a subarray `length` bytes in size, starting from the current
* position in the buffer, and moves the position forward.
*/
readRawBytes(length) {
const start = this.offset
if (start + length > this.buf.byteLength) {
throw new RangeError('subarray exceeds buffer size')
}
this.offset += length
return this.buf.subarray(start, this.offset)
}
/**
* Extracts `length` bytes from the buffer, starting from the current position,
* and returns the UTF-8 string decoding of those bytes.
*/
readRawString(length) {
return utf8ToString(this.readRawBytes(length))
}
/**
* Extracts a subarray from the current position in the buffer, prefixed with
* its length in bytes (encoded as an unsigned LEB128 integer).
*/
readPrefixedBytes() {
return this.readRawBytes(this.readUint53())
}
/**
* Reads a UTF-8 string from the current position in the buffer, prefixed with its
* length in bytes (where the length is encoded as an unsigned LEB128 integer).
*/
readPrefixedString() {
return utf8ToString(this.readPrefixedBytes())
}
/**
* Reads a byte array from the current position in the buffer, prefixed with its
* length in bytes. Returns that byte array converted to a hexadecimal string.
*/
readHexString() {
return bytesToHexString(this.readPrefixedBytes())
}
}
/**
* An encoder that uses run-length encoding to compress sequences of repeated
* values. The constructor argument specifies the type of values, which may be
* either 'int', 'uint', or 'utf8'. Besides valid values of the selected
* datatype, values may also be null.
*
* The encoded buffer starts with a LEB128-encoded signed integer, the
* repetition count. The interpretation of the following values depends on this
* repetition count:
* - If this number is a positive value n, the next value in the buffer
* (encoded as the specified datatype) is repeated n times in the sequence.
* - If the repetition count is a negative value -n, then the next n values
* (encoded as the specified datatype) in the buffer are treated as a
* literal, i.e. they appear in the sequence without any further
* interpretation or repetition.
* - If the repetition count is zero, then the next value in the buffer is a
* LEB128-encoded unsigned integer indicating the number of null values
* that appear at the current position in the sequence.
*
* After one of these three has completed, the process repeats, starting again
* with a repetition count, until we reach the end of the buffer.
*/
class RLEEncoder extends Encoder {
constructor(type) {
super()
this.type = type
this.state = 'empty'
this.lastValue = undefined
this.count = 0
this.literal = []
}
/**
* Appends a new value to the sequence. If `repetitions` is given, the value is repeated
* `repetitions` times.
*/
appendValue(value, repetitions = 1) {
this._appendValue(value, repetitions)
}
/**
* Like `appendValue()`, but this method is not overridden by `DeltaEncoder`.
*/
_appendValue(value, repetitions = 1) {
if (repetitions <= 0) return
if (this.state === 'empty') {
this.state = (value === null ? 'nulls' : (repetitions === 1 ? 'loneValue' : 'repetition'))
this.lastValue = value
this.count = repetitions
} else if (this.state === 'loneValue') {
if (value === null) {
this.flush()
this.state = 'nulls'
this.count = repetitions
} else if (value === this.lastValue) {
this.state = 'repetition'
this.count = 1 + repetitions
} else if (repetitions > 1) {
this.flush()
this.state = 'repetition'
this.count = repetitions
this.lastValue = value
} else {
this.state = 'literal'
this.literal = [this.lastValue]
this.lastValue = value
}
} else if (this.state === 'repetition') {
if (value === null) {
this.flush()
this.state = 'nulls'
this.count = repetitions
} else if (value === this.lastValue) {
this.count += repetitions
} else if (repetitions > 1) {
this.flush()
this.state = 'repetition'
this.count = repetitions
this.lastValue = value
} else {
this.flush()
this.state = 'loneValue'
this.lastValue = value
}
} else if (this.state === 'literal') {
if (value === null) {
this.literal.push(this.lastValue)
this.flush()
this.state = 'nulls'
this.count = repetitions
} else if (value === this.lastValue) {
this.flush()
this.state = 'repetition'
this.count = 1 + repetitions
} else if (repetitions > 1) {
this.literal.push(this.lastValue)
this.flush()
this.state = 'repetition'
this.count = repetitions
this.lastValue = value
} else {
this.literal.push(this.lastValue)
this.lastValue = value
}
} else if (this.state === 'nulls') {
if (value === null) {
this.count += repetitions
} else if (repetitions > 1) {
this.flush()
this.state = 'repetition'
this.count = repetitions
this.lastValue = value
} else {
this.flush()
this.state = 'loneValue'
this.lastValue = value
}
}
}
/**
* Copies values from the RLEDecoder `decoder` into this encoder. The `options` object may
* contain the following keys:
* - `count`: The number of values to copy. If not specified, copies all remaining values.
* - `sumValues`: If true, the function computes the sum of all numeric values as they are
* copied (null values are counted as zero), and returns that number.
* - `sumShift`: If set, values are shifted right by `sumShift` bits before adding to the sum.
*
* Returns an object of the form `{nonNullValues, sum}` where `nonNullValues` is the number of
* non-null values copied, and `sum` is the sum (only if the `sumValues` option is set).
*/
copyFrom(decoder, options = {}) {
const { count, sumValues, sumShift } = options
if (!(decoder instanceof RLEDecoder) || (decoder.type !== this.type)) {
throw new TypeError('incompatible type of decoder')
}
let remaining = (typeof count === 'number' ? count : Number.MAX_SAFE_INTEGER)
let nonNullValues = 0, sum = 0
if (count && remaining > 0 && decoder.done) throw new RangeError(`cannot copy ${count} values`)
if (remaining === 0 || decoder.done) return sumValues ? {nonNullValues, sum} : {nonNullValues}
// Copy a value so that we have a well-defined starting state. NB: when super.copyFrom() is
// called by the DeltaEncoder subclass, the following calls to readValue() and appendValue()
// refer to the overridden methods, while later readRecord(), readRawValue() and _appendValue()
// calls refer to the non-overridden RLEDecoder/RLEEncoder methods.
let firstValue = decoder.readValue()
if (firstValue === null) {
const numNulls = Math.min(decoder.count + 1, remaining)
remaining -= numNulls
decoder.count -= numNulls - 1
this.appendValue(null, numNulls)
if (count && remaining > 0 && decoder.done) throw new RangeError(`cannot copy ${count} values`)
if (remaining === 0 || decoder.done) return sumValues ? {nonNullValues, sum} : {nonNullValues}
firstValue = decoder.readValue()
if (firstValue === null) throw new RangeError('null run must be followed by non-null value')
}
this.appendValue(firstValue)
remaining--
nonNullValues++
if (sumValues) sum += (sumShift ? (firstValue >>> sumShift) : firstValue)
if (count && remaining > 0 && decoder.done) throw new RangeError(`cannot copy ${count} values`)
if (remaining === 0 || decoder.done) return sumValues ? {nonNullValues, sum} : {nonNullValues}
// Copy data at the record level without expanding repetitions
let firstRun = (decoder.count > 0)
while (remaining > 0 && !decoder.done) {
if (!firstRun) decoder.readRecord()
const numValues = Math.min(decoder.count, remaining)
decoder.count -= numValues
if (decoder.state === 'literal') {
nonNullValues += numValues
for (let i = 0; i < numValues; i++) {
if (decoder.done) throw new RangeError('incomplete literal')
const value = decoder.readRawValue()
if (value === decoder.lastValue) throw new RangeError('Repetition of values is not allowed in literal')
decoder.lastValue = value
this._appendValue(value)
if (sumValues) sum += (sumShift ? (value >>> sumShift) : value)
}
} else if (decoder.state === 'repetition') {
nonNullValues += numValues
if (sumValues) sum += numValues * (sumShift ? (decoder.lastValue >>> sumShift) : decoder.lastValue)
const value = decoder.lastValue
this._appendValue(value)
if (numValues > 1) {
this._appendValue(value)
if (this.state !== 'repetition') throw new RangeError(`Unexpected state ${this.state}`)
this.count += numValues - 2
}
} else if (decoder.state === 'nulls') {
this._appendValue(null)
if (this.state !== 'nulls') throw new RangeError(`Unexpected state ${this.state}`)
this.count += numValues - 1
}
firstRun = false
remaining -= numValues
}
if (count && remaining > 0 && decoder.done) throw new RangeError(`cannot copy ${count} values`)
return sumValues ? {nonNullValues, sum} : {nonNullValues}
}
/**
* Private method, do not call from outside the class.
*/
flush() {
if (this.state === 'loneValue') {
this.appendInt32(-1)
this.appendRawValue(this.lastValue)
} else if (this.state === 'repetition') {
this.appendInt53(this.count)
this.appendRawValue(this.lastValue)
} else if (this.state === 'literal') {
this.appendInt53(-this.literal.length)
for (let v of this.literal) this.appendRawValue(v)
} else if (this.state === 'nulls') {
this.appendInt32(0)
this.appendUint53(this.count)
}
this.state = 'empty'
}
/**
* Private method, do not call from outside the class.
*/
appendRawValue(value) {
if (this.type === 'int') {
this.appendInt53(value)
} else if (this.type === 'uint') {
this.appendUint53(value)
} else if (this.type === 'utf8') {
this.appendPrefixedString(value)
} else {
throw new RangeError(`Unknown RLEEncoder datatype: ${this.type}`)
}
}
/**
* Flushes any unwritten data to the buffer. Call this before reading from
* the buffer constructed by this Encoder.
*/
finish() {
if (this.state === 'literal') this.literal.push(this.lastValue)
// Don't write anything if the only values we have seen are nulls
if (this.state !== 'nulls' || this.offset > 0) this.flush()
}
}
/**
* Counterpart to RLEEncoder: reads values from an RLE-compressed sequence,
* returning nulls and repeated values as required.
*/
class RLEDecoder extends Decoder {
constructor(type, buffer) {
super(buffer)
this.type = type
this.lastValue = undefined
this.count = 0
this.state = undefined
}
/**
* Returns false if there is still data to be read at the current decoding
* position, and true if we are at the end of the buffer.
*/
get done() {
return (this.count === 0) && (this.offset === this.buf.byteLength)
}
/**
* Resets the cursor position, so that the next read goes back to the
* beginning of the buffer.
*/
reset() {
this.offset = 0
this.lastValue = undefined
this.count = 0
this.state = undefined
}
/**
* Returns the next value (or null) in the sequence.
*/
readValue() {
if (this.done) return null
if (this.count === 0) this.readRecord()
this.count -= 1
if (this.state === 'literal') {
const value = this.readRawValue()
if (value === this.lastValue) throw new RangeError('Repetition of values is not allowed in literal')
this.lastValue = value
return value
} else {
return this.lastValue
}
}
/**
* Discards the next `numSkip` values in the sequence.
*/
skipValues(numSkip) {
while (numSkip > 0 && !this.done) {
if (this.count === 0) {
this.count = this.readInt53()
if (this.count > 0) {
this.lastValue = (this.count <= numSkip) ? this.skipRawValues(1) : this.readRawValue()
this.state = 'repetition'
} else if (this.count < 0) {
this.count = -this.count
this.state = 'literal'
} else { // this.count == 0
this.count = this.readUint53()
this.lastValue = null
this.state = 'nulls'
}
}
const consume = Math.min(numSkip, this.count)
if (this.state === 'literal') this.skipRawValues(consume)
numSkip -= consume
this.count -= consume
}
}
/**
* Private method, do not call from outside the class.
* Reads a repetition count from the buffer and sets up the state appropriately.
*/
readRecord() {
this.count = this.readInt53()
if (this.count > 1) {
const value = this.readRawValue()
if ((this.state === 'repetition' || this.state === 'literal') && this.lastValue === value) {
throw new RangeError('Successive repetitions with the same value are not allowed')
}
this.state = 'repetition'
this.lastValue = value
} else if (this.count === 1) {
throw new RangeError('Repetition count of 1 is not allowed, use a literal instead')
} else if (this.count < 0) {
this.count = -this.count
if (this.state === 'literal') throw new RangeError('Successive literals are not allowed')
this.state = 'literal'
} else { // this.count == 0
if (this.state === 'nulls') throw new RangeError('Successive null runs are not allowed')
this.count = this.readUint53()
if (this.count === 0) throw new RangeError('Zero-length null runs are not allowed')
this.lastValue = null
this.state = 'nulls'
}
}
/**
* Private method, do not call from outside the class.
* Reads one value of the datatype configured on construction.
*/
readRawValue() {
if (this.type === 'int') {
return this.readInt53()
} else if (this.type === 'uint') {
return this.readUint53()
} else if (this.type === 'utf8') {
return this.readPrefixedString()
} else {
throw new RangeError(`Unknown RLEDecoder datatype: ${this.type}`)
}
}
/**
* Private method, do not call from outside the class.
* Skips over `num` values of the datatype configured on construction.
*/
skipRawValues(num) {
if (this.type === 'utf8') {
for (let i = 0; i < num; i++) this.skip(this.readUint53())
} else {
while (num > 0 && this.offset < this.buf.byteLength) {
if ((this.buf[this.offset] & 0x80) === 0) num--
this.offset++
}
if (num > 0) throw new RangeError('cannot skip beyond end of buffer')
}
}
}
/**
* A variant of RLEEncoder: rather than storing the actual values passed to
* appendValue(), this version stores only the first value, and for all
* subsequent values it stores the difference to the previous value. This
* encoding is good when values tend to come in sequentially incrementing runs,
* because the delta between successive values is 1, and repeated values of 1
* are easily compressed with run-length encoding.
*
* Null values are also allowed, as with RLEEncoder.
*/
class DeltaEncoder extends RLEEncoder {
constructor() {
super('int')
this.absoluteValue = 0
}
/**
* Appends a new integer value to the sequence. If `repetitions` is given, the value is repeated
* `repetitions` times.
*/
appendValue(value, repetitions = 1) {
if (repetitions <= 0) return
if (typeof value === 'number') {
super.appendValue(value - this.absoluteValue, 1)
this.absoluteValue = value
if (repetitions > 1) super.appendValue(0, repetitions - 1)
} else {
super.appendValue(value, repetitions)
}
}
/**
* Copies values from the DeltaDecoder `decoder` into this encoder. The `options` object may
* contain the key `count`, indicating the number of values to copy. If not specified, copies
* all remaining values in the decoder.
*/
copyFrom(decoder, options = {}) {
if (options.sumValues) {
throw new RangeError('unsupported options for DeltaEncoder.copyFrom()')
}
if (!(decoder instanceof DeltaDecoder)) {
throw new TypeError('incompatible type of decoder')
}
let remaining = options.count
if (remaining > 0 && decoder.done) throw new RangeError(`cannot copy ${remaining} values`)
if (remaining === 0 || decoder.done) return
// Copy any null values, and the first non-null value, so that appendValue() computes the
// difference between the encoder's last value and the decoder's first (absolute) value.
let value = decoder.readValue(), nulls = 0
this.appendValue(value)
if (value === null) {
nulls = decoder.count + 1
if (remaining !== undefined && remaining < nulls) nulls = remaining
decoder.count -= nulls - 1
this.count += nulls - 1
if (remaining > nulls && decoder.done) throw new RangeError(`cannot copy ${remaining} values`)
if (remaining === nulls || decoder.done) return
// The next value read is certain to be non-null because we're not at the end of the decoder,
// and a run of nulls must be followed by a run of non-nulls.
if (decoder.count === 0) this.appendValue(decoder.readValue())
}
// Once we have the first value, the subsequent relative values can be copied verbatim without
// any further processing. Note that the first value copied by super.copyFrom() is an absolute
// value, while subsequent values are relative. Thus, the sum of all of the (non-null) copied
// values must equal the absolute value of the final element copied.
if (remaining !== undefined) remaining -= nulls + 1
const { nonNullValues, sum } = super.copyFrom(decoder, {count: remaining, sumValues: true})
if (nonNullValues > 0) {
this.absoluteValue = sum
decoder.absoluteValue = sum
}
}
}
/**
* Counterpart to DeltaEncoder: reads values from a delta-compressed sequence of
* numbers (may include null values).
*/
class DeltaDecoder extends RLEDecoder {
constructor(buffer) {
super('int', buffer)
this.absoluteValue = 0
}
/**
* Resets the cursor position, so that the next read goes back to the
* beginning of the buffer.
*/
reset() {
this.offset = 0
this.lastValue = undefined
this.count = 0
this.state = undefined
this.absoluteValue = 0
}
/**
* Returns the next integer (or null) value in the sequence.
*/
readValue() {
const value = super.readValue()
if (value === null) return null
this.absoluteValue += value
return this.absoluteValue
}
/**
* Discards the next `numSkip` values in the sequence.
*/
skipValues(numSkip) {
while (numSkip > 0 && !this.done) {
if (this.count === 0) this.readRecord()
const consume = Math.min(numSkip, this.count)
if (this.state === 'literal') {
for (let i = 0; i < consume; i++) {
this.lastValue = this.readRawValue()
this.absoluteValue += this.lastValue
}
} else if (this.state === 'repetition') {
this.absoluteValue += consume * this.lastValue
}
numSkip -= consume
this.count -= consume
}
}
}
/**
* Encodes a sequence of boolean values by mapping it to a sequence of integers:
* the number of false values, followed by the number of true values, followed
* by the number of false values, and so on. Each number is encoded as a LEB128
* unsigned integer. This encoding is a bit like RLEEncoder, except that we
* only encode the repetition count but not the actual value, since the values
* just alternate between false and true (starting with false).
*/
class BooleanEncoder extends Encoder {
constructor() {
super()
this.lastValue = false
this.count = 0
}
/**
* Appends a new value to the sequence. If `repetitions` is given, the value is repeated
* `repetitions` times.
*/
appendValue(value, repetitions = 1) {
if (value !== false && value !== true) {
throw new RangeError(`Unsupported value for BooleanEncoder: ${value}`)
}
if (repetitions <= 0) return
if (this.lastValue === value) {
this.count += repetitions
} else {
this.appendUint53(this.count)
this.lastValue = value
this.count = repetitions
}
}
/**
* Copies values from the BooleanDecoder `decoder` into this encoder. The `options` object may
* contain the key `count`, indicating the number of values to copy. If not specified, copies
* all remaining values in the decoder.
*/
copyFrom(decoder, options = {}) {
if (!(decoder instanceof BooleanDecoder)) {
throw new TypeError('incompatible type of decoder')
}
const { count } = options
let remaining = (typeof count === 'number' ? count : Number.MAX_SAFE_INTEGER)
if (count && remaining > 0 && decoder.done) throw new RangeError(`cannot copy ${count} values`)
if (remaining === 0 || decoder.done) return
// Copy one value to bring decoder and encoder state into sync, then finish that value's repetitions
this.appendValue(decoder.readValue())
remaining--
const firstCopy = Math.min(decoder.count, remaining)
this.count += firstCopy
decoder.count -= firstCopy
remaining -= firstCopy
while (remaining > 0 && !decoder.done) {
decoder.count = decoder.readUint53()
if (decoder.count === 0) throw new RangeError('Zero-length runs are not allowed')
decoder.lastValue = !decoder.lastValue
this.appendUint53(this.count)
const numCopied = Math.min(decoder.count, remaining)
this.count = numCopied
this.lastValue = decoder.lastValue
decoder.count -= numCopied
remaining -= numCopied
}
if (count && remaining > 0 && decoder.done) throw new RangeError(`cannot copy ${count} values`)
}
/**
* Flushes any unwritten data to the buffer. Call this before reading from
* the buffer constructed by this Encoder.
*/
finish() {
if (this.count > 0) {
this.appendUint53(this.count)
this.count = 0
}
}
}
/**
* Counterpart to BooleanEncoder: reads boolean values from a runlength-encoded
* sequence.
*/
class BooleanDecoder extends Decoder {
constructor(buffer) {
super(buffer)
this.lastValue = true // is negated the first time we read a count
this.firstRun = true
this.count = 0
}
/**
* Returns false if there is still data to be read at the current decoding
* position, and true if we are at the end of the buffer.
*/
get done() {
return (this.count === 0) && (this.offset === this.buf.byteLength)
}
/**
* Resets the cursor position, so that the next read goes back to the
* beginning of the buffer.
*/
reset() {
this.offset = 0
this.lastValue = true
this.firstRun = true
this.count = 0
}
/**
* Returns the next value in the sequence.
*/
readValue() {
if (this.done) return false
while (this.count === 0) {
this.count = this.readUint53()
this.lastValue = !this.lastValue
if (this.count === 0 && !this.firstRun) {
throw new RangeError('Zero-length runs are not allowed')
}
this.firstRun = false
}
this.count -= 1
return this.lastValue
}
/**
* Discards the next `numSkip` values in the sequence.
*/
skipValues(numSkip) {
while (numSkip > 0 && !this.done) {
if (this.count === 0) {
this.count = this.readUint53()
this.lastValue = !this.lastValue
if (this.count === 0 && !this.firstRun) {
throw new RangeError('Zero-length runs are not allowed')
}
this.firstRun = false
}
if (this.count < numSkip) {
numSkip -= this.count
this.count = 0
} else {
this.count -= numSkip
numSkip = 0
}
}
}
}
module.exports = {
stringToUtf8, utf8ToString, hexStringToBytes, bytesToHexString,
Encoder, Decoder, RLEEncoder, RLEDecoder, DeltaEncoder, DeltaDecoder, BooleanEncoder, BooleanDecoder
}
================================================
FILE: backend/index.js
================================================
const { init, clone, free, applyChanges, applyLocalChange, save, load, loadChanges, getPatch, getHeads, getAllChanges, getChanges, getChangesAdded, getChangeByHash, getMissingDeps } = require("./backend")
const { receiveSyncMessage, generateSyncMessage, encodeSyncMessage, decodeSyncMessage, encodeSyncState, decodeSyncState, initSyncState } = require('./sync')
module.exports = {
init, clone, free, applyChanges, applyLocalChange, save, load, loadChanges, getPatch,
getHeads, getAllChanges, getChanges, getChangesAdded, getChangeByHash, getMissingDeps,
receiveSyncMessage, generateSyncMessage, encodeSyncMessage, decodeSyncMessage, encodeSyncState, decodeSyncState, initSyncState
}
================================================
FILE: backend/new.js
================================================
const { parseOpId, copyObject } = require('../src/common')
const { COLUMN_TYPE, VALUE_TYPE, ACTIONS, OBJECT_TYPE, DOC_OPS_COLUMNS, CHANGE_COLUMNS, DOCUMENT_COLUMNS,
encoderByColumnId, decoderByColumnId, makeDecoders, decodeValue,
encodeChange, decodeChangeColumns, decodeChangeMeta, decodeChanges, decodeDocumentHeader, encodeDocumentHeader } = require('./columnar')
const MAX_BLOCK_SIZE = 600 // operations
const BLOOM_BITS_PER_ENTRY = 10, BLOOM_NUM_PROBES = 7 // 1% false positive rate
const BLOOM_FILTER_SIZE = Math.floor(BLOOM_BITS_PER_ENTRY * MAX_BLOCK_SIZE / 8) // bytes
const objActorIdx = 0, objCtrIdx = 1, keyActorIdx = 2, keyCtrIdx = 3, keyStrIdx = 4,
idActorIdx = 5, idCtrIdx = 6, insertIdx = 7, actionIdx = 8, valLenIdx = 9, valRawIdx = 10,
predNumIdx = 13, predActorIdx = 14, predCtrIdx = 15, succNumIdx = 13, succActorIdx = 14, succCtrIdx = 15
const PRED_COLUMN_IDS = CHANGE_COLUMNS
.filter(column => ['predNum', 'predActor', 'predCtr'].includes(column.columnName))
.map(column => column.columnId)
/**
* Updates `objectTree`, which is a tree of nested objects, so that afterwards
* `objectTree[path[0]][path[1]][...] === value`. Only the root object is mutated, whereas any
* nested objects are copied before updating. This means that once the root object has been
* shallow-copied, this function can be used to update it without mutating the previous version.
*/
function deepCopyUpdate(objectTree, path, value) {
if (path.length === 1) {
objectTree[path[0]] = value
} else {
let child = Object.assign({}, objectTree[path[0]])
deepCopyUpdate(child, path.slice(1), value)
objectTree[path[0]] = child
}
}
/**
* Scans a block of document operations, encoded as columns `docCols`, to find the position at which
* an operation (or sequence of operations) `ops` should be applied. `actorIds` is the array that
* maps actor numbers to hexadecimal actor IDs. `resumeInsertion` is true if we're performing a list
* insertion and we already found the reference element in a previous block, but we reached the end
* of that previous block while scanning for the actual insertion position, and so we're continuing
* the scan in a subsequent block.
*
* Returns an object with keys:
* - `found`: false if we were scanning for a reference element in a list but couldn't find it;
* true otherwise.
* - `skipCount`: the number of operations, counted from the start of the block, after which the
* new operations should be inserted or applied.
* - `visibleCount`: if modifying a list object, the number of visible (i.e. non-deleted) list
* elements that precede the position where the new operations should be applied.
*/
function seekWithinBlock(ops, docCols, actorIds, resumeInsertion) {
for (let col of docCols) col.decoder.reset()
const { objActor, objCtr, keyActor, keyCtr, keyStr, idActor, idCtr, insert } = ops
const [objActorD, objCtrD, /* keyActorD */, /* keyCtrD */, keyStrD, idActorD, idCtrD, insertD, actionD,
/* valLenD */, /* valRawD */, /* chldActorD */, /* chldCtrD */, succNumD] = docCols.map(col => col.decoder)
let skipCount = 0, visibleCount = 0, elemVisible = false, nextObjActor = null, nextObjCtr = null
let nextIdActor = null, nextIdCtr = null, nextKeyStr = null, nextInsert = null, nextSuccNum = 0
// Seek to the beginning of the object being updated
if (objCtr !== null && !resumeInsertion) {
while (!objCtrD.done || !objActorD.done || !actionD.done) {
nextObjCtr = objCtrD.readValue()
nextObjActor = actorIds[objActorD.readValue()]
actionD.skipValues(1)
if (nextObjCtr === null || !nextObjActor || nextObjCtr < objCtr ||
(nextObjCtr === objCtr && nextObjActor < objActor)) {
skipCount += 1
} else {
break
}
}
}
if ((nextObjCtr !== objCtr || nextObjActor !== objActor) && !resumeInsertion) {
return {found: true, skipCount, visibleCount}
}
// Seek to the appropriate key (if string key is used)
if (keyStr !== null) {
keyStrD.skipValues(skipCount)
while (!keyStrD.done) {
const objActorIndex = objActorD.readValue()
nextObjActor = objActorIndex === null ? null : actorIds[objActorIndex]
nextObjCtr = objCtrD.readValue()
nextKeyStr = keyStrD.readValue()
if (nextKeyStr !== null && nextKeyStr < keyStr &&
nextObjCtr === objCtr && nextObjActor === objActor) {
skipCount += 1
} else {
break
}
}
return {found: true, skipCount, visibleCount}
}
idCtrD.skipValues(skipCount)
idActorD.skipValues(skipCount)
insertD.skipValues(skipCount)
succNumD.skipValues(skipCount)
nextIdCtr = idCtrD.readValue()
nextIdActor = actorIds[idActorD.readValue()]
nextInsert = insertD.readValue()
nextSuccNum = succNumD.readValue()
// If we are inserting into a list, an opId key is used, and we need to seek to a position *after*
// the referenced operation. Moreover, we need to skip over any existing operations with a greater
// opId than the new insertion, for CRDT convergence on concurrent insertions in the same place.
if (insert) {
// If insertion is not at the head, search for the reference element
if (!resumeInsertion && keyCtr !== null && keyCtr > 0 && keyActor !== null) {
skipCount += 1
while (!idCtrD.done && !idActorD.done && (nextIdCtr !== keyCtr || nextIdActor !== keyActor)) {
if (nextInsert) elemVisible = false
if (nextSuccNum === 0 && !elemVisible) {
visibleCount += 1
elemVisible = true
}
nextIdCtr = idCtrD.readValue()
nextIdActor = actorIds[idActorD.readValue()]
nextObjCtr = objCtrD.readValue()
nextObjActor = actorIds[objActorD.readValue()]
nextInsert = insertD.readValue()
nextSuccNum = succNumD.readValue()
if (nextObjCtr === objCtr && nextObjActor === objActor) skipCount += 1; else break
}
if (nextObjCtr !== objCtr || nextObjActor !== objActor || nextIdCtr !== keyCtr ||
nextIdActor !== keyActor || !nextInsert) {
return {found: false, skipCount, visibleCount}
}
if (nextInsert) elemVisible = false
if (nextSuccNum === 0 && !elemVisible) {
visibleCount += 1
elemVisible = true
}
// Set up the next* variables to the operation following the reference element
if (idCtrD.done || idActorD.done) return {found: true, skipCount, visibleCount}
nextIdCtr = idCtrD.readValue()
nextIdActor = actorIds[idActorD.readValue()]
nextObjCtr = objCtrD.readValue()
nextObjActor = actorIds[objActorD.readValue()]
nextInsert = insertD.readValue()
nextSuccNum = succNumD.readValue()
}
// Skip over any list elements with greater ID than the new one, and any non-insertions
while ((!nextInsert || nextIdCtr > idCtr || (nextIdCtr === idCtr && nextIdActor > idActor)) &&
nextObjCtr === objCtr && nextObjActor === objActor) {
skipCount += 1
if (nextInsert) elemVisible = false
if (nextSuccNum === 0 && !elemVisible) {
visibleCount += 1
elemVisible = true
}
if (!idCtrD.done && !idActorD.done) {
nextIdCtr = idCtrD.readValue()
nextIdActor = actorIds[idActorD.readValue()]
nextObjCtr = objCtrD.readValue()
nextObjActor = actorIds[objActorD.readValue()]
nextInsert = insertD.readValue()
nextSuccNum = succNumD.readValue()
} else {
break
}
}
} else if (keyCtr !== null && keyCtr > 0 && keyActor !== null) {
// If we are updating an existing list element, seek to just before the referenced ID
while ((!nextInsert || nextIdCtr !== keyCtr || nextIdActor !== keyActor) &&
nextObjCtr === objCtr && nextObjActor === objActor) {
skipCount += 1
if (nextInsert) elemVisible = false
if (nextSuccNum === 0 && !elemVisible) {
visibleCount += 1
elemVisible = true
}
if (!idCtrD.done && !idActorD.done) {
nextIdCtr = idCtrD.readValue()
nextIdActor = actorIds[idActorD.readValue()]
nextObjCtr = objCtrD.readValue()
nextObjActor = actorIds[objActorD.readValue()]
nextInsert = insertD.readValue()
nextSuccNum = succNumD.readValue()
} else {
break
}
}
if (nextObjCtr !== objCtr || nextObjActor !== objActor || nextIdCtr !== keyCtr ||
nextIdActor !== keyActor || !nextInsert) {
return {found: false, skipCount, visibleCount}
}
}
return {found: true, skipCount, visibleCount}
}
/**
* Returns the number of list elements that should be added to a list index when skipping over the
* block with index `blockIndex` in the list object with object ID consisting of actor number
* `objActorNum` and counter `objCtr`.
*/
function visibleListElements(docState, blockIndex, objActorNum, objCtr) {
const thisBlock = docState.blocks[blockIndex]
const nextBlock = docState.blocks[blockIndex + 1]
if (thisBlock.lastObjectActor !== objActorNum || thisBlock.lastObjectCtr !== objCtr ||
thisBlock.numVisible === undefined) {
return 0
// If a list element is split across the block boundary, don't double-count it
} else if (thisBlock.lastVisibleActor === nextBlock.firstVisibleActor &&
thisBlock.lastVisibleActor !== undefined &&
thisBlock.lastVisibleCtr === nextBlock.firstVisibleCtr &&
thisBlock.lastVisibleCtr !== undefined) {
return thisBlock.numVisible - 1
} else {
return thisBlock.numVisible
}
}
/**
* Scans the blocks of document operations to find the position where a new operation should be
* inserted. Returns an object with keys:
* - `blockIndex`: the index of the block into which we should insert the new operation
* - `skipCount`: the number of operations, counted from the start of the block, after which the
* new operations should be inserted or merged.
* - `visibleCount`: if modifying a list object, the number of visible (i.e. non-deleted) list
* elements that precede the position where the new operations should be applied.
*/
function seekToOp(docState, ops) {
const { objActor, objActorNum, objCtr, keyActor, keyCtr, keyStr } = ops
let blockIndex = 0, totalVisible = 0
// Skip any blocks that contain only objects with lower objectIds
if (objCtr !== null) {
while (blockIndex < docState.blocks.length - 1) {
const blockActor = docState.blocks[blockIndex].lastObjectActor === undefined ? undefined
: docState.actorIds[docState.blocks[blockIndex].lastObjectActor]
const blockCtr = docState.blocks[blockIndex].lastObjectCtr
if (blockCtr === null || blockCtr < objCtr || (blockCtr === objCtr && blockActor < objActor)) {
blockIndex++
} else {
break
}
}
}
if (keyStr !== null) {
// String key is used. First skip any blocks that contain only lower keys
while (blockIndex < docState.blocks.length - 1) {
const { lastObjectActor, lastObjectCtr, lastKey } = docState.blocks[blockIndex]
if (objCtr === lastObjectCtr && objActorNum === lastObjectActor &&
lastKey !== undefined && lastKey < keyStr) blockIndex++; else break
}
// When we have a candidate block, decode it to find the exact insertion position
const {skipCount} = seekWithinBlock(ops, docState.blocks[blockIndex].columns, docState.actorIds, false)
return {blockIndex, skipCount, visibleCount: 0}
} else {
// List operation
const insertAtHead = keyCtr === null || keyCtr === 0 || keyActor === null
const keyActorNum = keyActor === null ? null : docState.actorIds.indexOf(keyActor)
let resumeInsertion = false
while (true) {
// Search for the reference element, skipping any blocks whose Bloom filter does not contain
// the reference element. We only do this if not inserting at the head (in which case there is
// no reference element), or if we already found the reference element in an earlier block (in
// which case we have resumeInsertion === true). The latter case arises with concurrent
// insertions at the same position, and so we have to scan beyond the reference element to
// find the actual insertion position, and that further scan crosses a block boundary.
if (!insertAtHead && !resumeInsertion) {
while (blockIndex < docState.blocks.length - 1 &&
docState.blocks[blockIndex].lastObjectActor === objActorNum &&
docState.blocks[blockIndex].lastObjectCtr === objCtr &&
!bloomFilterContains(docState.blocks[blockIndex].bloom, keyActorNum, keyCtr)) {
// If we reach the end of the list object without a Bloom filter hit, the reference element
// doesn't exist
if (docState.blocks[blockIndex].lastObjectCtr > objCtr) {
throw new RangeError(`Reference element not found: ${keyCtr}@${keyActor}`)
}
// Add up number of visible list elements in any blocks we skip, for list index computation
totalVisible += visibleListElements(docState, blockIndex, objActorNum, objCtr)
blockIndex++
}
}
// We have a candidate block. Decode it to see whether it really contains the reference element
const {found, skipCount, visibleCount} = seekWithinBlock(ops,
docState.blocks[blockIndex].columns,
docState.actorIds,
resumeInsertion)
if (blockIndex === docState.blocks.length - 1 ||
docState.blocks[blockIndex].lastObjectActor !== objActorNum ||
docState.blocks[blockIndex].lastObjectCtr !== objCtr) {
// Last block: if we haven't found the reference element by now, it's an error
if (found) {
return {blockIndex, skipCount, visibleCount: totalVisible + visibleCount}
} else {
throw new RangeError(`Reference element not found: ${keyCtr}@${keyActor}`)
}
} else if (found && skipCount < docState.blocks[blockIndex].numOps) {
// The insertion position lies within the current block
return {blockIndex, skipCount, visibleCount: totalVisible + visibleCount}
}
// Reference element not found and there are still blocks left ==> it was probably a false positive.
// Reference element found, but we skipped all the way to the end of the block ==> we need to
// continue scanning the next block to find the actual insertion position.
// Either way, go back round the loop again to skip blocks until the next Bloom filter hit.
resumeInsertion = found && ops.insert
totalVisible += visibleListElements(docState, blockIndex, objActorNum, objCtr)
blockIndex++
}
}
}
/**
* Updates Bloom filter `bloom`, given as a Uint8Array, to contain the list element ID consisting of
* counter `elemIdCtr` and actor number `elemIdActor`. We don't actually bother computing a hash
* function, since those two integers serve perfectly fine as input. We turn the two integers into a
* sequence of probe indexes using the triple hashing algorithm from the following paper:
*
* Peter C. Dillinger and Panagiotis Manolios. Bloom Filters in Probabilistic Verification.
* 5th International Conference on Formal Methods in Computer-Aided Design (FMCAD), November 2004.
* http://www.ccis.northeastern.edu/home/pete/pub/bloom-filters-verification.pdf
*/
function bloomFilterAdd(bloom, elemIdActor, elemIdCtr) {
let modulo = 8 * bloom.byteLength, x = elemIdCtr % modulo, y = elemIdActor % modulo
// Use one step of FNV-1a to compute a third value from the two inputs.
// Taken from http://www.isthe.com/chongo/tech/comp/fnv/index.html
// The prime is just over 2^24, so elemIdCtr can be up to about 2^29 = 500 million before the
// result of the multiplication exceeds 2^53. And even if it does exceed 2^53 and loses precision,
// that shouldn't be a problem as it should still be deterministic, and the Bloom filter
// computation only needs to be internally consistent within this library.
let z = ((elemIdCtr ^ elemIdActor) * 16777619 >>> 0) % modulo
for (let i = 0; i < BLOOM_NUM_PROBES; i++) {
bloom[x >>> 3] |= 1 << (x & 7)
x = (x + y) % modulo
y = (y + z) % modulo
}
}
/**
* Returns true if the list element ID consisting of counter `elemIdCtr` and actor number
* `elemIdActor` is likely to be contained in the Bloom filter `bloom`.
*/
function bloomFilterContains(bloom, elemIdActor, elemIdCtr) {
let modulo = 8 * bloom.byteLength, x = elemIdCtr % modulo, y = elemIdActor % modulo
let z = ((elemIdCtr ^ elemIdActor) * 16777619 >>> 0) % modulo
// See comments in the bloomFilterAdd function for an explanation
for (let i = 0; i < BLOOM_NUM_PROBES; i++) {
if ((bloom[x >>> 3] & (1 << (x & 7))) === 0) {
return false
}
x = (x + y) % modulo
y = (y + z) % modulo
}
return true
}
/**
* Reads the relevant columns of a block of operations and updates that block to contain the
* metadata we need to efficiently figure out where to insert new operations.
*/
function updateBlockMetadata(block) {
block.bloom = new Uint8Array(BLOOM_FILTER_SIZE)
block.numOps = 0
block.lastKey = undefined
block.numVisible = undefined
block.lastObjectActor = undefined
block.lastObjectCtr = undefined
block.firstVisibleActor = undefined
block.firstVisibleCtr = undefined
block.lastVisibleActor = undefined
block.lastVisibleCtr = undefined
for (let col of block.columns) col.decoder.reset()
const [objActorD, objCtrD, keyActorD, keyCtrD, keyStrD, idActorD, idCtrD, insertD, /* actionD */,
/* valLenD */, /* valRawD */, /* chldActorD */, /* chldCtrD */, succNumD] = block.columns.map(col => col.decoder)
while (!idCtrD.done) {
block.numOps += 1
const objActor = objActorD.readValue(), objCtr = objCtrD.readValue()
const keyActor = keyActorD.readValue(), keyCtr = keyCtrD.readValue(), keyStr = keyStrD.readValue()
const idActor = idActorD.readValue(), idCtr = idCtrD.readValue()
const insert = insertD.readValue(), succNum = succNumD.readValue()
if (block.lastObjectActor !== objActor || block.lastObjectCtr !== objCtr) {
block.numVisible = 0
block.lastObjectActor = objActor
block.lastObjectCtr = objCtr
}
if (keyStr !== null) {
// Map key: for each object, record the highest key contained in the block
block.lastKey = keyStr
} else if (insert || keyCtr !== null) {
// List element
block.lastKey = undefined
const elemIdActor = insert ? idActor : keyActor
const elemIdCtr = insert ? idCtr : keyCtr
bloomFilterAdd(block.bloom, elemIdActor, elemIdCtr)
// If the list element is visible, update the block metadata accordingly
if (succNum === 0) {
if (block.firstVisibleActor === undefined) block.firstVisibleActor = elemIdActor
if (block.firstVisibleCtr === undefined) block.firstVisibleCtr = elemIdCtr
if (block.lastVisibleActor !== elemIdActor || block.lastVisibleCtr !== elemIdCtr) {
block.numVisible += 1
block.lastVisibleActor = elemIdActor
block.lastVisibleCtr = elemIdCtr
}
}
}
}
}
/**
* Updates a block's metadata based on an operation being added to a block.
*/
function addBlockOperation(block, op, actorIds, isChangeOp) {
if (op[keyStrIdx] !== null) {
// TODO this comparison should use UTF-8 encoding, not JavaScript's UTF-16
if (block.lastObjectCtr === op[objCtrIdx] && block.lastObjectActor === op[objActorIdx] &&
(block.lastKey === undefined || block.lastKey < op[keyStrIdx])) {
block.lastKey = op[keyStrIdx]
}
} else {
// List element
const elemIdActor = op[insertIdx] ? op[idActorIdx] : op[keyActorIdx]
const elemIdCtr = op[insertIdx] ? op[idCtrIdx] : op[keyCtrIdx]
bloomFilterAdd(block.bloom, elemIdActor, elemIdCtr)
// Set lastVisible on the assumption that this is the last op in the block; if there are further
// ops after this one in the block, lastVisible will be overwritten again later.
if (op[succNumIdx] === 0 || isChangeOp) {
if (block.firstVisibleActor === undefined) block.firstVisibleActor = elemIdActor
if (block.firstVisibleCtr === undefined) block.firstVisibleCtr = elemIdCtr
block.lastVisibleActor = elemIdActor
block.lastVisibleCtr = elemIdCtr
}
}
// Keep track of the largest objectId contained within a block
if (block.lastObjectCtr === undefined ||
op[objActorIdx] !== null && op[objCtrIdx] !== null &&
(block.lastObjectCtr === null || block.lastObjectCtr < op[objCtrIdx] ||
(block.lastObjectCtr === op[objCtrIdx] && actorIds[block.lastObjectActor] < actorIds[op[objActorIdx]]))) {
block.lastObjectActor = op[objActorIdx]
block.lastObjectCtr = op[objCtrIdx]
block.lastKey = (op[keyStrIdx] !== null ? op[keyStrIdx] : undefined)
block.numVisible = 0
}
}
/**
* Takes a block containing too many operations, and splits it into a sequence of adjacent blocks of
* roughly equal size.
*/
function splitBlock(block) {
for (let col of block.columns) col.decoder.reset()
// Make each of the resulting blocks between 50% and 80% full (leaving a bit of space in each
// block so that it doesn't get split again right away the next time an operation is added).
// The upper bound cannot be lower than 75% since otherwise we would end up with a block less than
// 50% full when going from two to three blocks.
const numBlocks = Math.ceil(block.numOps / (0.8 * MAX_BLOCK_SIZE))
let blocks = [], opsSoFar = 0
for (let i = 1; i <= numBlocks; i++) {
const opsToCopy = Math.ceil(i * block.numOps / numBlocks) - opsSoFar
const encoders = block.columns.map(col => ({columnId: col.columnId, encoder: encoderByColumnId(col.columnId)}))
copyColumns(encoders, block.columns, opsToCopy)
const decoders = encoders.map(col => {
const decoder = decoderByColumnId(col.columnId, col.encoder.buffer)
return {columnId: col.columnId, decoder}
})
const newBlock = {columns: decoders}
updateBlockMetadata(newBlock)
blocks.push(newBlock)
opsSoFar += opsToCopy
}
return blocks
}
/**
* Takes an array of blocks and concatenates the corresponding columns across all of the blocks.
*/
function concatBlocks(blocks) {
const encoders = blocks[0].columns.map(col => ({columnId: col.columnId, encoder: encoderByColumnId(col.columnId)}))
for (let block of blocks) {
for (let col of block.columns) col.decoder.reset()
copyColumns(encoders, block.columns, block.numOps)
}
return encoders
}
/**
* Copies `count` rows from the set of input columns `inCols` to the set of output columns
* `outCols`. The input columns are given as an array of `{columnId, decoder}` objects, and the
* output columns are given as an array of `{columnId, encoder}` objects. Both are sorted in
* increasing order of columnId. If there is no matching input column for a given output column, it
* is filled in with `count` blank values (according to the column type).
*/
function copyColumns(outCols, inCols, count) {
if (count === 0) return
let inIndex = 0, lastGroup = -1, lastCardinality = 0, valueColumn = -1, valueBytes = 0
for (let outCol of outCols) {
while (inIndex < inCols.length && inCols[inIndex].columnId < outCol.columnId) inIndex++
let inCol = null
if (inIndex < inCols.length && inCols[inIndex].columnId === outCol.columnId &&
inCols[inIndex].decoder.buf.byteLength > 0) {
inCol = inCols[inIndex].decoder
}
const colCount = (outCol.columnId >> 4 === lastGroup) ? lastCardinality : count
if (outCol.columnId % 8 === COLUMN_TYPE.GROUP_CARD) {
lastGroup = outCol.columnId >> 4
if (inCol) {
lastCardinality = outCol.encoder.copyFrom(inCol, {count, sumValues: true}).sum
} else {
outCol.encoder.appendValue(0, count)
lastCardinality = 0
}
} else if (outCol.columnId % 8 === COLUMN_TYPE.VALUE_LEN) {
if (inCol) {
if (inIndex + 1 === inCols.length || inCols[inIndex + 1].columnId !== outCol.columnId + 1) {
throw new RangeError('VALUE_LEN column without accompanying VALUE_RAW column')
}
valueColumn = outCol.columnId + 1
valueBytes = outCol.encoder.copyFrom(inCol, {count: colCount, sumValues: true, sumShift: 4}).sum
} else {
outCol.encoder.appendValue(null, colCount)
valueColumn = outCol.columnId + 1
valueBytes = 0
}
} else if (outCol.columnId % 8 === COLUMN_TYPE.VALUE_RAW) {
if (outCol.columnId !== valueColumn) {
throw new RangeError('VALUE_RAW column without accompanying VALUE_LEN column')
}
if (valueBytes > 0) {
outCol.encoder.appendRawBytes(inCol.readRawBytes(valueBytes))
}
} else { // ACTOR_ID, INT_RLE, INT_DELTA, BOOLEAN, or STRING_RLE
if (inCol) {
outCol.encoder.copyFrom(inCol, {count: colCount})
} else {
const blankValue = (outCol.columnId % 8 === COLUMN_TYPE.BOOLEAN) ? false : null
outCol.encoder.appendValue(blankValue, colCount)
}
}
}
}
/**
* Parses one operation from a set of columns. The argument `columns` contains a list of objects
* with `columnId` and `decoder` properties. Returns an array in which the i'th element is the
* value read from the i'th column in `columns`. Does not interpret datatypes; the only
* interpretation of values is that if `actorTable` is given, a value `v` in a column of type
* ACTOR_ID is replaced with `actorTable[v]`.
*/
function readOperation(columns, actorTable) {
let operation = [], colValue, lastGroup = -1, lastCardinality = 0, valueColumn = -1, valueBytes = 0
for (let col of columns) {
if (col.columnId % 8 === COLUMN_TYPE.VALUE_RAW) {
if (col.columnId !== valueColumn) throw new RangeError('unexpected VALUE_RAW column')
colValue = col.decoder.readRawBytes(valueBytes)
} else if (col.columnId % 8 === COLUMN_TYPE.GROUP_CARD) {
lastGroup = col.columnId >> 4
lastCardinality = col.decoder.readValue() || 0
colValue = lastCardinality
} else if (col.columnId >> 4 === lastGroup) {
colValue = []
if (col.columnId % 8 === COLUMN_TYPE.VALUE_LEN) {
valueColumn = col.columnId + 1
valueBytes = 0
}
for (let i = 0; i < lastCardinality; i++) {
let value = col.decoder.readValue()
if (col.columnId % 8 === COLUMN_TYPE.ACTOR_ID && actorTable && typeof value === 'number') {
value = actorTable[value]
}
if (col.columnId % 8 === COLUMN_TYPE.VALUE_LEN) {
valueBytes += colValue >>> 4
}
colValue.push(value)
}
} else {
colValue = col.decoder.readValue()
if (col.columnId % 8 === COLUMN_TYPE.ACTOR_ID && actorTable && typeof colValue === 'number') {
colValue = actorTable[colValue]
}
if (col.columnId % 8 === COLUMN_TYPE.VALUE_LEN) {
valueColumn = col.columnId + 1
valueBytes = colValue >>> 4
}
}
operation.push(colValue)
}
return operation
}
/**
* Appends `operation`, in the form returned by `readOperation()`, to the columns in `outCols`. The
* argument `inCols` provides metadata about the types of columns in `operation`; the value
* `operation[i]` comes from the column `inCols[i]`.
*/
function appendOperation(outCols, inCols, operation) {
let inIndex = 0, lastGroup = -1, lastCardinality = 0
for (let outCol of outCols) {
while (inIndex < inCols.length && inCols[inIndex].columnId < outCol.columnId) inIndex++
if (inIndex < inCols.length && inCols[inIndex].columnId === outCol.columnId) {
const colValue = operation[inIndex]
if (outCol.columnId % 8 === COLUMN_TYPE.GROUP_CARD) {
lastGroup = outCol.columnId >> 4
lastCardinality = colValue
outCol.encoder.appendValue(colValue)
} else if (outCol.columnId >> 4 === lastGroup) {
if (!Array.isArray(colValue) || colValue.length !== lastCardinality) {
throw new RangeError('bad group value')
}
for (let v of colValue) outCol.encoder.appendValue(v)
} else if (outCol.columnId % 8 === COLUMN_TYPE.VALUE_RAW) {
if (colValue) outCol.encoder.appendRawBytes(colValue)
} else {
outCol.encoder.appendValue(colValue)
}
} else if (outCol.columnId % 8 === COLUMN_TYPE.GROUP_CARD) {
lastGroup = outCol.columnId >> 4
lastCardinality = 0
outCol.encoder.appendValue(0)
} else if (outCol.columnId % 8 !== COLUMN_TYPE.VALUE_RAW) {
const count = (outCol.columnId >> 4 === lastGroup) ? lastCardinality : 1
let blankValue = null
if (outCol.columnId % 8 === COLUMN_TYPE.BOOLEAN) blankValue = false
if (outCol.columnId % 8 === COLUMN_TYPE.VALUE_LEN) blankValue = 0
outCol.encoder.appendValue(blankValue, count)
}
}
}
/**
* Parses the next operation from block `blockIndex` of the document. Returns an object of the form
* `{docOp, blockIndex}` where `docOp` is an operation in the form returned by `readOperation()`,
* and `blockIndex` is the block number to use on the next call (it moves on to the next block when
* we reach the end of the current block). `docOp` is null if there are no more operations.
*/
function readNextDocOp(docState, blockIndex) {
let block = docState.blocks[blockIndex]
if (!block.columns[actionIdx].decoder.done) {
return {docOp: readOperation(block.columns), blockIndex}
} else if (blockIndex === docState.blocks.length - 1) {
return {docOp: null, blockIndex}
} else {
blockIndex += 1
block = docState.blocks[blockIndex]
for (let col of block.columns) col.decoder.reset()
return {docOp: readOperation(block.columns), blockIndex}
}
}
/**
* Parses the next operation from a sequence of changes. `changeState` serves as the state of this
* pseudo-iterator, and it is mutated to reflect the new operation. In particular,
* `changeState.nextOp` is set to the operation that was read, and `changeState.done` is set to true
* when we have finished reading the last operation in the last change.
*/
function readNextChangeOp(docState, changeState) {
// If we've finished reading one change, move to the next change that contains at least one op
while (changeState.changeIndex < changeState.changes.length - 1 &&
(!changeState.columns || changeState.columns[actionIdx].decoder.done)) {
changeState.changeIndex += 1
const change = changeState.changes[changeState.changeIndex]
changeState.columns = makeDecoders(change.columns, CHANGE_COLUMNS)
changeState.opCtr = change.startOp
// If it's an empty change (no ops), set its maxOp here since it won't be set below
if (changeState.columns[actionIdx].decoder.done) {
change.maxOp = change.startOp - 1
}
// Update docState based on the information in the change
updateBlockColumns(docState, changeState.columns)
const {actorIds, actorTable} = getActorTable(docState.actorIds, change)
docState.actorIds = actorIds
changeState.actorTable = actorTable
changeState.actorIndex = docState.actorIds.indexOf(change.actorIds[0])
}
// Reached the end of the last change?
if (changeState.columns[actionIdx].decoder.done) {
changeState.done = true
changeState.nextOp = null
return
}
changeState.nextOp = readOperation(changeState.columns, changeState.actorTable)
changeState.nextOp[idActorIdx] = changeState.actorIndex
changeState.nextOp[idCtrIdx] = changeState.opCtr
changeState.changes[changeState.changeIndex].maxOp = changeState.opCtr
if (changeState.opCtr > docState.maxOp) docState.maxOp = changeState.opCtr
changeState.opCtr += 1
const op = changeState.nextOp
if ((op[objCtrIdx] === null && op[objActorIdx] !== null) ||
(op[objCtrIdx] !== null && op[objActorIdx] === null)) {
throw new RangeError(`Mismatched object reference: (${op[objCtrIdx]}, ${op[objActorIdx]})`)
}
if ((op[keyCtrIdx] === null && op[keyActorIdx] !== null) ||
(op[keyCtrIdx] === 0 && op[keyActorIdx] !== null) ||
(op[keyCtrIdx] > 0 && op[keyActorIdx] === null)) {
throw new RangeError(`Mismatched operation key: (${op[keyCtrIdx]}, ${op[keyActorIdx]})`)
}
}
function emptyObjectPatch(objectId, type) {
if (type === 'list' || type === 'text') {
return {objectId, type, edits: []}
} else {
return {objectId, type, props: {}}
}
}
/**
* Returns true if the two given operation IDs have the same actor ID, and the counter of `id2` is
* exactly `delta` greater than the counter of `id1`.
*/
function opIdDelta(id1, id2, delta = 1) {
const parsed1 = parseOpId(id1), parsed2 = parseOpId(id2)
return parsed1.actorId === parsed2.actorId && parsed1.counter + delta === parsed2.counter
}
/**
* Appends a list edit operation (insert, update, remove) to an array of existing operations. If the
* last existing operation can be extended (as a multi-op), we do that.
*/
function appendEdit(existingEdits, nextEdit) {
if (existingEdits.length === 0) {
existingEdits.push(nextEdit)
return
}
let lastEdit = existingEdits[existingEdits.length - 1]
if (lastEdit.action === 'insert' && nextEdit.action === 'insert' &&
lastEdit.index === nextEdit.index - 1 &&
lastEdit.value.type === 'value' && nextEdit.value.type === 'value' &&
lastEdit.elemId === lastEdit.opId && nextEdit.elemId === nextEdit.opId &&
opIdDelta(lastEdit.elemId, nextEdit.elemId, 1) &&
lastEdit.value.datatype === nextEdit.value.datatype &&
typeof lastEdit.value.value === typeof nextEdit.value.value) {
lastEdit.action = 'multi-insert'
if (nextEdit.value.datatype) lastEdit.datatype = nextEdit.value.datatype
lastEdit.values = [lastEdit.value.value, nextEdit.value.value]
delete lastEdit.value
delete lastEdit.opId
} else if (lastEdit.action === 'multi-insert' && nextEdit.action === 'insert' &&
lastEdit.index + lastEdit.values.length === nextEdit.index &&
nextEdit.value.type === 'value' && nextEdit.elemId === nextEdit.opId &&
opIdDelta(lastEdit.elemId, nextEdit.elemId, lastEdit.values.length) &&
lastEdit.datatype === nextEdit.value.datatype &&
typeof lastEdit.values[0] === typeof nextEdit.value.value) {
lastEdit.values.push(nextEdit.value.value)
} else if (lastEdit.action === 'remove' && nextEdit.action === 'remove' &&
lastEdit.index === nextEdit.index) {
lastEdit.count += nextEdit.count
} else {
existingEdits.push(nextEdit)
}
}
/**
* `edits` is an array of (SingleInsertEdit | MultiInsertEdit | UpdateEdit | RemoveEdit) list edits
* for a patch. This function appends an UpdateEdit to this array. A conflict is represented by
* having several consecutive edits with the same index, and this can be realised by calling
* `appendUpdate` several times for the same list element. On the first such call, `firstUpdate`
* must be true.
*
* It is possible that coincidentally the previous edit (potentially arising from a different
* change) is for the same index. If this is the case, to avoid accidentally treating consecutive
* updates for the same index as a conflict, we remove the previous edit for the same index. This is
* safe because the previous edit is overwritten by the new edit being appended, and we know that
* it's for the same list elements because there are no intervening insertions/deletions that could
* have changed the indexes.
*/
function appendUpdate(edits, index, elemId, opId, value, firstUpdate) {
let insert = false
if (firstUpdate) {
// Pop all edits for the same index off the end of the edits array. This sequence may begin with
// either an insert or an update. If it's an insert, we remember that fact, and use it below.
while (!insert && edits.length > 0) {
const lastEdit = edits[edits.length - 1]
if ((lastEdit.action === 'insert' || lastEdit.action === 'update') && lastEdit.index === index) {
edits.pop()
insert = (lastEdit.action === 'insert')
} else if (lastEdit.action === 'multi-insert' && lastEdit.index + lastEdit.values.length - 1 === index) {
lastEdit.values.pop()
insert = true
} else {
break
}
}
}
// If we popped an insert edit off the edits array, we need to turn the new update into an insert
// in order to ensure the list element still gets inserted (just with a new value).
if (insert) {
appendEdit(edits, {action: 'insert', index, elemId, opId, value})
} else {
appendEdit(edits, {action: 'update', index, opId, value})
}
}
/**
* `edits` is an array of (SingleInsertEdit | MultiInsertEdit | UpdateEdit | RemoveEdit) list edits
* for a patch. We assume that there is a suffix of this array that consists of an insertion at
* position `index`, followed by zero or more UpdateEdits at the same index. This function rewrites
* that suffix to be all updates instead. This is needed because sometimes when generating a patch
* we think we are performing a list insertion, but then it later turns out that there was already
* an existing value at that list element, and so we actually need to do an update, not an insert.
*
* If the suffix is preceded by one or more updates at the same index, those earlier updates are
* removed by `appendUpdate()` to ensure we don't inadvertently treat them as part of the same
* conflict.
*/
function convertInsertToUpdate(edits, index, elemId) {
let updates = []
while (edits.length > 0) {
let lastEdit = edits[edits.length - 1]
if (lastEdit.action === 'insert') {
if (lastEdit.index !== index) throw new RangeError('last edit has unexpected index')
updates.unshift(edits.pop())
break
} else if (lastEdit.action === 'update') {
if (lastEdit.index !== index) throw new RangeError('last edit has unexpected index')
updates.unshift(edits.pop())
} else {
// It's impossible to encounter a remove edit here because the state machine in
// updatePatchProperty() ensures that a property can have either an insert or a remove edit,
// but not both. It's impossible to encounter a multi-insert here because multi-inserts always
// have equal elemId and opId (i.e. they can only be used for the operation that first inserts
// an element, but not for any subsequent assignments to that list element); moreover,
// convertInsertToUpdate is only called if an insert action is followed by a non-overwritten
// document op. The fact that there is a non-overwritten document op after another op on the
// same list element implies that the original insertion op for that list element must be
// overwritten, and thus the original insertion op cannot have given rise to a multi-insert.
throw new RangeError('last edit has unexpected action')
}
}
// Now take the edits we popped off and push them back onto the list again
let firstUpdate = true
for (let update of updates) {
appendUpdate(edits, index, elemId, update.opId, update.value, firstUpdate)
firstUpdate = false
}
}
/**
* Updates `patches` to reflect the operation `op` within the document with state `docState`.
* Can be called multiple times if there are multiple operations for the same property (e.g. due
* to a conflict). `propState` is an object that carries over state between such successive
* invocations for the same property. If the current object is a list, `listIndex` is the index
* into that list (counting only visible elements). If the operation `op` was already previously
* in the document, `oldSuccNum` is the value of `op[succNumIdx]` before the current change was
* applied (allowing us to determine whether this operation was overwritten or deleted in the
* current change). `oldSuccNum` must be undefined if the operation came from the current change.
* If we are creating an incremental patch as a result of applying one or more changes, `newBlock`
* is the block to which the operations are getting written; we will update the metadata on this
* block. `newBlock` should be null if we are creating a patch for the whole document.
*/
function updatePatchProperty(patches, newBlock, objectId, op, docState, propState, listIndex, oldSuccNum) {
const isWholeDoc = !newBlock
const type = op[actionIdx] < ACTIONS.length ? OBJECT_TYPE[ACTIONS[op[actionIdx]]] : null
const opId = `${op[idCtrIdx]}@${docState.actorIds[op[idActorIdx]]}`
const elemIdActor = op[insertIdx] ? op[idActorIdx] : op[keyActorIdx]
const elemIdCtr = op[insertIdx] ? op[idCtrIdx] : op[keyCtrIdx]
const elemId = op[keyStrIdx] ? op[keyStrIdx] : `${elemIdCtr}@${docState.actorIds[elemIdActor]}`
// When the change contains a new make* operation (i.e. with an even-numbered action), record the
// new parent-child relationship in objectMeta. TODO: also handle link/move operations.
if (op[actionIdx] % 2 === 0 && !docState.objectMeta[opId]) {
docState.objectMeta[opId] = {parentObj: objectId, parentKey: elemId, opId, type, children: {}}
deepCopyUpdate(docState.objectMeta, [objectId, 'children', elemId, opId], {objectId: opId, type, props: {}})
}
// firstOp is true if the current operation is the first of a sequence of ops for the same key
const firstOp = !propState[elemId]
if (!propState[elemId]) propState[elemId] = {visibleOps: [], hasChild: false}
// An operation is overwritten if it is a document operation that has at least one successor
const isOverwritten = (oldSuccNum !== undefined && op[succNumIdx] > 0)
// Record all visible values for the property, and whether it has any child object
if (!isOverwritten) {
propState[elemId].visibleOps.push(op)
propState[elemId].hasChild = propState[elemId].hasChild || (op[actionIdx] % 2) === 0 // even-numbered action == make* operation
}
// If one or more of the values of the property is a child object, we update objectMeta to store
// all of the visible values of the property (even the non-child-object values). Then, when we
// subsequently process an update within that child object, we can construct the patch to
// contain the conflicting values.
const prevChildren = docState.objectMeta[objectId].children[elemId]
if (propState[elemId].hasChild || (prevChildren && Object.keys(prevChildren).length > 0)) {
let values = {}
for (let visible of propState[elemId].visibleOps) {
const opId = `${visible[idCtrIdx]}@${docState.actorIds[visible[idActorIdx]]}`
if (ACTIONS[visible[actionIdx]] === 'set') {
values[opId] = Object.assign({type: 'value'}, decodeValue(visible[valLenIdx], visible[valRawIdx]))
} else if (visible[actionIdx] % 2 === 0) {
const objType = visible[actionIdx] < ACTIONS.length ? OBJECT_TYPE[ACTIONS[visible[actionIdx]]] : null
values[opId] = emptyObjectPatch(opId, objType)
}
}
// Copy so that objectMeta is not modified if an exception is thrown while applying change
deepCopyUpdate(docState.objectMeta, [objectId, 'children', elemId], values)
}
let patchKey, patchValue
// For counters, increment operations are succs to the set operation that created the counter,
// but in this case we want to add the values rather than overwriting them.
if (isOverwritten && ACTIONS[op[actionIdx]] === 'set' && (op[valLenIdx] & 0x0f) === VALUE_TYPE.COUNTER) {
// This is the initial set operation that creates a counter. Initialise the counter state
// to contain all successors of the set operation. Only if we later find that each of these
// successor operations is an increment, we make the counter visible in the patch.
if (!propState[elemId]) propState[elemId] = {visibleOps: [], hasChild: false}
if (!propState[elemId].counterStates) propState[elemId].counterStates = {}
let counterStates = propState[elemId].counterStates
let counterState = {opId, value: decodeValue(op[valLenIdx], op[valRawIdx]).value, succs: {}}
for (let i = 0; i < op[succNumIdx]; i++) {
const succOp = `${op[succCtrIdx][i]}@${docState.actorIds[op[succActorIdx][i]]}`
counterStates[succOp] = counterState
counterState.succs[succOp] = true
}
} else if (ACTIONS[op[actionIdx]] === 'inc') {
// Incrementing a previously created counter.
if (!propState[elemId] || !propState[elemId].counterStates || !propState[elemId].counterStates[opId]) {
throw new RangeError(`increment operation ${opId} for unknown counter`)
}
let counterState = propState[elemId].counterStates[opId]
counterState.value += decodeValue(op[valLenIdx], op[valRawIdx]).value
delete counterState.succs[opId]
if (Object.keys(counterState.succs).length === 0) {
patchKey = counterState.opId
patchValue = {type: 'value', datatype: 'counter', value: counterState.value}
// TODO if the counter is in a list element, we need to add a 'remove' action when deleted
}
} else if (!isOverwritten) {
// Add the value to the patch if it is not overwritten (i.e. if it has no succs).
if (ACTIONS[op[actionIdx]] === 'set') {
patchKey = opId
patchValue = Object.assign({type: 'value'}, decodeValue(op[valLenIdx], op[valRawIdx]))
} else if (op[actionIdx] % 2 === 0) { // even-numbered action == make* operation
if (!patches[opId]) patches[opId] = emptyObjectPatch(opId, type)
patchKey = opId
patchValue = patches[opId]
}
}
if (!patches[objectId]) patches[objectId] = emptyObjectPatch(objectId, docState.objectMeta[objectId].type)
const patch = patches[objectId]
// Updating a list or text object (with elemId key)
if (op[keyStrIdx] === null) {
// If we come across any document op that was previously non-overwritten/non-deleted, that
// means the current list element already had a value before this change was applied, and
// therefore the current element cannot be an insert. If we already registered an insert, we
// have to convert it into an update.
if (oldSuccNum === 0 && !isWholeDoc && propState[elemId].action === 'insert') {
propState[elemId].action = 'update'
convertInsertToUpdate(patch.edits, listIndex, elemId)
if (newBlock && newBlock.lastObjectActor === op[objActorIdx] && newBlock.lastObjectCtr === op[objCtrIdx]) {
newBlock.numVisible -= 1
}
}
if (patchValue) {
// If the op has a non-overwritten value and it came from the change, it's an insert.
// (It's not necessarily the case that op[insertIdx] is true: if a list element is concurrently
// deleted and updated, the node that first processes the deletion and then the update will
// observe the update as a re-insertion of the deleted list element.)
if (!propState[elemId].action && (oldSuccNum === undefined || isWholeDoc)) {
propState[elemId].action = 'insert'
appendEdit(patch.edits, {action: 'insert', index: listIndex, elemId, opId: patchKey, value: patchValue})
if (newBlock && newBlock.lastObjectActor === op[objActorIdx] && newBlock.lastObjectCtr === op[objCtrIdx]) {
newBlock.numVisible += 1
}
// If the property has a value and it's not an insert, then it must be an update.
// We might have previously registered it as a remove, in which case we convert it to update.
} else if (propState[elemId].action === 'remove') {
let lastEdit = patch.edits[patch.edits.length - 1]
if (lastEdit.action !== 'remove') throw new RangeError('last edit has unexpected type')
if (lastEdit.count > 1) lastEdit.count -= 1; else patch.edits.pop()
propState[elemId].action = 'update'
appendUpdate(patch.edits, listIndex, elemId, patchKey, patchValue, true)
if (newBlock && newBlock.lastObjectActor === op[objActorIdx] && newBlock.lastObjectCtr === op[objCtrIdx]) {
newBlock.numVisible += 1
}
} else {
// A 'normal' update
appendUpdate(patch.edits, listIndex, elemId, patchKey, patchValue, !propState[elemId].action)
if (!propState[elemId].action) propState[elemId].action = 'update'
}
} else if (oldSuccNum === 0 && !propState[elemId].action) {
// If the property used to have a non-overwritten/non-deleted value, but no longer, it's a remove
propState[elemId].action = 'remove'
appendEdit(patch.edits, {action: 'remove', index: listIndex, count: 1})
if (newBlock && newBlock.lastObjectActor === op[objActorIdx] && newBlock.lastObjectCtr === op[objCtrIdx]) {
newBlock.numVisible -= 1
}
}
} else if (patchValue || !isWholeDoc) {
// Updating a map or table (with string key)
if (firstOp || !patch.props[op[keyStrIdx]]) patch.props[op[keyStrIdx]] = {}
if (patchValue) patch.props[op[keyStrIdx]][patchKey] = patchValue
}
}
/**
* Applies operations (from one or more changes) to the document by merging the sequence of change
* ops into the sequence of document ops. The two inputs are `changeState` and `docState`
* respectively. Assumes that the decoders of both sets of columns are at the position where we want
* to start merging. `patches` is mutated to reflect the effect of the change operations. `ops` is
* the operation sequence to apply (as decoded by `groupRelatedOps()`). `docState` is as
* documented in `applyOps()`. If the operations are updating a list or text object, `listIndex`
* is the number of visible elements that precede the position at which we start merging.
* `blockIndex` is the document block number from which we are currently reading.
*/
function mergeDocChangeOps(patches, newBlock, outCols, changeState, docState, listIndex, blockIndex) {
const firstOp = changeState.nextOp, insert = firstOp[insertIdx]
const objActor = firstOp[objActorIdx], objCtr = firstOp[objCtrIdx]
const objectId = objActor === null ? '_root' : `${objCtr}@${docState.actorIds[objActor]}`
const idActorIndex = changeState.actorIndex, idActor = docState.actorIds[idActorIndex]
let foundListElem = false, elemVisible = false, propState = {}, docOp
;({ docOp, blockIndex } = readNextDocOp(docState, blockIndex))
let docOpsConsumed = (docOp === null ? 0 : 1)
let docOpOldSuccNum = (docOp === null ? 0 : docOp[succNumIdx])
let changeOp = null, changeOps = [], changeCols = [], predSeen = [], lastChangeKey = null
changeState.objectIds.add(objectId)
// Merge the two inputs: the sequence of ops in the doc, and the sequence of ops in the change.
// At each iteration, we either output the doc's op (possibly updated based on the change's ops)
// or output an op from the change.
while (true) {
// The array `changeOps` contains operations from the change(s) we're applying. When the array
// is empty, we load changes from the change. Typically we load only a single operation at a
// time, with two exceptions: 1. all operations that update the same key or list element in the
// same object are put into changeOps at the same time (this is needed so that we can update the
// succ columns of the document ops correctly); 2. a run of consecutive insertions is also
// placed into changeOps in one go.
//
// When we have processed all the ops in changeOps we try to see whether there are further
// operations that we can also process while we're at it. Those operations must be for the same
// object, they must be for a key or list element that appears later in the document, they must
// either all be insertions or all be non-insertions, and if insertions, they must be
// consecutive. If these conditions are satisfied, that means the operations can be processed in
// the same pass. If we encounter an operation that does not meet these conditions, we leave
// changeOps empty, and this function returns after having processed any remaining document ops.
//
// Any operations that could not be processed in a single pass remain in changeState; applyOps
// will seek to the appropriate position and then call mergeDocChangeOps again.
if (changeOps.length === 0) {
foundListElem = false
let nextOp = changeState.nextOp
while (!changeState.done && nextOp[idActorIdx] === idActorIndex && nextOp[insertIdx] === insert &&
nextOp[objActorIdx] === firstOp[objActorIdx] && nextOp[objCtrIdx] === firstOp[objCtrIdx]) {
// Check if the operation's pred references a previous operation in changeOps
const lastOp = (changeOps.length > 0) ? changeOps[changeOps.length - 1] : null
let isOverwrite = false
for (let i = 0; i < nextOp[predNumIdx]; i++) {
for (let prevOp of changeOps) {
if (nextOp[predActorIdx][i] === prevOp[idActorIdx] && nextOp[predCtrIdx][i] === prevOp[idCtrIdx]) {
isOverwrite = true
}
}
}
// If any of the following `if` statements is true, we add `nextOp` to `changeOps`. If they
// are all false, we break out of the loop and stop adding to `changeOps`.
if (nextOp === firstOp) {
// First change operation in a mergeDocChangeOps call is always used
} else if (insert && lastOp !== null && nextOp[keyStrIdx] === null &&
nextOp[keyActorIdx] === lastOp[idActorIdx] &&
nextOp[keyCtrIdx] === lastOp[idCtrIdx]) {
// Collect consecutive insertions
} else if (!insert && lastOp !== null && nextOp[keyStrIdx] !== null &&
nextOp[keyStrIdx] === lastOp[keyStrIdx] && !isOverwrite) {
// Collect several updates to the same key
} else if (!insert && lastOp !== null &&
nextOp[keyStrIdx] === null && lastOp[keyStrIdx] === null &&
nextOp[keyActorIdx] === lastOp[keyActorIdx] &&
nextOp[keyCtrIdx] === lastOp[keyCtrIdx] && !isOverwrite) {
// Collect several updates to the same list element
} else if (!insert && lastOp === null && nextOp[keyStrIdx] === null &&
docOp && docOp[insertIdx] && docOp[keyStrIdx] === null &&
docOp[idActorIdx] === nextOp[keyActorIdx] &&
docOp[idCtrIdx] === nextOp[keyCtrIdx]) {
// When updating/deleting list elements, keep
gitextract_5mussk42/ ├── .babelrc ├── .eslintrc.json ├── .github/ │ └── workflows/ │ └── automerge-ci.yml ├── .gitignore ├── .mocharc.yaml ├── @types/ │ └── automerge/ │ └── index.d.ts ├── CHANGELOG.md ├── LICENSE ├── README.md ├── backend/ │ ├── backend.js │ ├── columnar.js │ ├── encoding.js │ ├── index.js │ ├── new.js │ ├── sync.js │ └── util.js ├── frontend/ │ ├── apply_patch.js │ ├── constants.js │ ├── context.js │ ├── counter.js │ ├── index.js │ ├── numbers.js │ ├── observable.js │ ├── proxies.js │ ├── table.js │ └── text.js ├── karma.conf.js ├── karma.sauce.js ├── package.json ├── src/ │ ├── automerge.js │ ├── common.js │ └── uuid.js ├── test/ │ ├── backend_test.js │ ├── columnar_test.js │ ├── context_test.js │ ├── encoding_test.js │ ├── frontend_test.js │ ├── fuzz_test.js │ ├── helpers.js │ ├── new_backend_test.js │ ├── observable_test.js │ ├── proxies_test.js │ ├── sync_test.js │ ├── table_test.js │ ├── test.js │ ├── text_test.js │ ├── typescript_test.ts │ ├── uuid_test.js │ └── wasm.js ├── tsconfig.json └── webpack.config.js
SYMBOL INDEX (498 symbols across 34 files)
FILE: @types/automerge/index.d.ts
type Doc (line 6) | type Doc<T> = FreezeObject<T>
type ChangeFn (line 8) | type ChangeFn<T> = (doc: T) => void
type InitOptions (line 17) | type InitOptions<T> =
type ChangeOptions (line 27) | type ChangeOptions<T> =
type PatchCallback (line 35) | type PatchCallback<T> = (patch: Patch, before: T, after: T, local: boole...
type ObserverCallback (line 36) | type ObserverCallback<T> = (diff: MapDiff | ListDiff | ValueDiff, before...
class Observable (line 38) | class Observable {
class TableRow (line 70) | class TableRow {
class Table (line 74) | class Table<T> {
class List (line 84) | class List<T> extends Array<T> {
class Text (line 89) | class Text extends List<string> {
class Counter (line 101) | class Counter extends Number {
class Int (line 110) | class Int { constructor(value: number) }
class Uint (line 111) | class Uint { constructor(value: number) }
class Float64 (line 112) | class Float64 { constructor(value: number) }
type ReadonlyTable (line 116) | type ReadonlyTable<T> = ReadonlyArray<T> & Table<T>
type ReadonlyList (line 117) | type ReadonlyList<T> = ReadonlyArray<T> & List<T>
type ReadonlyText (line 118) | type ReadonlyText = ReadonlyList<string> & Text
type Hash (line 166) | type Hash = string // 64-digit hex string
type OpId (line 167) | type OpId = string // of the form `${counter}@${actorId}`
type UUID (line 169) | type UUID = string
type UUIDGenerator (line 170) | type UUIDGenerator = () => UUID
type UUIDFactory (line 171) | interface UUIDFactory extends UUIDGenerator {
type Clock (line 177) | interface Clock {
type State (line 181) | interface State<T> {
type BackendState (line 186) | interface BackendState {
type BinaryChange (line 190) | type BinaryChange = Uint8Array & { __binaryChange: true }
type BinaryDocument (line 191) | type BinaryDocument = Uint8Array & { __binaryDocument: true }
type BinarySyncState (line 192) | type BinarySyncState = Uint8Array & { __binarySyncState: true }
type BinarySyncMessage (line 193) | type BinarySyncMessage = Uint8Array & { __binarySyncMessage: true }
type SyncState (line 195) | interface SyncState {
type SyncMessage (line 199) | interface SyncMessage {
type SyncHave (line 206) | interface SyncHave {
type Change (line 211) | interface Change {
type Op (line 222) | interface Op {
type Patch (line 236) | interface Patch {
type MapDiff (line 248) | interface MapDiff {
type ListDiff (line 260) | interface ListDiff {
type SingleInsertEdit (line 269) | interface SingleInsertEdit {
type MultiInsertEdit (line 283) | interface MultiInsertEdit {
type UpdateEdit (line 295) | interface UpdateEdit {
type RemoveEdit (line 304) | interface RemoveEdit {
type ValueDiff (line 312) | interface ValueDiff {
type OpAction (line 318) | type OpAction =
type CollectionType (line 328) | type CollectionType =
type DataType (line 334) | type DataType =
type Freeze (line 348) | type Freeze<T> =
type FreezeTable (line 358) | interface FreezeTable<T> extends ReadonlyTable<Freeze<T>> {}
type FreezeList (line 359) | interface FreezeList<T> extends ReadonlyList<Freeze<T>> {}
type FreezeArray (line 360) | interface FreezeArray<T> extends ReadonlyArray<Freeze<T>> {}
type FreezeMap (line 361) | interface FreezeMap<K, V> extends ReadonlyMap<Freeze<K>, Freeze<V>> {}
type FreezeObject (line 362) | type FreezeObject<T> = { readonly [P in keyof T]: Freeze<T[P]> }
FILE: backend/backend.js
function init (line 8) | function init() {
function clone (line 12) | function clone(backend) {
function free (line 16) | function free(backend) {
function applyChanges (line 27) | function applyChanges(backend, changes) {
function hashByActor (line 34) | function hashByActor(state, actorId, index) {
function applyLocalChange (line 54) | function applyLocalChange(backend, change) {
function save (line 96) | function save(backend) {
function load (line 104) | function load(data) {
function loadChanges (line 116) | function loadChanges(backend, changes) {
function getPatch (line 127) | function getPatch(backend) {
function getHeads (line 135) | function getHeads(backend) {
function getAllChanges (line 142) | function getAllChanges(backend) {
function getChanges (line 151) | function getChanges(backend, haveDeps) {
function getChangesAdded (line 166) | function getChangesAdded(backend1, backend2) {
function getChangeByHash (line 176) | function getChangeByHash(backend, hash) {
function getMissingDeps (line 190) | function getMissingDeps(backend, heads = []) {
FILE: backend/columnar.js
constant MAGIC_BYTES (line 24) | const MAGIC_BYTES = new Uint8Array([0x85, 0x6f, 0x4a, 0x83])
constant CHUNK_TYPE_DOCUMENT (line 26) | const CHUNK_TYPE_DOCUMENT = 0
constant CHUNK_TYPE_CHANGE (line 27) | const CHUNK_TYPE_CHANGE = 1
constant CHUNK_TYPE_DEFLATE (line 28) | const CHUNK_TYPE_DEFLATE = 2 // like CHUNK_TYPE_CHANGE but with DEFLATE ...
constant DEFLATE_MIN_SIZE (line 32) | const DEFLATE_MIN_SIZE = 256
constant COLUMN_TYPE (line 35) | const COLUMN_TYPE = {
constant COLUMN_TYPE_DEFLATE (line 41) | const COLUMN_TYPE_DEFLATE = 8
constant VALUE_TYPE (line 46) | const VALUE_TYPE = {
constant ACTIONS (line 52) | const ACTIONS = ['makeMap', 'set', 'makeList', 'del', 'makeText', 'inc',...
constant OBJECT_TYPE (line 54) | const OBJECT_TYPE = {makeMap: 'map', makeList: 'list', makeText: 'text',...
constant COMMON_COLUMNS (line 56) | const COMMON_COLUMNS = [
constant CHANGE_COLUMNS (line 72) | const CHANGE_COLUMNS = COMMON_COLUMNS.concat([
constant DOC_OPS_COLUMNS (line 78) | const DOC_OPS_COLUMNS = COMMON_COLUMNS.concat([
constant DOCUMENT_COLUMNS (line 84) | const DOCUMENT_COLUMNS = [
function actorIdToActorNum (line 101) | function actorIdToActorNum(opId, actorIds) {
function compareParsedOpIds (line 114) | function compareParsedOpIds(id1, id2) {
function parseAllOpIds (line 133) | function parseAllOpIds(changes, single) {
function encodeObjectId (line 176) | function encodeObjectId(op, columns) {
function encodeOperationKey (line 192) | function encodeOperationKey(op, columns) {
function encodeOperationAction (line 213) | function encodeOperationAction(op, columns) {
function getNumberTypeAndValue (line 228) | function getNumberTypeAndValue(op) {
function encodeValue (line 259) | function encodeValue(op, columns) {
function decodeValue (line 300) | function decodeValue(sizeTag, bytes) {
function decodeValueColumns (line 339) | function decodeValueColumns(columns, colIndex, actorIds, result) {
function encodeOps (line 370) | function encodeOps(ops, forDocument) {
function validDatatype (line 438) | function validDatatype(value, datatype) {
function expandMultiOps (line 446) | function expandMultiOps(ops, startOp, actor) {
function decodeOps (line 483) | function decodeOps(ops, forDocument) {
function checkSortedOpIds (line 515) | function checkSortedOpIds(opIds) {
function encoderByColumnId (line 525) | function encoderByColumnId(columnId) {
function decoderByColumnId (line 539) | function decoderByColumnId(columnId, buffer) {
function makeDecoders (line 553) | function makeDecoders(columns, columnSpec) {
function decodeColumns (line 577) | function decodeColumns(columns, actorIds, columnSpec) {
function decodeColumnInfo (line 609) | function decodeColumnInfo(decoder) {
function encodeColumnInfo (line 626) | function encodeColumnInfo(encoder, columns) {
function decodeChangeHeader (line 635) | function decodeChangeHeader(decoder) {
function encodeContainer (line 659) | function encodeContainer(chunkType, encodeContentsCallback) {
function decodeContainerHeader (line 688) | function decodeContainerHeader(decoder, computeHash) {
function encodeChange (line 710) | function encodeChange(changeObj) {
function decodeChangeColumns (line 741) | function decodeChangeColumns(buffer) {
function decodeChange (line 770) | function decodeChange(buffer) {
function decodeChangeMeta (line 783) | function decodeChangeMeta(buffer, computeHash) {
function deflateChange (line 798) | function deflateChange(buffer) {
function inflateChange (line 813) | function inflateChange(buffer) {
function splitContainers (line 829) | function splitContainers(buffer) {
function decodeChanges (line 843) | function decodeChanges(binaryChanges) {
function sortOpIds (line 859) | function sortOpIds(a, b) {
function groupChangeOps (line 876) | function groupChangeOps(changes, ops) {
function decodeDocumentChanges (line 945) | function decodeDocumentChanges(changes, expectedHeads) {
function encodeDocumentHeader (line 983) | function encodeDocumentHeader(doc) {
function decodeDocumentHeader (line 1006) | function decodeDocumentHeader(buffer) {
function decodeDocument (line 1040) | function decodeDocument(buffer) {
function deflateColumn (line 1052) | function deflateColumn(column) {
function inflateColumn (line 1062) | function inflateColumn(column) {
FILE: backend/encoding.js
function stringToUtf8 (line 11) | function stringToUtf8(string) {
function utf8ToString (line 15) | function utf8ToString(buffer) {
function hexStringToBytes (line 22) | function hexStringToBytes(value) {
constant NIBBLE_TO_HEX (line 36) | const NIBBLE_TO_HEX = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9',...
constant BYTE_TO_HEX (line 37) | const BYTE_TO_HEX = new Array(256)
function bytesToHexString (line 45) | function bytesToHexString(bytes) {
class Encoder (line 57) | class Encoder {
method constructor (line 58) | constructor() {
method buffer (line 66) | get buffer() {
method grow (line 74) | grow(minSize = 0) {
method appendByte (line 86) | appendByte(value) {
method appendUint32 (line 97) | appendUint32(value) {
method appendInt32 (line 117) | appendInt32(value) {
method appendUint53 (line 137) | appendUint53(value) {
method appendInt53 (line 152) | appendInt53(value) {
method appendUint64 (line 168) | appendUint64(high32, low32) {
method appendInt64 (line 201) | appendInt64(high32, low32) {
method appendRawBytes (line 232) | appendRawBytes(data) {
method appendRawString (line 245) | appendRawString(value) {
method appendPrefixedBytes (line 254) | appendPrefixedBytes(data) {
method appendPrefixedString (line 264) | appendPrefixedString(value) {
method appendHexString (line 275) | appendHexString(value) {
method finish (line 284) | finish() {
class Decoder (line 293) | class Decoder {
method constructor (line 294) | constructor(buffer) {
method done (line 306) | get done() {
method reset (line 314) | reset() {
method skip (line 322) | skip(bytes) {
method readByte (line 332) | readByte() {
method readUint32 (line 341) | readUint32() {
method readInt32 (line 360) | readInt32() {
method readUint53 (line 389) | readUint53() {
method readInt53 (line 402) | readInt53() {
method readUint64 (line 416) | readUint64() {
method readInt64 (line 450) | readInt64() {
method readRawBytes (line 494) | readRawBytes(length) {
method readRawString (line 507) | readRawString(length) {
method readPrefixedBytes (line 515) | readPrefixedBytes() {
method readPrefixedString (line 523) | readPrefixedString() {
method readHexString (line 531) | readHexString() {
class RLEEncoder (line 558) | class RLEEncoder extends Encoder {
method constructor (line 559) | constructor(type) {
method appendValue (line 572) | appendValue(value, repetitions = 1) {
method _appendValue (line 579) | _appendValue(value, repetitions = 1) {
method copyFrom (line 667) | copyFrom(decoder, options = {}) {
method flush (line 742) | flush() {
method appendRawValue (line 762) | appendRawValue(value) {
method finish (line 778) | finish() {
class RLEDecoder (line 789) | class RLEDecoder extends Decoder {
method constructor (line 790) | constructor(type, buffer) {
method done (line 802) | get done() {
method reset (line 810) | reset() {
method readValue (line 820) | readValue() {
method skipValues (line 837) | skipValues(numSkip) {
method readRecord (line 865) | readRecord() {
method readRawValue (line 893) | readRawValue() {
method skipRawValues (line 909) | skipRawValues(num) {
class DeltaEncoder (line 932) | class DeltaEncoder extends RLEEncoder {
method constructor (line 933) | constructor() {
method appendValue (line 942) | appendValue(value, repetitions = 1) {
method copyFrom (line 958) | copyFrom(decoder, options = {}) {
class DeltaDecoder (line 1004) | class DeltaDecoder extends RLEDecoder {
method constructor (line 1005) | constructor(buffer) {
method reset (line 1014) | reset() {
method readValue (line 1025) | readValue() {
method skipValues (line 1035) | skipValues(numSkip) {
class BooleanEncoder (line 1061) | class BooleanEncoder extends Encoder {
method constructor (line 1062) | constructor() {
method appendValue (line 1072) | appendValue(value, repetitions = 1) {
method copyFrom (line 1091) | copyFrom(decoder, options = {}) {
method finish (line 1129) | finish() {
class BooleanDecoder (line 1141) | class BooleanDecoder extends Decoder {
method constructor (line 1142) | constructor(buffer) {
method done (line 1153) | get done() {
method reset (line 1161) | reset() {
method readValue (line 1171) | readValue() {
method skipValues (line 1188) | skipValues(numSkip) {
FILE: backend/new.js
constant MAX_BLOCK_SIZE (line 6) | const MAX_BLOCK_SIZE = 600 // operations
constant BLOOM_BITS_PER_ENTRY (line 7) | const BLOOM_BITS_PER_ENTRY = 10, BLOOM_NUM_PROBES = 7 // 1% false positi...
constant BLOOM_NUM_PROBES (line 7) | const BLOOM_BITS_PER_ENTRY = 10, BLOOM_NUM_PROBES = 7 // 1% false positi...
constant BLOOM_FILTER_SIZE (line 8) | const BLOOM_FILTER_SIZE = Math.floor(BLOOM_BITS_PER_ENTRY * MAX_BLOCK_SI...
constant PRED_COLUMN_IDS (line 14) | const PRED_COLUMN_IDS = CHANGE_COLUMNS
function deepCopyUpdate (line 24) | function deepCopyUpdate(objectTree, path, value) {
function seekWithinBlock (line 50) | function seekWithinBlock(ops, docCols, actorIds, resumeInsertion) {
function visibleListElements (line 199) | function visibleListElements(docState, blockIndex, objActorNum, objCtr) {
function seekToOp (line 227) | function seekToOp(docState, ops) {
function bloomFilterAdd (line 329) | function bloomFilterAdd(bloom, elemIdActor, elemIdCtr) {
function bloomFilterContains (line 351) | function bloomFilterContains(bloom, elemIdActor, elemIdCtr) {
function updateBlockMetadata (line 370) | function updateBlockMetadata(block) {
function addBlockOperation (line 426) | function addBlockOperation(block, op, actorIds, isChangeOp) {
function splitBlock (line 465) | function splitBlock(block) {
function concatBlocks (line 496) | function concatBlocks(blocks) {
function copyColumns (line 513) | function copyColumns(outCols, inCols, count) {
function readOperation (line 570) | function readOperation(columns, actorTable) {
function appendOperation (line 617) | function appendOperation(outCols, inCols, operation) {
function readNextDocOp (line 658) | function readNextDocOp(docState, blockIndex) {
function readNextChangeOp (line 678) | function readNextChangeOp(docState, changeState) {
function emptyObjectPatch (line 726) | function emptyObjectPatch(objectId, type) {
function opIdDelta (line 738) | function opIdDelta(id1, id2, delta = 1) {
function appendEdit (line 747) | function appendEdit(existingEdits, nextEdit) {
function appendUpdate (line 798) | function appendUpdate(edits, index, elemId, opId, value, firstUpdate) {
function convertInsertToUpdate (line 838) | function convertInsertToUpdate(edits, index, elemId) {
function updatePatchProperty (line 884) | function updatePatchProperty(patches, newBlock, objectId, op, docState, ...
function mergeDocChangeOps (line 1052) | function mergeDocChangeOps(patches, newBlock, outCols, changeState, docS...
function applyOps (line 1304) | function applyOps(patches, changeState, docState) {
function updateBlockColumns (line 1387) | function updateBlockColumns(docState, changeCols) {
function getActorTable (line 1434) | function getActorTable(actorIds, change) {
function setupPatches (line 1461) | function setupPatches(patches, objectIds, docState) {
function applyChanges (line 1550) | function applyChanges(patches, decodedChanges, docState, objectIds, thro...
function documentPatch (line 1604) | function documentPatch(docState) {
function readDocumentChanges (line 1645) | function readDocumentChanges(doc) {
function appendChange (line 1680) | function appendChange(columns, change, actorIds, changeIndexByHash) {
class BackendDoc (line 1694) | class BackendDoc {
method constructor (line 1695) | constructor(buffer) {
method clone (line 1773) | clone() {
method applyChanges (line 1797) | applyChanges(changeBuffers, isLocal = false) {
method computeHashGraph (line 1887) | computeHashGraph() {
method getChanges (line 1921) | getChanges(haveDeps) {
method getChangesAdded (line 1979) | getChangesAdded(other) {
method getChangeByHash (line 1999) | getChangeByHash(hash) {
method getMissingDeps (line 2014) | getMissingDeps(heads = []) {
method save (line 2033) | save() {
method getPatch (line 2060) | getPatch() {
FILE: backend/sync.js
constant HASH_SIZE (line 24) | const HASH_SIZE = 32 // 256 bits = 32 bytes
constant MESSAGE_TYPE_SYNC (line 25) | const MESSAGE_TYPE_SYNC = 0x42 // first byte of a sync message, for iden...
constant PEER_STATE_TYPE (line 26) | const PEER_STATE_TYPE = 0x43 // first byte of an encoded peer state, for...
constant BITS_PER_ENTRY (line 31) | const BITS_PER_ENTRY = 10, NUM_PROBES = 7
constant NUM_PROBES (line 31) | const BITS_PER_ENTRY = 10, NUM_PROBES = 7
class BloomFilter (line 38) | class BloomFilter {
method constructor (line 39) | constructor (arg) {
method bytes (line 68) | get bytes() {
method getProbes (line 88) | getProbes(hash) {
method addHash (line 107) | addHash(hash) {
method containsHash (line 116) | containsHash(hash) {
function encodeHashes (line 130) | function encodeHashes(encoder, hashes) {
function decodeHashes (line 145) | function decodeHashes(decoder) {
function encodeSyncMessage (line 157) | function encodeSyncMessage(message) {
function decodeSyncMessage (line 177) | function decodeSyncMessage(bytes) {
function encodeSyncState (line 206) | function encodeSyncState(syncState) {
function decodeSyncState (line 217) | function decodeSyncState(bytes) {
function makeBloomFilter (line 234) | function makeBloomFilter(backend, lastSync) {
function getChangesToSend (line 246) | function getChangesToSend(backend, have, need) {
function initSyncState (line 308) | function initSyncState() {
function compareArrays (line 319) | function compareArrays(a, b) {
function generateSyncMessage (line 327) | function generateSyncMessage(backend, syncState) {
function advanceHeads (line 408) | function advanceHeads(myOldHeads, myNewHeads, ourOldSharedHeads) {
function receiveSyncMessage (line 420) | function receiveSyncMessage(backend, oldSyncState, binaryMessage) {
FILE: backend/util.js
function backendState (line 1) | function backendState(backend) {
FILE: frontend/apply_patch.js
function getValue (line 10) | function getValue(patch, object, updated) {
function lamportCompare (line 33) | function lamportCompare(ts1, ts2) {
function applyProperties (line 57) | function applyProperties(props, object, conflicts, updated) {
function cloneMapObject (line 85) | function cloneMapObject(originalObject, objectId) {
function updateMapObject (line 98) | function updateMapObject(patch, obj, updated) {
function updateTableObject (line 114) | function updateTableObject(patch, obj, updated) {
function cloneListObject (line 141) | function cloneListObject(originalList, objectId) {
function updateListObject (line 156) | function updateListObject(patch, obj, updated) {
function updateTextObject (line 220) | function updateTextObject(patch, obj, updated) {
function interpretPatch (line 266) | function interpretPatch(patch, obj, updated) {
function cloneRootObject (line 289) | function cloneRootObject(root) {
FILE: frontend/constants.js
constant OPTIONS (line 2) | const OPTIONS = Symbol('_options') // object containing options pass...
constant CACHE (line 3) | const CACHE = Symbol('_cache') // map from objectId to immutable...
constant STATE (line 4) | const STATE = Symbol('_state') // object containing metadata abo...
constant OBJECT_ID (line 7) | const OBJECT_ID = Symbol('_objectId') // the object ID of the current o...
constant CONFLICTS (line 8) | const CONFLICTS = Symbol('_conflicts') // map or list (depending on obje...
constant CHANGE (line 9) | const CHANGE = Symbol('_change') // the context object on proxy ob...
constant ELEM_IDS (line 10) | const ELEM_IDS = Symbol('_elemIds') // list containing the element ID...
FILE: frontend/context.js
class Context (line 16) | class Context {
method constructor (line 17) | constructor (doc, actorId, applyPatch) {
method addOp (line 29) | addOp(operation) {
method nextOpId (line 44) | nextOpId() {
method getValueDescription (line 51) | getValueDescription(value) {
method getValuesDescriptions (line 100) | getValuesDescriptions(path, object, key) {
method getPropertyValue (line 128) | getPropertyValue(object, key, opId) {
method getSubpatch (line 142) | getSubpatch(patch, path) {
method getObject (line 178) | getObject(objectId) {
method getObjectType (line 188) | getObjectType(objectId) {
method getObjectField (line 201) | getObjectField(path, objectId, key) {
method createNestedObjects (line 230) | createNestedObjects(obj, key, value, insert, pred, elemId) {
method setValue (line 289) | setValue(objectId, key, value, insert, pred, elemId) {
method applyAtPath (line 315) | applyAtPath(path, callback) {
method setMapKey (line 325) | setMapKey(path, key, value) {
method deleteMapKey (line 351) | deleteMapKey(path, key) {
method insertListItems (line 370) | insertListItems(subpatch, index, values, newObject) {
method setListIndex (line 411) | setListIndex(path, index, value) {
method splice (line 441) | splice(path, start, deletions, insertions) {
method addTableRow (line 508) | addTableRow(path, row) {
method deleteTableRow (line 531) | deleteTableRow(path, rowId, pred) {
method increment (line 546) | increment(path, key, delta) {
function getPred (line 576) | function getPred(object, key) {
function getElemId (line 588) | function getElemId(list, index, insert = false) {
FILE: frontend/counter.js
class Counter (line 6) | class Counter {
method constructor (line 7) | constructor(value) {
method valueOf (line 20) | valueOf() {
method toString (line 29) | toString() {
method toJSON (line 37) | toJSON() {
class WriteableCounter (line 46) | class WriteableCounter extends Counter {
method increment (line 51) | increment(delta) {
method decrement (line 62) | decrement(delta) {
function getWriteableCounter (line 74) | function getWriteableCounter(value, context, path, objectId, key) {
FILE: frontend/index.js
function checkActorId (line 17) | function checkActorId(actorId) {
function updateRootObject (line 34) | function updateRootObject(doc, updated, state) {
function makeChange (line 78) | function makeChange(doc, context, options) {
function countOps (line 120) | function countOps(ops) {
function getLastLocalChange (line 135) | function getLastLocalChange(doc) {
function applyPatchToDoc (line 146) | function applyPatchToDoc(doc, patch, state, fromBackend) {
function init (line 166) | function init(options) {
function from (line 207) | function from(initialState, options) {
function change (line 224) | function change(doc, options, callback) {
function emptyChange (line 264) | function emptyChange(doc, options) {
function applyPatch (line 288) | function applyPatch(doc, patch, backendState = undefined) {
function getObjectId (line 332) | function getObjectId(object) {
function getObjectById (line 341) | function getObjectById(doc, objectId) {
function getActorId (line 355) | function getActorId(doc) {
function setActorId (line 363) | function setActorId(doc, actorId) {
function getConflicts (line 374) | function getConflicts(object, key) {
function getBackendState (line 385) | function getBackendState(doc, callerName = null, argPos = 'first') {
function getElementIds (line 403) | function getElementIds(list) {
FILE: frontend/numbers.js
class Int (line 3) | class Int {
method constructor (line 4) | constructor(value) {
class Uint (line 13) | class Uint {
method constructor (line 14) | constructor(value) {
class Float64 (line 23) | class Float64 {
method constructor (line 24) | constructor(value) {
FILE: frontend/observable.js
class Observable (line 9) | class Observable {
method constructor (line 10) | constructor() {
method patchCallback (line 21) | patchCallback(patch, before, after, local, changes) {
method _objectUpdate (line 29) | _objectUpdate(diff, before, after, local, changes) {
method observe (line 106) | observe(object, callback) {
FILE: frontend/proxies.js
function parseListIndex (line 6) | function parseListIndex(key) {
function listMethods (line 17) | function listMethods(context, listId, path) {
method get (line 115) | get (target, key) {
method set (line 123) | set (target, key, value) {
method deleteProperty (line 132) | deleteProperty (target, key) {
method has (line 141) | has (target, key) {
method getOwnPropertyDescriptor (line 146) | getOwnPropertyDescriptor (target, key) {
method ownKeys (line 157) | ownKeys (target) {
method get (line 164) | get (target, key) {
method set (line 176) | set (target, key, value) {
method deleteProperty (line 194) | deleteProperty (target, key) {
method has (line 200) | has (target, key) {
method getOwnPropertyDescriptor (line 208) | getOwnPropertyDescriptor (target, key) {
method ownKeys (line 224) | ownKeys (target) {
function mapProxy (line 233) | function mapProxy(context, objectId, path, readonly) {
function listProxy (line 237) | function listProxy(context, objectId, path) {
function instantiateProxy (line 247) | function instantiateProxy(path, objectId, readonly) {
function rootObjectProxy (line 258) | function rootObjectProxy(context) {
FILE: frontend/table.js
function compareRows (line 4) | function compareRows(properties, row1, row2) {
class Table (line 25) | class Table {
method constructor (line 30) | constructor() {
method byId (line 39) | byId(id) {
method ids (line 47) | get ids() {
method count (line 57) | get count() {
method rows (line 65) | get rows() {
method filter (line 73) | filter(callback, thisArg) {
method find (line 81) | find(callback, thisArg) {
method map (line 89) | map(callback, thisArg) {
method sort (line 103) | sort(arg) {
method _clone (line 140) | _clone() {
method _set (line 152) | _set(id, value, opId) {
method remove (line 166) | remove(id) {
method _freeze (line 177) | _freeze() {
method getWriteable (line 188) | getWriteable(context, path) {
method toJSON (line 206) | toJSON() {
method [Symbol.iterator] (line 121) | [Symbol.iterator] () {
class WriteableTable (line 217) | class WriteableTable extends Table {
method byId (line 222) | byId(id) {
method add (line 234) | add(row) {
method remove (line 242) | remove(id) {
function instantiateTable (line 255) | function instantiateTable(objectId, entries, opIds) {
FILE: frontend/text.js
class Text (line 4) | class Text {
method constructor (line 5) | constructor (text) {
method length (line 19) | get length () {
method get (line 23) | get (index) {
method getElemId (line 34) | getElemId (index) {
method toString (line 60) | toString() {
method toSpans (line 78) | toSpans() {
method toJSON (line 102) | toJSON() {
method getWriteable (line 111) | getWriteable(context, path) {
method set (line 125) | set (index, value) {
method insertAt (line 139) | insertAt(index, ...values) {
method deleteAt (line 154) | deleteAt(index, numDelete = 1) {
method [Symbol.iterator] (line 42) | [Symbol.iterator] () {
function instantiateText (line 176) | function instantiateText(objectId, elems) {
FILE: src/automerge.js
function init (line 14) | function init(options) {
function from (line 28) | function from(initialState, options) {
function change (line 33) | function change(doc, options, callback) {
function emptyChange (line 38) | function emptyChange(doc, options) {
function clone (line 43) | function clone(doc, options = {}) {
function free (line 48) | function free(doc) {
function load (line 52) | function load(data, options = {}) {
function save (line 57) | function save(doc) {
function merge (line 61) | function merge(localDoc, remoteDoc) {
function getChanges (line 69) | function getChanges(oldDoc, newDoc) {
function getAllChanges (line 75) | function getAllChanges(doc) {
function applyPatch (line 79) | function applyPatch(doc, patch, backendState, changes, options) {
function applyChanges (line 88) | function applyChanges(doc, changes, options = {}) {
function equals (line 94) | function equals(val1, val2) {
function getHistory (line 105) | function getHistory(doc) {
function generateSyncMessage (line 120) | function generateSyncMessage(doc, syncState) {
function receiveSyncMessage (line 125) | function receiveSyncMessage(doc, oldSyncState, message) {
function initSyncState (line 139) | function initSyncState() {
function setDefaultBackend (line 147) | function setDefaultBackend(newBackend) {
method Backend (line 156) | get Backend() { return backend }
FILE: src/common.js
function isObject (line 1) | function isObject(obj) {
function copyObject (line 9) | function copyObject(obj) {
function parseOpId (line 22) | function parseOpId(opId) {
function equalBytes (line 33) | function equalBytes(array1, array2) {
function createArrayOfNulls (line 47) | function createArrayOfNulls(length) {
FILE: src/uuid.js
function defaultFactory (line 3) | function defaultFactory() {
function makeUuid (line 9) | function makeUuid() {
FILE: test/backend_test.js
function hash (line 8) | function hash(change) {
FILE: test/encoding_test.js
function encode (line 9) | function encode(value) {
function encode (line 49) | function encode(value) {
function encode (line 124) | function encode(value) {
function encode (line 165) | function encode(value) {
function encode (line 260) | function encode(high32, low32) {
function encode (line 316) | function encode(high32, low32) {
function encodeRLE (line 508) | function encodeRLE(type, values) {
function decodeRLE (line 514) | function decodeRLE(type, buffer) {
function doCopy (line 629) | function doCopy(input1, input2, options = {}) {
function encodeDelta (line 752) | function encodeDelta(values) {
function decodeDelta (line 758) | function decodeDelta(buffer) {
function doCopy (line 803) | function doCopy(input1, input2, options = {}) {
function encodeBools (line 895) | function encodeBools(values) {
function decodeBools (line 901) | function decodeBools(buffer) {
function doCopy (line 960) | function doCopy(input1, input2, options = {}) {
FILE: test/frontend_test.js
constant UUID_PATTERN (line 7) | const UUID_PATTERN = /^[0-9a-f]{32}$/
function getRequests (line 242) | function getRequests(doc) {
FILE: test/fuzz_test.js
class Micromerge (line 12) | class Micromerge {
method constructor (line 13) | constructor() {
method root (line 19) | get root() {
method applyChange (line 27) | applyChange(change) {
method applyOp (line 50) | applyOp(op) {
method applyListInsert (line 79) | applyListInsert(op) {
method applyListUpdate (line 97) | applyListUpdate(op) {
method findListElement (line 116) | findListElement(objectId, elemId) {
method compareOpIds (line 131) | compareOpIds(id1, id2) {
FILE: test/helpers.js
function assertEqualsOneOf (line 6) | function assertEqualsOneOf(actual, ...expected) {
function checkEncoded (line 22) | function checkEncoded(encoder, bytes, detail) {
FILE: test/new_backend_test.js
function checkColumns (line 7) | function checkColumns(block, expectedCols) {
function hash (line 24) | function hash(change) {
FILE: test/proxies_test.js
constant UUID_PATTERN (line 4) | const UUID_PATTERN = /^[0-9a-f]{32}$/
FILE: test/sync_test.js
function getHeads (line 7) | function getHeads(doc) {
function getMissingDeps (line 11) | function getMissingDeps(doc) {
function sync (line 15) | function sync(a, b, aSyncState = initSyncState(), bSyncState = initSyncS...
FILE: test/table_test.js
constant DDIA (line 8) | const DDIA = {
constant RSDP (line 13) | const RSDP = {
FILE: test/test.js
constant UUID_PATTERN (line 5) | const UUID_PATTERN = /^[0-9a-f]{32}$/
constant OPID_PATTERN (line 6) | const OPID_PATTERN = /^[0-9]+@[0-9a-f]{32}$/
method patchCallback (line 1266) | patchCallback(patch, before, after, local) {
method patchCallback (line 1462) | patchCallback(patch, before, after, local) {
FILE: test/text_test.js
function attributeStateToAttributes (line 5) | function attributeStateToAttributes(accumulatedAttributes) {
function isEquivalent (line 15) | function isEquivalent(a, b) {
function isControlMarker (line 33) | function isControlMarker(pseudoCharacter) {
function opFrom (line 37) | function opFrom(text, attributes) {
function accumulateAttributes (line 45) | function accumulateAttributes(span, accumulatedAttributes) {
function automergeTextToDeltaDoc (line 67) | function automergeTextToDeltaDoc(text) {
function inverseAttributes (line 112) | function inverseAttributes(attributes) {
function applyDeleteOp (line 120) | function applyDeleteOp(text, offset, op) {
function applyRetainOp (line 134) | function applyRetainOp(text, offset, op) {
function applyInsertOp (line 159) | function applyInsertOp(text, offset, op) {
function applyDeltaDocToAutomergeText (line 183) | function applyDeltaDocToAutomergeText(delta, doc) {
FILE: test/typescript_test.ts
constant UUID_PATTERN (line 5) | const UUID_PATTERN = /^[0-9a-f]{32}$/
type BirdList (line 7) | interface BirdList {
type NumberBox (line 11) | interface NumberBox {
type TextDoc (line 339) | interface TextDoc {
type Book (line 407) | interface Book {
type BookDb (line 413) | interface BookDb {
type BookDeluxe (line 454) | interface BookDeluxe extends Book {
type BookDeluxeDb (line 458) | interface BookDeluxeDb {
type CounterMap (line 500) | interface CounterMap {
type CounterList (line 504) | interface CounterList {
type BirdCounterMap (line 508) | interface BirdCounterMap {
type TextDoc (line 582) | interface TextDoc {
FILE: test/uuid_test.js
function customUuid (line 20) | function customUuid() {
FILE: test/wasm.js
function interopTests (line 37) | function interopTests(sourceBackend, destBackend) {
Condensed preview — 51 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (904K chars).
[
{
"path": ".babelrc",
"chars": 61,
"preview": "{\n \"presets\": [\n [\n \"@babel/preset-env\"\n ]\n ]\n}\n"
},
{
"path": ".eslintrc.json",
"chars": 8876,
"preview": "{\n \"env\": {\n \"browser\": true,\n \"commonjs\": true,\n \"es2015\": true,\n \"node\": true,\n "
},
{
"path": ".github/workflows/automerge-ci.yml",
"chars": 2358,
"preview": "name: CI\non: [push, pull_request]\n\njobs:\n node-build:\n runs-on: ubuntu-latest\n strategy:\n matrix:\n no"
},
{
"path": ".gitignore",
"chars": 50,
"preview": "/coverage\n/dist\n/node_modules\n.nyc_output\n.vscode\n"
},
{
"path": ".mocharc.yaml",
"chars": 222,
"preview": "use_strict: true\nrequire:\n - ts-node/register\n - tsconfig-paths/register\nwatch-files:\n - 'src/*.js'\n - 'frontend/*.j"
},
{
"path": "@types/automerge/index.d.ts",
"chars": 12985,
"preview": "declare module 'automerge' {\n /**\n * The return type of `Automerge.init<T>()`, `Automerge.change<T>()`, etc. where `T"
},
{
"path": "CHANGELOG.md",
"chars": 24154,
"preview": "# Changelog\n\nAutomerge adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html) for assigning\nversion number"
},
{
"path": "LICENSE",
"chars": 1118,
"preview": "Copyright (c) 2017-present Martin Kleppmann, Ink & Switch LLC, and the Automerge contributors\n\nPermission is hereby gran"
},
{
"path": "README.md",
"chars": 1898,
"preview": "<img src='./img/sign.svg' width='500' alt='Automerge logo' />\n\n## Deprecation Notice\n\nAutomerge now has a shiny new impl"
},
{
"path": "backend/backend.js",
"chars": 7405,
"preview": "const { encodeChange } = require('./columnar')\nconst { BackendDoc } = require('./new')\nconst { backendState } = require("
},
{
"path": "backend/columnar.js",
"chars": 42456,
"preview": "const pako = require('pako')\nconst { copyObject, parseOpId, equalBytes } = require('../src/common')\nconst {\n utf8ToStri"
},
{
"path": "backend/encoding.js",
"chars": 41472,
"preview": "/**\n * UTF-8 decoding and encoding using API that is supported in Node >= 12 and modern browsers:\n * https://developer.m"
},
{
"path": "backend/index.js",
"chars": 690,
"preview": "const { init, clone, free, applyChanges, applyLocalChange, save, load, loadChanges, getPatch, getHeads, getAllChanges, g"
},
{
"path": "backend/new.js",
"chars": 98256,
"preview": "const { parseOpId, copyObject } = require('../src/common')\nconst { COLUMN_TYPE, VALUE_TYPE, ACTIONS, OBJECT_TYPE, DOC_OP"
},
{
"path": "backend/sync.js",
"chars": 20367,
"preview": "/**\n * Implementation of the data synchronisation protocol that brings a local and a remote document\n * into the same st"
},
{
"path": "backend/util.js",
"chars": 375,
"preview": "function backendState(backend) {\n if (backend.frozen) {\n throw new Error(\n 'Attempting to use an outdated Autom"
},
{
"path": "frontend/apply_patch.js",
"chars": 11129,
"preview": "const { isObject, copyObject, parseOpId } = require('../src/common')\nconst { OBJECT_ID, CONFLICTS, ELEM_IDS } = require("
},
{
"path": "frontend/constants.js",
"chars": 823,
"preview": "// Properties of the document root object\nconst OPTIONS = Symbol('_options') // object containing options passed to "
},
{
"path": "frontend/context.js",
"chars": 24956,
"preview": "const { CACHE, OBJECT_ID, CONFLICTS, ELEM_IDS, STATE } = require('./constants')\nconst { interpretPatch } = require('./ap"
},
{
"path": "frontend/counter.js",
"chars": 2586,
"preview": "/**\n * The most basic CRDT: an integer value that can be changed only by\n * incrementing and decrementing. Since additio"
},
{
"path": "frontend/index.js",
"chars": 14633,
"preview": "const { OPTIONS, CACHE, STATE, OBJECT_ID, CONFLICTS, CHANGE, ELEM_IDS } = require('./constants')\nconst { isObject, copyO"
},
{
"path": "frontend/numbers.js",
"chars": 843,
"preview": "// Convience classes to allow users to stricly specify the number type they want\n\nclass Int {\n constructor(value) {\n "
},
{
"path": "frontend/observable.js",
"chars": 4746,
"preview": "const { OBJECT_ID, CONFLICTS } = require('./constants')\n\n/**\n * Allows an application to register a callback when a part"
},
{
"path": "frontend/proxies.js",
"chars": 8340,
"preview": "const { OBJECT_ID, CHANGE, STATE } = require('./constants')\nconst { isObject, createArrayOfNulls } = require('../src/com"
},
{
"path": "frontend/table.js",
"chars": 7796,
"preview": "const { OBJECT_ID, CONFLICTS } = require('./constants')\nconst { isObject, copyObject } = require('../src/common')\n\nfunct"
},
{
"path": "frontend/text.js",
"chars": 5172,
"preview": "const { OBJECT_ID } = require('./constants')\nconst { isObject } = require('../src/common')\n\nclass Text {\n constructor ("
},
{
"path": "karma.conf.js",
"chars": 1347,
"preview": "const path = require('path')\nconst webpack = require('webpack')\nconst webpackConfig = require('./webpack.config.js')\n\n//"
},
{
"path": "karma.sauce.js",
"chars": 2481,
"preview": "const path = require('path')\nconst webpack = require('webpack')\nconst webpackConfig = require(\"./webpack.config.js\")\n\n//"
},
{
"path": "package.json",
"chars": 2180,
"preview": "{\n \"name\": \"automerge\",\n \"version\": \"1.0.1-preview.7\",\n \"description\": \"Data structures for building collaborative ap"
},
{
"path": "src/automerge.js",
"chars": 5560,
"preview": "const uuid = require('./uuid')\nconst Frontend = require('../frontend')\nconst { OPTIONS } = require('../frontend/constant"
},
{
"path": "src/common.js",
"chars": 1531,
"preview": "function isObject(obj) {\n return typeof obj === 'object' && obj !== null\n}\n\n/**\n * Returns a shallow copy of the object"
},
{
"path": "src/uuid.js",
"chars": 317,
"preview": "const { v4: uuid } = require('uuid')\n\nfunction defaultFactory() {\n return uuid().replace(/-/g, '')\n}\n\nlet factory = def"
},
{
"path": "test/backend_test.js",
"chars": 69722,
"preview": "/* eslint-disable no-unused-vars */\nconst assert = require('assert')\nconst Automerge = process.env.TEST_DIST === '1' ? r"
},
{
"path": "test/columnar_test.js",
"chars": 4460,
"preview": "const assert = require('assert')\nconst { checkEncoded } = require('./helpers')\nconst Automerge = process.env.TEST_DIST ="
},
{
"path": "test/context_test.js",
"chars": 21206,
"preview": "const assert = require('assert')\nconst sinon = require('sinon')\nconst { Context } = require('../frontend/context')\nconst"
},
{
"path": "test/encoding_test.js",
"chars": 58565,
"preview": "const assert = require('assert')\nconst { checkEncoded } = require('./helpers')\nconst { Encoder, Decoder, RLEEncoder, RLE"
},
{
"path": "test/frontend_test.js",
"chars": 36390,
"preview": "const assert = require('assert')\nconst Frontend = require('../frontend')\nconst { decodeChange } = require('../backend/co"
},
{
"path": "test/fuzz_test.js",
"chars": 7963,
"preview": "/**\n * Miniature implementation of a subset of Automerge, which is used below as definition of the\n * expected behaviour"
},
{
"path": "test/helpers.js",
"chars": 1190,
"preview": "const assert = require('assert')\nconst { Encoder } = require('../backend/encoding')\n\n// Assertion that succeeds if the f"
},
{
"path": "test/new_backend_test.js",
"chars": 114926,
"preview": "const assert = require('assert')\nconst { checkEncoded } = require('./helpers')\nconst { DOC_OPS_COLUMNS, encodeChange, de"
},
{
"path": "test/observable_test.js",
"chars": 7494,
"preview": "const assert = require('assert')\nconst Automerge = process.env.TEST_DIST === '1' ? require('../dist/automerge') : requir"
},
{
"path": "test/proxies_test.js",
"chars": 18910,
"preview": "const assert = require('assert')\nconst Automerge = process.env.TEST_DIST === '1' ? require('../dist/automerge') : requir"
},
{
"path": "test/sync_test.js",
"chars": 42220,
"preview": "const assert = require('assert')\nconst Automerge = process.env.TEST_DIST === '1' ? require('../dist/automerge') : requir"
},
{
"path": "test/table_test.js",
"chars": 7114,
"preview": "const assert = require('assert')\nconst Automerge = process.env.TEST_DIST === '1' ? require('../dist/automerge') : requir"
},
{
"path": "test/test.js",
"chars": 70654,
"preview": "const assert = require('assert')\nconst Automerge = process.env.TEST_DIST === '1' ? require('../dist/automerge') : requir"
},
{
"path": "test/text_test.js",
"chars": 22845,
"preview": "const assert = require('assert')\nconst Automerge = process.env.TEST_DIST === '1' ? require('../dist/automerge') : requir"
},
{
"path": "test/typescript_test.ts",
"chars": 24965,
"preview": "import * as assert from 'assert'\nimport * as Automerge from 'automerge'\nimport { Backend, Frontend, Counter, Doc } from "
},
{
"path": "test/uuid_test.js",
"chars": 743,
"preview": "const assert = require('assert')\nconst Automerge = process.env.TEST_DIST === '1' ? require('../dist/automerge') : requir"
},
{
"path": "test/wasm.js",
"chars": 12427,
"preview": "/* eslint-disable no-unused-vars */\n// This file is used for running the test suite against an alternative backend\n// im"
},
{
"path": "tsconfig.json",
"chars": 396,
"preview": "{\n \"compilerOptions\": {\n \"allowJs\": false,\n \"baseUrl\": \".\",\n \"esModuleInterop\": true,\n \"lib\": [\"dom\", \"esne"
},
{
"path": "webpack.config.js",
"chars": 484,
"preview": "const path = require('path')\n\nmodule.exports = {\n entry: './src/automerge.js',\n mode: 'development',\n output: {\n f"
}
]
About this extraction
This page contains the full source code of the automerge/automerge-classic GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 51 files (859.2 KB), approximately 261.7k tokens, and a symbol index with 498 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.