Showing preview only (2,618K chars total). Download the full file or copy to clipboard to get everything.
Repository: attic-labs/noms
Branch: master
Commit: e5fa29d95e8b
Files: 339
Total size: 2.4 MB
Directory structure:
gitextract_tbm5d7az/
├── .dockerignore
├── .gitignore
├── .travis.yml
├── CONTRIBUTING.md
├── Dockerfile
├── HACKING.md
├── LICENSE
├── README.md
├── cmd/
│ └── util/
│ └── kingpin_command.go
├── codecov.yml
├── doc/
│ ├── cli-tour.md
│ ├── decent/
│ │ ├── about.md
│ │ ├── architectures.md
│ │ ├── demo-ipfs-chat.md
│ │ ├── demo-p2p-chat.md
│ │ └── quickstart.md
│ ├── faq.md
│ ├── go-tour.md
│ ├── intro.md
│ └── spelling.md
├── go/
│ ├── chunks/
│ │ ├── chunk.go
│ │ ├── chunk_serializer.go
│ │ ├── chunk_serializer_test.go
│ │ ├── chunk_store.go
│ │ ├── chunk_store_common_test.go
│ │ ├── chunk_test.go
│ │ ├── memory_store.go
│ │ ├── memory_store_test.go
│ │ ├── put_cache.go
│ │ ├── remote_requests.go
│ │ ├── remote_requests_test.go
│ │ └── test_utils.go
│ ├── config/
│ │ ├── config.go
│ │ ├── config_test.go
│ │ ├── resolver.go
│ │ └── resolver_test.go
│ ├── constants/
│ │ ├── http.go
│ │ └── version.go
│ ├── d/
│ │ ├── check_error.go
│ │ ├── try.go
│ │ └── try_test.go
│ ├── datas/
│ │ ├── commit.go
│ │ ├── commit_options.go
│ │ ├── commit_test.go
│ │ ├── database.go
│ │ ├── database_common.go
│ │ ├── database_server.go
│ │ ├── database_test.go
│ │ ├── dataset.go
│ │ ├── dataset_test.go
│ │ ├── http_chunk_store.go
│ │ ├── http_chunk_store_test.go
│ │ ├── pull.go
│ │ ├── pull_test.go
│ │ ├── pulling.md
│ │ ├── remote_database_handlers.go
│ │ ├── remote_database_handlers_test.go
│ │ ├── serialize_hashes.go
│ │ └── serialize_hashes_test.go
│ ├── diff/
│ │ ├── apply_patch.go
│ │ ├── apply_patch_test.go
│ │ ├── diff.go
│ │ ├── diff_test.go
│ │ ├── patch.go
│ │ ├── patch_test.go
│ │ ├── print_diff.go
│ │ └── summary.go
│ ├── hash/
│ │ ├── base32.go
│ │ ├── base32_test.go
│ │ ├── hash.go
│ │ ├── hash_slice.go
│ │ ├── hash_slice_test.go
│ │ └── hash_test.go
│ ├── marshal/
│ │ ├── decode.go
│ │ ├── decode_test.go
│ │ ├── encode.go
│ │ ├── encode_test.go
│ │ ├── encode_type.go
│ │ └── encode_type_test.go
│ ├── merge/
│ │ ├── candidate.go
│ │ ├── three_way.go
│ │ ├── three_way_keyval_test.go
│ │ ├── three_way_list.go
│ │ ├── three_way_list_test.go
│ │ ├── three_way_ordered_sequence.go
│ │ ├── three_way_set_test.go
│ │ └── three_way_test.go
│ ├── metrics/
│ │ ├── histogram.go
│ │ └── histogram_test.go
│ ├── nbs/
│ │ ├── NBS-on-AWS.md
│ │ ├── README.md
│ │ ├── aws_chunk_source.go
│ │ ├── aws_chunk_source_test.go
│ │ ├── aws_table_persister.go
│ │ ├── aws_table_persister_test.go
│ │ ├── block_store_test.go
│ │ ├── cache.go
│ │ ├── conjoiner.go
│ │ ├── conjoiner_test.go
│ │ ├── dynamo_fake_test.go
│ │ ├── dynamo_manifest.go
│ │ ├── dynamo_manifest_test.go
│ │ ├── dynamo_table_reader.go
│ │ ├── dynamo_table_reader_test.go
│ │ ├── factory.go
│ │ ├── factory_test.go
│ │ ├── fd_cache.go
│ │ ├── fd_cache_test.go
│ │ ├── file_manifest.go
│ │ ├── file_manifest_test.go
│ │ ├── file_table_persister.go
│ │ ├── file_table_persister_test.go
│ │ ├── fs_table_cache.go
│ │ ├── fs_table_cache_test.go
│ │ ├── manifest.go
│ │ ├── manifest_cache.go
│ │ ├── manifest_cache_test.go
│ │ ├── mem_table.go
│ │ ├── mem_table_test.go
│ │ ├── mmap_table_reader.go
│ │ ├── mmap_table_reader_test.go
│ │ ├── persisting_chunk_source.go
│ │ ├── persisting_chunk_source_test.go
│ │ ├── root_tracker_test.go
│ │ ├── s3_fake_test.go
│ │ ├── s3_table_reader.go
│ │ ├── s3_table_reader_test.go
│ │ ├── stats.go
│ │ ├── stats_test.go
│ │ ├── store.go
│ │ ├── table.go
│ │ ├── table_persister.go
│ │ ├── table_persister_test.go
│ │ ├── table_reader.go
│ │ ├── table_set.go
│ │ ├── table_set_test.go
│ │ ├── table_test.go
│ │ ├── table_writer.go
│ │ └── test/
│ │ └── manifest_clobber.go
│ ├── ngql/
│ │ ├── README.md
│ │ ├── query.go
│ │ ├── query_test.go
│ │ └── types.go
│ ├── nomdl/
│ │ ├── lexer.go
│ │ ├── parser.go
│ │ └── parser_test.go
│ ├── perf/
│ │ ├── hash-perf-rig/
│ │ │ ├── README.md
│ │ │ └── main.go
│ │ └── suite/
│ │ ├── suite.go
│ │ └── suite_test.go
│ ├── sloppy/
│ │ ├── sloppy.go
│ │ └── sloppy_test.go
│ ├── spec/
│ │ ├── absolute_path.go
│ │ ├── absolute_path_test.go
│ │ ├── commit_meta.go
│ │ ├── commit_meta_test.go
│ │ ├── spec.go
│ │ ├── spec_test.go
│ │ └── util.go
│ ├── types/
│ │ ├── blob.go
│ │ ├── blob_editor.go
│ │ ├── blob_editor_test.go
│ │ ├── blob_leaf_sequence.go
│ │ ├── blob_test.go
│ │ ├── bool.go
│ │ ├── codec.go
│ │ ├── codec_test.go
│ │ ├── collection.go
│ │ ├── collection_test.go
│ │ ├── common_supertype.go
│ │ ├── common_supertype_test.go
│ │ ├── compare_test.go
│ │ ├── edit_distance.go
│ │ ├── edit_distance_test.go
│ │ ├── encode_human_readable.go
│ │ ├── encode_human_readable_test.go
│ │ ├── encoding_test.go
│ │ ├── equals_test.go
│ │ ├── get_hash.go
│ │ ├── graph_builder.go
│ │ ├── graph_builder_test.go
│ │ ├── incremental_test.go
│ │ ├── indexed_sequence_diff.go
│ │ ├── indexed_sequences.go
│ │ ├── leaf_sequence.go
│ │ ├── less.go
│ │ ├── list.go
│ │ ├── list_editor.go
│ │ ├── list_editor_test.go
│ │ ├── list_iterator.go
│ │ ├── list_iterator_test.go
│ │ ├── list_leaf_sequence.go
│ │ ├── list_test.go
│ │ ├── make_type.go
│ │ ├── map.go
│ │ ├── map_editor.go
│ │ ├── map_iterator.go
│ │ ├── map_iterator_test.go
│ │ ├── map_leaf_sequence.go
│ │ ├── map_test.go
│ │ ├── meta_sequence.go
│ │ ├── noms_kind.go
│ │ ├── number.go
│ │ ├── number_util.go
│ │ ├── opcache.go
│ │ ├── opcache_compare.go
│ │ ├── opcache_test.go
│ │ ├── ordered_sequences.go
│ │ ├── ordered_sequences_diff.go
│ │ ├── ordered_sequences_diff_test.go
│ │ ├── path.go
│ │ ├── path_test.go
│ │ ├── perf/
│ │ │ ├── dummy.go
│ │ │ └── perf_test.go
│ │ ├── primitives_test.go
│ │ ├── ref.go
│ │ ├── ref_heap.go
│ │ ├── ref_heap_test.go
│ │ ├── ref_test.go
│ │ ├── rolling_value_hasher.go
│ │ ├── rungen.go
│ │ ├── sequence.go
│ │ ├── sequence_chunker.go
│ │ ├── sequence_concat.go
│ │ ├── sequence_cursor.go
│ │ ├── sequence_cursor_test.go
│ │ ├── set.go
│ │ ├── set_editor.go
│ │ ├── set_iterator.go
│ │ ├── set_iterator_test.go
│ │ ├── set_leaf_sequence.go
│ │ ├── set_test.go
│ │ ├── simplify.go
│ │ ├── simplify_test.go
│ │ ├── string.go
│ │ ├── string_test.go
│ │ ├── struct.go
│ │ ├── struct_test.go
│ │ ├── subtype.go
│ │ ├── subtype_test.go
│ │ ├── type.go
│ │ ├── type_desc.go
│ │ ├── type_test.go
│ │ ├── util_test.go
│ │ ├── validate_type.go
│ │ ├── validating_decoder.go
│ │ ├── validating_decoder_test.go
│ │ ├── value.go
│ │ ├── value_decoder.go
│ │ ├── value_stats.go
│ │ ├── value_store.go
│ │ ├── value_store_test.go
│ │ ├── walk.go
│ │ ├── walk_refs.go
│ │ ├── walk_refs_test.go
│ │ └── walk_test.go
│ └── util/
│ ├── clienttest/
│ │ └── client_test_suite.go
│ ├── datetime/
│ │ ├── date_time.go
│ │ └── date_time_test.go
│ ├── exit/
│ │ └── exit.go
│ ├── functions/
│ │ ├── all.go
│ │ └── all_test.go
│ ├── json/
│ │ ├── from_json.go
│ │ ├── from_json_test.go
│ │ ├── to_json.go
│ │ └── to_json_test.go
│ ├── math/
│ │ └── minmax.go
│ ├── outputpager/
│ │ └── page_output.go
│ ├── profile/
│ │ └── profile.go
│ ├── progressreader/
│ │ └── reader.go
│ ├── random/
│ │ ├── id.go
│ │ └── id_test.go
│ ├── sizecache/
│ │ ├── size_cache.go
│ │ └── size_cache_test.go
│ ├── status/
│ │ └── status.go
│ ├── test/
│ │ └── equals_ignore_hashes.go
│ ├── verbose/
│ │ └── verbose.go
│ └── writers/
│ ├── max_line_writer.go
│ ├── prefix_writer.go
│ └── writers_test.go
├── go.mod
├── go.sum
├── release.sh
├── samples/
│ ├── cli/
│ │ └── nomsconfig/
│ │ └── README.md
│ └── go/
│ ├── csv/
│ │ ├── README.md
│ │ ├── common.go
│ │ ├── csv_reader.go
│ │ ├── csv_reader_test.go
│ │ ├── kind_slice.go
│ │ ├── kind_slice_test.go
│ │ ├── read.go
│ │ ├── read_test.go
│ │ ├── schema.go
│ │ ├── schema_test.go
│ │ ├── write.go
│ │ └── write_test.go
│ ├── decent/
│ │ ├── README.md
│ │ ├── data/
│ │ │ ├── godfather.html
│ │ │ ├── godfather2.html
│ │ │ └── godfather3.html
│ │ ├── dbg/
│ │ │ └── debug.go
│ │ ├── ipfs-chat/
│ │ │ └── main.go
│ │ ├── lib/
│ │ │ ├── datapager.go
│ │ │ ├── event.go
│ │ │ ├── importer.go
│ │ │ ├── logger.go
│ │ │ ├── model.go
│ │ │ ├── model_test.go
│ │ │ ├── pubsub.go
│ │ │ ├── term_index.go
│ │ │ ├── term_index_test.go
│ │ │ └── termui.go
│ │ └── p2p-chat/
│ │ ├── README.md
│ │ └── main.go
│ └── nomdex/
│ ├── Readme.md
│ ├── expr.go
│ ├── nomdex.go
│ ├── nomdex_find.go
│ ├── nomdex_test.go
│ ├── nomdex_update.go
│ ├── parser.go
│ ├── parser_test.go
│ ├── query_range.go
│ └── query_range_test.go
└── tools/
├── file/
│ ├── file.go
│ └── file_test.go
├── licensify.py
├── loadtest/
│ └── loadtest.go
├── noms/
│ ├── README.md
│ ├── __init__.py
│ ├── copy.py
│ ├── copy_test.py
│ ├── pushd.py
│ ├── staging.py
│ ├── staging_test.py
│ ├── symlink.py
│ └── symlink_test.py
└── runner/
├── serial.go
└── serial_test.go
================================================
FILE CONTENTS
================================================
================================================
FILE: .dockerignore
================================================
.git
doc
codecov.yml
CONTRIBUTING.md
LICENSE
README.md
samples
================================================
FILE: .gitignore
================================================
noms.iml
*.pyc
*.swp
.vscode
.idea
.noms
.nomsconfig
.DS_Store
node_modules
================================================
FILE: .travis.yml
================================================
language: go
go:
- 1.x
os:
- linux
env:
- GO111MODULE=on
before_install:
# gox simplifies building for multiple architectures
- go get github.com/mitchellh/gox
script:
- go build ./...
- go test ./...
- go vet ./...
- mkdir linux
- mkdir darwin
- gox -os="linux darwin" -arch="amd64" -output="{{.OS}}/{{.Dir}}" ./cmd/noms
- gox -os="linux darwin" -arch="amd64" -output="{{.OS}}/{{.Dir}}" ./samples/go/csv/csv-import
- gox -os="linux darwin" -arch="amd64" -output="{{.OS}}/{{.Dir}}" ./samples/go/csv/csv-export
- gox -os="linux darwin" -arch="amd64" -output="{{.OS}}/{{.Dir}}" ./samples/go/xml-import
- mv darwin osx
- zip -r linux linux
- zip -r osx osx
deploy:
overwrite: true
provider: releases
skip_cleanup: true
api_key:
secure: "N2LCdQDlPquU31TK8WZwlYPRT7SSyfsGPBpNPSp5gpJPtF5hlqLf96Fd1R7SYn/LfTcri8baFMxgPVK4FowAzIsTxwkG57vCnJR24atOFVLkaKzVPdQZ30zXDHq2WO1zYw7KzAZq49YWdzwKSShzT7+SpiNZWEE2UiB5ZSQcd7/fii1TUkphzWPeHCB+d9wf1qUyJmm6HQ3PKe9yYRQHczGin6INUV5o+nzlRws2+5Kj7eg519htLgRY0oloncY0fdwTEwbSTTkkja3eoAWQrdPMJH7mDMwpbdgPl3jW8wDdTPHO5mQHRF4GvJHrY18qMJ9Kf8iQ3bdRtIS5XM8kvo8+Le22XQbYH7Q7Ryj/bJN+71KpVLwqWQhOr3fWRrL7r8DDPAG/myw0SK1uMaXCzT3KiYckJv7Q3el9MkHNblvFNxWC4tIrwE0LtP4hbSiIlZ/MV58yJxU8WXVej9AoFnKHLA7hgJUhHy0EIlfeETalDBrqNrh40iNP0maUrhpNJxLGtpOgAUhrdQ3gd//6pWwejkfvMTQ2b+1Qq11wWsSmRI/U1WGbcO/wzjKgVv2PT2sYPgx7TPwPWje5uFTZ4/sehwGG/LDcvuZ5uBXLRcpFIz9oh31nIFzsxhdatSKaaK4zlMzkxec+xqBGe0SVKeL/rW0MeQUbeSqyAf0wtBQ="
file:
- linux.zip
- osx.zip
on:
repo: attic-labs/noms
tags: true
================================================
FILE: CONTRIBUTING.md
================================================
Contributing to Noms
====================
## Install Go
First setup Go on your machine per https://golang.org/doc/install. You need *at least* Go version 1.11. You can test that you're setup correctly like so:
```shell
# Must be at least 1.11
go version
```
## Get and build Noms
Noms uses [Go modules](https://github.com/golang/go/wiki/Modules) a new feature of Go. Therefore if you're an existing Go user, you need to be careful to check out into a directory **other than $GOPATH** (or else use the environment variable GO11MODULES=on to force it on). Hopefully this gets easier to understand in future Go versions when Go modules become stabilized.
```shell
cd <any directory other than $GOPATH>
git clone https://github.com/attic-labs/noms
cd noms
go install ./cmd/noms
go test ./...
```
## License
Noms is open source software, licensed under the [Apache License, Version 2.0](LICENSE).
## Contributing code
Due to legal reasons, all contributors must sign a contributor agreement, either for an [individual](https://attic-labs.github.io/ca/individual.html) or [corporation](https://attic-labs.github.io/ca/corporation.html), before a pull request can be accepted.
## Languages
* Use Go, JS, or Python.
* Shell script is not allowed.
## Coding style
* Go uses `gofmt`, advisable to hook into your editor
* JS follows the [Airbnb Style Guide](https://github.com/airbnb/javascript)
* Tag PRs with either `toward: #<bug>` or `fixes: #<bug>` to help establish context for why a change is happening
* Commit messages follow [Chris Beam's awesome commit message style guide](http://chris.beams.io/posts/git-commit/)
### Go error reporting
In general, for Public API in Noms, we use the Go-style of returning errors by default.
For non-exposed code, we do provide, and use, some wrappers to do Exception-style error handling. There *must* be an overriding rationale for using this style, however. One reason to use the Exception-style is that the current code doesn't know how to proceed and needs to panic, but you want to signal that a calling function somewhere up the stack might be able to recover from the failure and continue.
For these cases, please use the following family of functions to 'raise' a 'catchable' error (see [go/d/try.go](https://godoc.org/github.com/attic-labs/noms/go/d)):
* d.PanicIfError()
* d.PanicIfTrue()
* d.PanicIfFalse()
You might see some old code that uses functions that seem similar starting with `d.Chk`, however we are going to remove those and don't want to use them for new code. See #3258 for details.
## Submitting PRs
We follow a code review protocol dervied from the one that the [Chromium team](https://www.chromium.org/) uses:
1. Create a GitHub fork of the repo you want to modify (e.g., fork `https://github.com/attic-labs/noms` to `https://github.com/<username>/noms`).
2. Add your own fork as a remote to your github repo: `git remote add <username> https://github.com/<username>/noms`.
3. Push your changes to a branch at your fork: `git push <username> <branch>`
4. Create a PR using the branch you just created. Usually you can do this by just navigating to https://github.com/attic-labs/noms in a browser - GitHub recognizes the new branch and offers to create a PR for you.
5. When you're ready for review, make a comment in the issue asking for a review. Sometimes people won't review until you do this because we're not sure if you think the PR is ready for review.
6. Iterate with your reviewer using the normal Github review flow.
7. Once the reviewer is happy with the changes, they will submit them.
## Running the tests
You can use `go test` command, e.g:
* `go test $(go list ./... | grep -v /vendor/)` should run every test except from vendor packages.
If you have commit rights, Jenkins automatically runs the Go tests on every PR, then every subsequent patch. To ask Jenkins to immediately run, any committer can reply (no quotes) "Jenkins: test this" to your PR.
### Perf tests
By default, neither `go test` nor Jenkins run the perf tests, because they take a while.
To run the tests yourself, use the `-perf` and `-v` flag to `go test`, e.g.:
* `go test -v ./samples/go/csv/... -perf mem`
See https://godoc.org/github.com/attic-labs/noms/go/perf/suite for full documentation and flags.
To ask Jenkins to run the perf tests for you, reply (no quotes) "Jenkins: perf this" to your PR. Your results will be viewable at http://perf.noms.io/?ds=http://demo.noms.io/perf::pr_$your-pull-request-number/csv-import. Again, only a committer can do this.
================================================
FILE: Dockerfile
================================================
FROM golang:latest AS build
ENV NOMS_SRC=$GOPATH/src/github.com/attic-labs/noms
ENV CGO_ENABLED=1
ENV GOOS=linux
ENV DOCKER=1
RUN mkdir -pv $NOMS_SRC
COPY . ${NOMS_SRC}
RUN go test github.com/attic-labs/noms/...
RUN go install -v github.com/attic-labs/noms/cmd/noms
RUN cp $GOPATH/bin/noms /bin/noms
FROM alpine:latest
COPY --from=build /bin/noms /bin/noms
VOLUME /data
EXPOSE 8000
ENTRYPOINT [ "noms" ]
CMD ["serve", "/data"]
================================================
FILE: HACKING.md
================================================
# Prerequisites
* [Go 1.13 or later](https://golang.org/dl/)
* Mac or Linux (Noms isn't currently supported on Windows)
# Get
```
git clone https://github.com/attic-labs/noms
```
# Build
```
cd noms
go build ./cmd/noms
```
# Test
```
cd noms
go test ./go/...
go test ./cmd/...
```
# Release
Travis automatically creates releases for tagged versions, so the following should do it:
```
git tag latest -f
git push origin latest
```
================================================
FILE: LICENSE
================================================
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "{}"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright {yyyy} {name of copyright owner}
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: README.md
================================================
# Warning - This project is not active
Noms is not being maintained. You shouldn't use it, except maybe for fun or research.
If you are interested in something like Noms, you probably want Dolt (https://github.com/dolthub/dolt) which is a fork of this project and actively maintained.
Send me (aaron at aaronboodman.com) a message if you have questions.
<hr>
<img src='doc/nommy_cropped_smaller.png' width='350' title='Nommy, the snacky otter'>
[Use Cases](#use-cases) | [Setup](#setup) | [Status](#status) | [Documentation](./doc/intro.md) | [Contact](#contact-us)
<br><br>
[](https://travis-ci.org/attic-labs/noms)
[](https://hub.docker.com/r/noms/noms/)
[](https://godoc.org/github.com/attic-labs/noms)
# Welcome
*Noms* is a decentralized database philosophically descendant from the Git version control system.
Like Git, Noms is:
* **Versioned:** By default, all previous versions of the database are retained. You can trivially track how the database evolved to its current state, easily and efficiently compare any two versions, or even rewind and branch from any previous version.
* **Synchronizable:** Instances of a single Noms database can be disconnected from each other for any amount of time, then later reconcile their changes efficiently and correctly.
Unlike Git, Noms is a database, so it also:
* Primarily **stores structured data**, not files and directories (see: [the Noms type system](https://github.com/attic-labs/noms/blob/master/doc/intro.md#types))
* **Scales well** to large amounts of data and concurrent clients
* Supports **atomic transactions** (a single instance of Noms is CP, but Noms is typically run in production backed by S3, in which case it is "[effectively CA](https://cloud.google.com/spanner/docs/whitepapers/SpannerAndCap.pdf)")
* Supports **efficient indexes** (see: [Noms prolly-trees](https://github.com/attic-labs/noms/blob/master/doc/intro.md#prolly-trees-probabilistic-b-trees))
* Features a **flexible query model** (see: [GraphQL](./go/ngql/README.md))
A Noms database can reside within a file system or in the cloud:
* The (built-in) [NBS](./go/nbs) `ChunkStore` implementation provides two back-ends which provide persistence for Noms databases: one for storage in a file system and one for storage in an S3 bucket.
Finally, because Noms is content-addressed, it yields a very pleasant programming model.
Working with Noms is ***declarative***. You don't `INSERT` new data, `UPDATE` existing data, or `DELETE` old data. You simply *declare* what the data ought to be right now. If you commit the same data twice, it will be deduplicated because of content-addressing. If you commit _almost_ the same data, only the part that is different will be written.
<br>
## Use Cases
#### [Decentralization](./doc/decent/about.md)
Because Noms is very good at sync, it makes a decent basis for rich, collaborative, fully-decentralized applications.
#### Mobile Offline-First Database
Embed Noms into mobile applications, making it easier to build offline-first, fully synchronizing mobile applications.
<br>
## Install
1. Download the latest release:
- [**Linux**](https://github.com/attic-labs/noms/releases/download/latest/linux.zip)
- [**Mac OS**](https://github.com/attic-labs/noms/releases/download/latest/osx.zip)
2. Unzip the directory somewhere and add it to your `$PATH`
3. Verify Noms is installed correctly:
```
$ noms version
format version: 7.18
built from <developer build>
```
<br>
## Run
Import some data:
```shell
go install github.com/attic-labs/noms/samples/go/csv/csv-import
curl 'https://data.cityofnewyork.us/api/views/kku6-nxdu/rows.csv?accessType=DOWNLOAD' > /tmp/data.csv
csv-import /tmp/data.csv /tmp/noms::nycdemo
```
Explore:
```shell
noms show /tmp/noms::nycdemo
```
Should show:
```go
struct Commit {
meta: struct Meta {
date: "2017-09-19T19:33:01Z",
inputFile: "/tmp/data.csv",
},
parents: set {},
value: [ // 236 items
struct Row {
countAmericanIndian: "0",
countAsianNonHispanic: "3",
countBlackNonHispanic: "21",
countCitizenStatusTotal: "44",
countCitizenStatusUnknown: "0",
countEthnicityTotal: "44",
...
```
<br>
## Status
Nobody is working on this right now. You shouldn't rely on it unless you're willing to take over development yourself.
### Major Open Issues
These are the major things you'd probably want to fix before relying on this for most systems.
* Sync performance with long commit chains (https://github.com/attic-labs/noms/issues/2233)
* Migration (https://github.com/attic-labs/noms/issues/3363)
* Garbage Collection (https://github.com/attic-labs/noms/issues/3374)
* Query language
* We started trying to hack in GraphQL but it's incomplete and maybe not the right thing. See: [ngql](./go/ngql/README.md)
* [Various other smaller bugs and improvements](https://github.com/attic-labs/noms/issues?q=is%3Aissue+is%3Aopen+label%3AP0)
<br>
## Learn More About Noms
For the decentralized web: [The Decentralized Database](doc/decent/about.md)
Learn the basics: [Technical Overview](doc/intro.md)
Tour the CLI: [Command-Line Interface Tour](doc/cli-tour.md)
Tour the Go API: [Go SDK Tour](doc/go-tour.md)
<br>
## Contact Us
Interested in using Noms? Awesome! We would be happy to work with you to help understand whether Noms is a fit for your problem. Reach out at:
- [Mailing List](https://groups.google.com/forum/#!forum/nomsdb)
- [Twitter](https://twitter.com/nomsdb)
## Licensing
Noms is open source software, licensed by Attic Labs, Inc. under the Apache License, Version 2.0.
================================================
FILE: cmd/util/kingpin_command.go
================================================
// Copyright 2017 Attic Labs, Inc. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
package util
import "github.com/attic-labs/kingpin"
type KingpinHandler func(input string) (exitCode int)
type KingpinCommand func(*kingpin.Application) (*kingpin.CmdClause, KingpinHandler)
================================================
FILE: codecov.yml
================================================
codecov:
branch: master
bot: "mikegray"
ci:
- "jenkins3.noms.io"
coverage:
precision: 2 # how many decimal places to display in the UI: 0 <= value <= 4
round: down # how coverage is rounded: down/up/nearest
range: 70...100 # custom range of coverage colors from red -> yellow -> green
notify:
slack:
default:
url: "secret:n+BYhIXTXsaCiMKB3vOf6yP68ytdKd3WpXtJFWPEUsEWXDiGnU5dTB5DO2yv8tR0COdxvs7K31hVpEfHEXdoXOaQhUw3FKf3fh8KZDLN7CGTbeDhw1uNGGyBr2d2TWnopzYtcXomdwMmuckARtiWQx0YXJiZY9YyCrIoDK9HIJQ="
branches: null
threshold: 5.0
attachments: "tree, diff"
status:
project:
default:
enabled: yes
target: auto
branches: null
threshold: null
if_no_uploads: error
if_not_found: success
if_ci_failed: error
patch:
default:
enabled: yes
target: auto
branches: null
threshold: null
if_no_uploads: error
if_not_found: success
if_ci_failed: error
changes:
default:
enabled: yes
branches: null
if_no_uploads: error
if_not_found: success
if_ci_failed: error
comment:
layout: "tree"
branches: null
behavior: default
================================================
FILE: doc/cli-tour.md
================================================
[Home](../README.md) »
[Technical Overview](intro.md) | [Use Cases](../README.md#use-cases) | **Command-Line Interface** | [Go bindings Tour](go-tour.md) | [Path Syntax](spelling.md) | [FAQ](faq.md)
<br><br>
# A Short Tour of the Noms CLI
This is a quick introduction to the Noms command-line interface. It should only take a few minutes to read. There's also a screencast covering some of this if you prefer:
[<img src="cli-screencast.png" width="500">](https://www.youtube.com/watch?v=ncITL4xdXh4)
## Install Noms
... if you haven't already. Follow the instructions [here](https://github.com/attic-labs/noms#setup).
## The `noms` command
Now you should be able to run `noms`:
```shell
> noms
Noms is a tool for goofing with Noms data.
Usage:
noms command [arguments]
The commands are:
diff Shows the difference between two objects
ds Noms dataset management
log Displays the history of a Noms dataset
serve Serves a Noms database over HTTP
show Shows a serialization of a Noms object
sync Moves datasets between or within databases
version Display noms version
Use "noms help [command]" for more information about a command.
```
Without any arguments, `noms` lists out all available commands. To get information on a specific command, we can use `noms help [command]`:
```shell
> noms help sync
usage: noms sync [options] <source-object> <dest-dataset>
See Spelling Objects at https://github.com/attic-labs/noms/blob/master/doc/spelling.md for details on the object and dataset arguments.
...
```
## noms ds
The `noms ds` command lists the _datasets_ within a particular database:
```shell
> noms ds http://demo.noms.io
...
sf-film-locations/raw
sf-film-locations
...
```
## noms log
Noms datasets are versioned. You can see the history with `log`:
```shell
> !? noms log http://demo.noms.io::sf-film-locations
commit aprsmg0j2eegk8eehbgj7cd3tmmd1be8
Parent: None
Date: "2017-09-19T21:42:46Z"
InputPath: "http://localhost:8000::#dksek6tuf8ens06bi4culq85tfp5q4cg.value"
...
```
Note that Noms is a typed system. What is being shown here for each entry is not text, but a serialization of the diff between two datasets.
## noms show
You can see the entire serialization of any object in the database with `noms show`:
```shell
> noms show 'http://demo.noms.io::#aprsmg0j2eegk8eehbgj7cd3tmmd1be8'
struct Commit {
meta: struct {},
parents: Set<Ref<Cycle<Commit>>>,
value: List<struct Row {
Actor1: String,
Actor2: String,
Actor3: String,
Director: String,
Distributor: String,
FunFacts: String,
Locations: String,
ProductionCompany: String,
ReleaseYear: Number,
Title: String,
Writer: String,
}>,
}({
meta: Meta {
date: "2016-07-25T18:34:00+0000",
inputPath: "http://localhost:8000::sf-film-locations/raw.value",
},
parents: {
c506ta03786j48a07he83ju669u78qa2,
},
value: [ // 1,241 items
Row {
Actor1: "Siddarth",
...
```
## noms sync
You can work with Noms databases that are remote exactly the same as you work with local databases. But it's frequently useful to move data to a local machine, for example, to make a private fork or to work with the data disconnected from the source database.
Moving data in Noms is done with the `sync` command. Note that unlike Git, we do not make a distinction between _push_ and _pull_. It's the same operation in both directions:
```shell
> noms sync http://demo.noms.io::sf-film-locations /tmp/noms::films
> noms ds /tmp/noms
films
```
We can now make an edit locally:
```shell
> go install github.com/attic-labs/noms/samples/go/csv/...
> csv-export /tmp/noms::films > /tmp/film-locations.csv
```
open /tmp/film-location.csv and edit it, then:
```shell
> csv-import --column-types=String,String,String,String,String,String,String,String,Number,String,String \
/tmp/film-locations.csv /tmp/noms::films
```
## noms diff
The `noms diff` command can show you the differences between any two values. Let's see our change:
```shell
> noms diff http://demo.noms.io::sf-film-locations /tmp/noms::films
./.meta {
- "date": "2016-07-25T18:51:23+0000"
+ "date": "2016-07-25T22:51:14+0000"
+ "inputFile": "/tmp/film-locations.csv"
- "inputPath": "http://demo.noms.io::sf-film-locations/raw.value"
./.parents {
- pckdvpvr9br1fie6c3pjudrlthe7na18
+ q4jcc2i7kntkjiipvjgpr5r02ldroj0g
}
./.value[0] {
- "Locations": "Epic Roasthouse (399 Embarcadero)"
+ "Locations": "Epic Roadhouse (399 Embarcadero)"
```
================================================
FILE: doc/decent/about.md
================================================
[Home](../../README.md) » [Use Cases](../../README.md#use-cases) » **Decentralized** »
**About** | [Quickstart](quickstart.md) | [Architectures](architectures.md) | [P2P Chat Demo](demo-p2p-chat.md) | [IPFS Chat Demo](demo-ipfs-chat.md)
<br><br>
# Noms — The Decentralized Database
[Noms](http://noms.io) makes it ~~easy~~ tractable to create rich,
multiuser, collaborative, fully-decentralized applications.
Like most databases, Noms features a rich data model, atomic
transactions, support for large-scale data, and efficient searches,
scans, reads, and updates.
Unlike any other database, Noms has built-in multiparty sync and
conflict resolution. This feature makes Noms a very good fit for P2P
decentralized applications.
Any number of dapp peers in a P2P network can
concurrently modify the same logical Noms database, and continuously
and efficiently sync their changes with each other. All peers will
converge to the same state.
For many applications, peers can store an entire local copy of the
data they are interested in. For larger applications, it should be
possible to back Noms by a decentralized blockstore like IPFS, Swarm,
or Sia (or in the future, Filecoin), and store large-scale data in a
completely decentralized way, without replicating it on every
node. Noms also has a blockstore for S3, which is ideal for
applications that have some centralized components.
**We'd love to talk to you about the possibility of using noms in your project** so please don't hestitate to contact us at [noms@attic.io](mailto:noms@attic.io).
## How it Works
Think of Noms like a programmable Git: changes are bundled as commits
which reference previous states of the database. Apps pull changes
from peers and merge them using a principled set of APIs and
strategies. Except that rather than users manually pulling and
merging, applications typically do this continuously, automatically
converging to a shared state.
Your application uses a [Go client
library](https://github.com/attic-labs/noms/blob/master/doc/go-tour.md)
to interact with Noms data. There is also a [command-line
interface](https://github.com/attic-labs/noms/blob/master/doc/cli-tour.md)
for working with data and initial support for a [GraphQL-based query
language](https://github.com/attic-labs/noms/blob/master/go/ngql/README.md).
Some additional features include:
* **Versioning**: It’s easy to use, compare, or revert to older database versions
* **Efficient diffs**: diffing even huge datasets is efficient due to
noms’ use of a novel BTree-like data structure called a [Prolly
Tree](https://github.com/attic-labs/noms/blob/master/doc/intro.md#prolly-trees-probabilistic-b-trees)
* **Efficient storage**: data are chunked and content-addressable, so
there is exactly one copy of each chunk in the database, shared by
other data that reference it. Small changes to massive data
structures always result in small operations.
* **Verifiable**: The entire database rolls up to a single 20-byte hash
that uniquely represents the database at that moment - anyone can
verify that a particular database hashes to the same value
Read the [Noms design overview](https://github.com/attic-labs/noms/blob/master/doc/decent/intro.md).
## Status
For overall status of the database, see [Noms Status](../../README.md#status).
For the decentralized use case in particular: we are fairly confident in this approach and are actively looking for partners to work with to build it out.
- [x] Demonstrate core concept of using Noms to continuously sync across many users (Done! See noms-chat demos)
- [ ] Demonstrate using libp2p or similar to traverse NATs
- [ ] Investigate backing IPFS with Noms rather than the reverse - this should improve stability and dramatically improve local performance
- [ ] Demonstrate using IPFS with a schema that permits nodes to disappear
**_If you would like to use noms in your project we’d love to hear from you_**:
drop us an email ([noms@attic.io](mailto:noms@attic.io)) or send us a
message in slack ([slack.noms.io](http://slack.noms.io)).
================================================
FILE: doc/decent/architectures.md
================================================
[Home](../../README.md) » [Use Cases](../../README.md#use-cases) » **Decentralized** »
[About](about.md) | [Quickstart](quickstart.md) | **Architectures** | [P2P Chat Demo](demo-p2p-chat.md) | [IPFS Chat Demo](demo-ipfs-chat.md)
<br><br>
# Architectures
There are many possible ways to use Noms as part of a decentralized application. Noms can naturally be mixed and matched with other decentralized tools like blockchains, IPFS, etc. This page lists a few approaches we find promising.
## Classic P2P Architecture
Noms can be used to implement apps in a peer-to-peer configuration. Each instance of the application (i.e., each "node") maintains a database locally with the data that is relevant to it. When a node creates new data, it commits that data to it's database and broadcasts a message to it's peers that contains the hash of it's lastest commit.
Peers that are listening for these message can decide if that data is relevant to them. Those that are interested can pull the new data from the publisher. The two clients efficiently communicate so that only data that isn't present in the requesting client is transmitted (much the same way that one git client sends source changes to another).
Peers can use a flow similar to the following in order to sync changes with one another:
```nohighlight
for {
listen for new message
if new msg is relevant {
if new msg is ancestor of current commit {
// nothing to do
continue
}
pull new data from sender of msg
if current head is ancestor of new msg {
// fast forward to the new commit
set head of dataset to new commit
continue
}
merge new with current head and commit
publish new commit
}
}
```
Noms has a default [merge policy](https://github.com/attic-labs/noms/blob/2d0e9e738370d49cc09e8fa6e290ceca1c3e2005/go/merge/three_way.go#L14) that covers many classes of concurrent operations. If the application restricts itself to only operations that are mergeable by this policy, then Noms can automatically merge all concurrent changes. In this case, the entire database is effectively a CRDT.
If this is not sufficient, then applications can create their own merge policies, implementing whatever merge is appropriate for their use case.
# Decentralized Chunkstore Architecture
Another potential architecture for decentralized apps uses a decentralized chunkstore (such as IPFS, Swarm, or Sia) rather than local databases. In this case, rather than each node maintaining a local datastore, Noms chunks are stored in a decentralized chunkstore. The underlying chunkstore is responsible for making chunks available when needed.
The flow used by peers to sync with one another is similar to the peer-to-peer architecture. The main difference is data is not duplicated on local machines and doesn't have to be pulled during sync. Each app keeps track of it's latest commit in the chunk store.
```nohighlight
for {
listen for new message
if new msg is relevant {
if new msg is ancestor of current commit {
// nothing to do
continue
}
// No pull necessary
if current head is ancestor of new msg {
// fast forward to the new commit
set head of dataset to new commit
continue
}
merge new with current head and commit
publish new commit
}
}
```
We have a prototype implementation of an IPFS-based chunkstore. If you are interested in pursuing this direction, let us know!
================================================
FILE: doc/decent/demo-ipfs-chat.md
================================================
[Home](../../README.md) » [Use Cases](../../README.md#use-cases) » **Decentralized** »
[About](about.md) | [Quickstart](quickstart.md) | [Architectures](architectures.md) | [P2P Chat Demo](demo-p2p-chat.md) | **IPFS Chat Demo**
<br><br>
# Demo App: IPFS-based Decentralized Chat
This sample app demonstrates backing a P2P noms app by a decentralized blockstore (in this case, IPFS). Data is pulled off the network dynamically as needed - each client doesn't need a complete copy.
# Build and Run
Demo app code is in the
[ipfs-chat](https://github.com/attic-labs/noms/tree/master/samples/go/decent/ipfs-chat/)
directory. To get it up and running take the following steps:
* Use git to clone the noms repository onto your computer:
```shell
go get github.com/attic-labs/noms/samples/go/decent/ipfs-chat
```
* From the noms/samples/go/decent/ipfs-chat directory, build the program with the following command:
```shell
go build
```
* Run the ipfs-chat client with the following command:
```shell
./ipfs-chat client --username <aname1> --node-idx=1 ipfs:/tmp/ipfs1::chat >& /tmp/err1
```
* Run a second ipfs-chat client with the following command:
```shell
./ipfs-chat client --username <aname2> --node-idx=2 ipfs:/tmp/ipfs2::chat >& /tmp/err2
```
If desired, ipfs-chat can be run as a daemon which will replicate all
chat content in a local store which will enable clients to go offline
without causing data to become unavailable to other clients:
```shell
./ipfs-chat daemon --node-idx=3 ipfs:/tmp/ipfs3::chat
```
Note: the 'node-idx' argument ensures that each IPFS-based program
uses a distinct set of ports. This is useful when running multiple
IPFS-based programs on the same machine.
================================================
FILE: doc/decent/demo-p2p-chat.md
================================================
[Home](../../README.md) » [Use Cases](../../README.md#use-cases) » **Decentralized** »
[About](about.md) | [Quickstart](quickstart.md) | [Architectures](architectures.md) | **P2P Chat Demo** | [IPFS Chat Demo](demo-ipfs-chat.md)
<br><br>
# Demo App: P2P Decentralized Chat
This sample demonstrates the simplest possible case of building a p2p app on top of Noms. Each node stores a complete copy of the data it is interested in, and peers find each other using [IPFS pubsub](https://ipfs.io/blog/25-pubsub/).
Currently, nodes have to have a publicly routable IP, but it should be possible to use [libP2P](https://github.com/libp2p) or similar to connect to most nodes.
# Build and Run
Demo app code is in the
[p2p](https://github.com/attic-labs/noms/tree/master/samples/go/decent/p2p-chat)
directory. To get it up and running take the following steps:
* Use git to clone the noms repository onto your computer:
```shell
go get github.com/attic-labs/noms/samples/go/decent/p2p-chat
```
* From the noms/samples/go/decent/p2p-chat directory, build the program with the following command:
```shell
go build
```
* Run the p2p client with the following command:
```shell
mkdir /tmp/noms1
./p2p-chat client --username=<aname1> --node-idx=1 /tmp/noms1 >& /tmp/err1
```
* Run a second p2p client with the following command:
```shell
mkdir /tmp/noms2
./p2p-chat client --username=<aname2> --node-idx=2 /tmp/noms2 >& /tmp/err2
```
Note: the p2p client relies on IPFS for it's pub/sub implementation. The
'node-idx' argument ensures that each IPFS-based node uses a distinct set
of ports. This is useful when running multiple IPFS-based programs on
the same machine.
================================================
FILE: doc/decent/quickstart.md
================================================
[Home](../../README.md) » [Use Cases](../../README.md#use-cases) » **Decentralized** »
[About](about.md) | **Quickstart** | [Architectures](architectures.md) | [P2P Chat Demo](demo-p2p-chat.md) | [IPFS Chat Demo](demo-ipfs-chat.md)
<br><br>
# How to Use Noms in a Decentralized App
If you’d like to use noms in your project we’d love to hear from you:
drop us an email ([noms@attic.io](mailto:noms@attic.io)) or send us a
message in slack ([slack.noms.io](http://slack.noms.io)).
The steps you’ll need to take are:
1. Decide how you’ll model your problem using noms’ datatypes: boolean,
number, string, blob, map, list, set, structs, ref, and
union. (Note: if you are interested in using CRDTs as an alternative
to classic datatypes please let us know.)
2. Consider...
* How peers will discover each other
* How peers will notify each other of changes
* How and when they will pull changes, and
* What potential there is for conflicting changes. Consider modeling
your problem so that changes commute in order to make merging
easier.
In our [p2p sample](https://github.com/attic-labs/noms/blob/master/doc/decent/demo-p2p-chat.md) application, all peers periodically broadcast their HEAD on a known channel using [IPFS pubsub](https://ipfs.io/blog/25-pubsub/), pull each others' changes immediately, and avoid conflicts by using operations that can be resolved with Noms' built in merge policies.
This is basically the simplest possible approach, but lots of options are possible. For example, an alternate approach for discoverability could be to keep a registry of all participating nodes in a blockchain (e.g., by storing them in an Ethereum smart contract). One could store either the current HEAD of each node (updated whenever the node changes state), or just an IPNS name that the node is writing to.
As an example of changes that commute consider modeling a stream
of chat messages. Appending messages from both parties to a list
is not commutative; the result depends on the order in which
messages are added to the list. An example of a commutative
strategy is adding the messages to a `Map` keyed by
`Struct{sender, ordinal}`: the resulting `Map` is the same no
matter what order messages are added.
3. Vendor the code into your project.
4. Decide which type of storage you'd like to use: memory (convenient for playing around), disk, IPFS, or S3. (If you want to implement a store on top of another type of storage that's possible too; email us or reach out on slack and we can help.)
5. Set up and instantiate a database for your storage. Generally, you use the spec package to parse a [dataset spec](https://github.com/attic-labs/noms/blob/master/doc/spelling.md) like `mem::mydataset` which you can then ask for [`Database`](https://github.com/attic-labs/noms/blob/master/go/datas/database.go) and [`Dataset`](https://github.com/attic-labs/noms/blob/master/go/datas/dataset.go).
* **Memory**: no setup required, just instantiate it:
```go
sp := spec.ForDataset("mem::test") // Dataset name is "test"
```
* **Disk**: identify a directory for storage, say `/path/to/chunks`, and then instantiate:
```go
sp := spec.ForDataset("/path/to/chunks::test") // Dataset name is "test"
```
* **IPFS**: identify an IPFS node by directory. If an IPFS node doesn't exist at that directory, one will be created:
```go
sp := spec.ForDataset("ipfs:/path/to/ipfs_repo::test") // Dataset name is "test"
```
* **S3**: Follow the [S3 setup instructions](https://github.com/attic-labs/noms/blob/master/go/nbs/NBS-on-AWS.md) then instantiate a database and dataset:
```go
sess := session.Must(session.NewSession(aws.NewConfig().WithRegion("us-west-2")))
store := nbs.NewAWSStore("dynamo-table", "store-name", "s3-bucket", s3.New(sess), dynamodb.New(sess), 1<<28))
database := datas.NewDatabase(store)
dataset := database.GetDataset("aws://dynamo-table:s3-bucket/store-name::test") // Dataset name is "test"
```
7. Implement using the [Go API](https://github.com/attic-labs/noms/blob/master/doc/go-tour.md). If you're just playing around you could try something like this:
```go
package main
import (
"fmt"
"os"
"github.com/attic-labs/noms/go/spec"
"github.com/attic-labs/noms/go/types"
)
// Usage: quickstart /path/to/store::ds
func main() {
sp, err := spec.ForDataset(os.Args[1])
if err != nil {
fmt.Fprintf(os.Stderr, "Unable to parse spec: %s, error: %s\n", sp, err)
os.Exit(1)
}
defer sp.Close()
db := sp.GetDatabase()
if headValue, ok := sp.GetDataset().MaybeHeadValue(); !ok {
data := types.NewList(sp.GetDatabase(),
newPerson("Rickon", true),
newPerson("Bran", true),
newPerson("Arya", false),
newPerson("Sansa", false),
)
fmt.Fprintf(os.Stdout, "data type: %v\n", types.TypeOf(data).Describe())
_, err = db.CommitValue(sp.GetDataset(), data)
if err != nil {
fmt.Fprint(os.Stderr, "Error commiting: %s\n", err)
os.Exit(1)
}
} else {
// type assertion to convert Head to List
personList := headValue.(types.List)
// type assertion to convert List Value to Struct
personStruct := personList.Get(0).(types.Struct)
// prints: Rickon
fmt.Fprintf(os.Stdout, "given: %v\n", personStruct.Get("given"))
}
}
func newPerson(givenName string, male bool) types.Struct {
return types.NewStruct("Person", types.StructData{
"given": types.String(givenName),
"male": types.Bool(male),
})
}
```
8. You can inspect data that you've committed via the [noms command-line interface](https://github.com/attic-labs/noms/blob/master/doc/cli-tour.md). For example:
```shell
noms log /path/to/store::ds
noms show /path/to/store::ds
```
> Note that Memory tables won't be inspectable because they exist only in the memory of the process that created them.
9. Implement pull and merge. The [pull API](../../go/datas/pull.go) is used pull changes from a peer and the [merge API](../../go/merge/) is used to merge changes before commit. There's an [example of merging in the IPFS-based-chat sample
app](https://github.com/attic-labs/noms/blob/master/samples/go/ipfs-chat/pubsub.go).
================================================
FILE: doc/faq.md
================================================
[Home](../README.md) »
[Technical Overview](intro.md) | [Use Cases](../README.md#use-cases) | [Command-Line Interface](cli-tour.md) | [Go bindings Tour](go-tour.md) | [Path Syntax](spelling.md) | **FAQ**
<br><br>
# Frequently Asked Questions
### Decentralized like BitTorrent?
No, decentralized like Git.
Specifically, Noms isn't itself a peer-to-peer network. If you can get two instances to share data, somehow, then they can synchronize. Noms doesn't define how this should happen though.
Currently, instances mainly share data via either HTTP/DNS or a filesystem. But it should be easy to add other mechanisms. For example, it seems like Noms could run well on top of BitTorrent, or IPFS. You should [look into it](https://github.com/attic-labs/noms/issues/2123).
### Isn't it wasteful to store every version?
Noms deduplicates chunks of data that are identical within one database. So if multiple versions of one dataset share a lot of data, or if the same data is present in multiple datasets, Noms only stores one copy.
That said, it is definitely possible to have write patterns that defeat this. Deduplication is done at the chunk level, and chunks are currently set to an average size of 4KB. So if you change about 1 byte in every 4096 in a single commit, and those changed bytes are well-distributed throughout the dataset, then we will end up making a complete copy of the dataset.
### Is there a way to not store the entire history?
Theoretically, definitely. In Git, for example, the concept of "shallow clones" exists, and we could do something similar in Noms. This has not been implemented yet.
### How does Noms handle conflicts?
Noms provides several built-in policies that can automatically merge common cases of conflicts. For example concurrent edits to sets are always mergeable and concurrent edits to different keys in a map or struct are also mergeable.
The conflict resolution system is pluggable so new policies that are application-specific can be added. However, it's possible to build surprisingly complex applications with just the built-in policies.
### Why don't you just use CRDTs?
[Convergent (or Commutative) Replicated Data Types (CRDTs)](http://hal.upmc.fr/inria-00555588/document) are a class of distributed data structures that provably converge to some agreed-upon state with no synchronization. Stated differently: CRDTs define a merge policy that is commutative over all their operations.
CRDTs are nice because they require no custom conflict/merge code from the developer.
Noms defines a set of intutive built-in merge policies for its core datatypes. For example, the default policy makes all operations on Noms Sets commute (add wins in the case of concurrent remove/add). This means that with the default policy, Noms Sets are a CRDT.
If your application uses only operations on Noms datatypes that can be merged with whatever merge policy you are using, then your schema is a CRDT. It's possible to build surprisingly complex applications this way with just the default policy.
Noms also allows you to provide your own custom policy. If your policy commutes, then the resulting datatype will be a CRDT.
However, it would be nice if application developers could more easily opt-in to using only mergeable operations, thereby enforcing that their schema is a CRDT, and providing confidence that custom merge logic doesn't need to be implemented.
More generally, perhaps there could be a way to test that all possible conflict cases have been handled by the developer. This would allow developers to implement their own custom CRDTs. This is something we'd like to research in the future.
### Why don't you support Windows?
We are a tiny team and we all personally use Macs as our development machines, and we use Linux in production. These two platforms are very close to identical, and so we can generally test on Mac and assume it will work on Linux. Adding Windows would add significant complexity to our code and build processes which we're not willing to take on.
### But you'll accept patches for Windows, right?
No, because then we'll have to maintain those patches.
### Are there any workaround for Windows?
You can use it in a virtual machine. We have also heard Noms works OK with gitbash or cygwin, but that's coincidence.
### Why is it called Noms?
1. It's insert-only. OMNOMNOM.
2. It's content addressed. Every value has its own hash, or [name](http://dictionary.reverso.net/french-english/nom).
### Are you sure Noms doesn't stand for something?
Pretty sure. But if you like, you can pretend it stands for Non-Mutable Store.
================================================
FILE: doc/go-tour.md
================================================
[Home](../README.md) »
[Technical Overview](intro.md) | [Use Cases](../README.md#use-cases) | [Command-Line Interface](cli-tour.md) | **Go bindings Tour** | [Path Syntax](spelling.md) | [FAQ](faq.md)
<br><br>
# A Short Tour of Noms for Go
This is a short introduction to using Noms from Go. It should only take a few minutes if you have some familiarity with Go.
During the tour, you can refer to the complete [Go SDK Reference](https://godoc.org/github.com/attic-labs/noms) for more information on anything you see.
## Requirements
* [Noms command-line tools](https://github.com/attic-labs/noms#setup)
* [Go v1.6+](https://golang.org/dl/)
* Ensure your [$GOPATH](https://github.com/golang/go/wiki/GOPATH) is configured
## Start a Local Database
Let's create a local database to play with:
```sh
> mkdir /tmp/noms-go-tour
> noms serve /tmp/noms-go-tour
```
## [Database](https://github.com/attic-labs/noms/blob/master/go/datas/database.go)
Leave the server running, and in a separate terminal:
```sh
> mkdir noms-tour
> cd noms-tour
```
Then use your favorite editor so that we can start to play with code. To get started with Noms, first create a Database:
```go
package main
import (
"fmt"
"os"
"github.com/attic-labs/noms/go/spec"
)
func main() {
sp, err := spec.ForDatabase("http://localhost:8000")
if err != nil {
fmt.Fprintf(os.Stderr, "Could not access database: %s\n", err)
return
}
defer sp.Close()
}
```
Now let's run it:
```sh
> go run noms-tour.go
```
If you did not leave the server running you would see output of ```Could not access database``` here, otherwise your program should exit cleanly.
See [Spelling in Noms](https://github.com/attic-labs/noms/blob/master/doc/spelling.md) for more information on database spec strings.
## [Dataset](https://github.com/attic-labs/noms/blob/master/go/dataset/dataset.go)
Datasets are the main interface you'll use to work with Noms. Let's update our example to use a Dataset spec string:
```go
package main
import (
"fmt"
"os"
"github.com/attic-labs/noms/go/spec"
)
func main() {
sp, err := spec.ForDataset("http://localhost:8000::people")
if err != nil {
fmt.Fprintf(os.Stderr, "Could not create dataset: %s\n", err)
return
}
defer sp.Close()
if _, ok := sp.GetDataset().MaybeHeadValue(); !ok {
fmt.Fprintf(os.Stdout, "head is empty\n")
}
}
```
Now let's run it:
```sh
> go run noms-tour.go
head is empty
```
Since the dataset does not yet have any values you see ```head is empty```. Let's add some data to make it more interesting:
```go
package main
import (
"fmt"
"os"
"github.com/attic-labs/noms/go/spec"
"github.com/attic-labs/noms/go/types"
)
func newPerson(givenName string, male bool) types.Struct {
return types.NewStruct("Person", types.StructData{
"given": types.String(givenName),
"male": types.Bool(male),
})
}
func main() {
sp, err := spec.ForDataset("http://localhost:8000::people")
if err != nil {
fmt.Fprintf(os.Stderr, "Could not create dataset: %s\n", err)
return
}
defer sp.Close()
db := sp.GetDatabase()
data := types.NewList(db,
newPerson("Rickon", true),
newPerson("Bran", true),
newPerson("Arya", false),
newPerson("Sansa", false),
)
fmt.Fprintf(os.Stdout, "data type: %v\n", types.TypeOf(data).Describe())
_, err = db.CommitValue(sp.GetDataset(), data)
if err != nil {
fmt.Fprint(os.Stderr, "Error commiting: %s\n", err)
}
}
```
Now you will get output of the data type of our Dataset value:
```shell
> go run noms-tour.go
data type: List<struct {
given: String
male: Bool
}>
```
Now you can access the data via your program:
```go
package main
import (
"fmt"
"os"
"github.com/attic-labs/noms/go/spec"
"github.com/attic-labs/noms/go/types"
)
func main() {
sp, err := spec.ForDataset("http://localhost:8000::people")
if err != nil {
fmt.Fprintf(os.Stderr, "Could not create dataset: %s\n", err)
return
}
defer sp.Close()
if headValue, ok := sp.GetDataset().MaybeHeadValue(); !ok {
fmt.Fprintf(os.Stdout, "head is empty\n")
} else {
// type assertion to convert Head to List
personList := headValue.(types.List)
// type assertion to convert List Value to Struct
personStruct := personList.Get(0).(types.Struct)
// prints: Rickon
fmt.Fprintf(os.Stdout, "given: %v\n", personStruct.Get("given"))
}
}
```
Running it now:
```sh
> go run noms-tour.go
given: Rickon
```
You can see this data using the command-line too:
```sh
> noms ds http://localhost:8000
people
> noms show http://localhost:8000::people
struct Commit {
meta: struct {},
parents: set {},
value: [ // 4 items
struct Person {
given: "Rickon",
male: true,
},
struct Person {
given: "Bran",
male: true,
},
struct Person {
given: "Arya",
male: false,
},
struct Person {
given: "Sansa",
male: false,
},
],
}
```
Let's add some more data.
```go
package main
import (
"fmt"
"os"
"github.com/attic-labs/noms/go/spec"
"github.com/attic-labs/noms/go/types"
)
func main() {
sp, err := spec.ForDataset("http://localhost:8000::people")
if err != nil {
fmt.Fprintf(os.Stderr, "Could not create dataset: %s\n", err)
return
}
defer sp.Close()
if headValue, ok := sp.GetDataset().MaybeHeadValue(); !ok {
fmt.Fprintf(os.Stdout, "head is empty\n")
} else {
// type assertion to convert Head to List
personList := headValue.(types.List)
personEditor := personList.Edit()
data := personEditor.Append(
types.NewStruct("Person", types.StructData{
"given": types.String("Jon"),
"family": types.String("Snow"),
"male": types.Bool(true),
}),
).List()
fmt.Fprintf(os.Stdout, "data type: %v\n", types.TypeOf(data).Describe())
_, err = sp.GetDatabase().CommitValue(sp.GetDataset(), data)
if err != nil {
fmt.Fprint(os.Stderr, "Error commiting: %s\n", err)
}
}
}
```
Running this:
```sh
> go run noms-tour.go
data type: List<Struct Person {
family?: String,
given: String,
male: Bool,
}>
```
Datasets are versioned. When you *commit* a new value, you aren't overwriting the old value, but adding to a historical log of values:
```sh
> noms log http://localhost:8000::people
commit ba3lvopbgcqqnofm3qk7sk4j2doroj1l
Parent: f0b1befu9jp82r1vcd4gmuhdno27uobi
(root) {
+ struct Person {
+ family: "Snow",
+ given: "Jon",
+ male: true,
+ }
}
commit f0b1befu9jp82r1vcd4gmuhdno27uobi
Parent: hshltip9kss28uu910qadq04mhk9kuko
commit hshltip9kss28uu910qadq04mhk9kuko
Parent: None
```
## Values
Noms supports a [variety of datatypes](https://github.com/attic-labs/noms/blob/master/doc/intro.md#types) beyond List, Struct, String, and Bool we used above.
## Samples
You can continue learning more about the Noms Go SDK by looking at the documentation and by reviewing the [samples](https://github.com/attic-labs/noms/blob/master/samples/go). The [hr sample](https://github.com/attic-labs/noms/blob/master/samples/go/hr) is a more complete implementation of our example above and will help you to see further usage of the other datatypes.
================================================
FILE: doc/intro.md
================================================
[Home](../README.md) »
**Technical Overview** | [Use Cases](../README.md#use-cases) | [Command-Line Interface](cli-tour.md) | [Go bindings Tour](go-tour.md) | [Path Syntax](spelling.md) | [FAQ](faq.md)
<br><br>
# Noms Technical Overview
Most conventional database systems share two central properties:
1. Data is modeled as a single point-in-time. Once a transaction commits, the previous state of the database is either lost, or available only as a fallback by reconstructing from transaction logs.
2. Data is modeled as a single source of truth. Even large-scale distributed databases which are internally a fault-tolerant network of nodes, present the abstraction to clients of being a single logical master, with which clients must coordinate in order to change state.
Noms blends the properties of decentralized systems, such as [Git](https://git-scm.com/), with properties of traditional databases in order to create a general-purpose decentralized database, in which:
1. Any peer’s state is as valid as any other.
2. All commits of the database are retained and available at any time.
3. Any peer is free to move forward independently of communication from any other—while retaining the ability to reconcile changes at some point in the future.
4. The basic properties of structured databases (efficient queries, updates, and range scans) are retained.
5. Diffs between any two sets of data can be computed efficiently.
6. Synchronization between disconnected copies of the database can be performed efficiently and correctly.
## Basics
As in Git, [Bitcoin](https://bitcoin.org/en/), [Ethereum](https://www.ethereum.org/), [IPFS](https://ipfs.io/), [Camlistore](https://camlistore.org/), [bup](https://bup.github.io/), and other systems, Noms models data as a [directed acyclic graph](https://en.wikipedia.org/wiki/Directed_acyclic_graph) of nodes in which every node has a _hash_. A node's hash is derived from the values encoded in the node and (transitively) from the values encoded in all nodes which are reachable from that node.
In other words, a Noms database is a single large [Merkle DAG](https://github.com/jbenet/random-ideas/issues/20).
When two nodes have the same hash, they represent identical logical values and the respective subgraph of nodes reachable from each are topologically equivalent. Importantly, in Noms, the reverse is also true: a single logical value has one and only one hash. When two nodes have differnet hashes, they represent different logical values.
Noms extends the ideas of prior systems to enable efficiently computing and reconciling differences, synchronizing state, and building indexes over large-scale, structured data.
## Databases and Datasets
A _database_ is the top-level abstraction in Noms.
A database has two responsibilities: it provides storage of [content-addressed](https://en.wikipedia.org/wiki/Content-addressable_storage) chunks of data, and it keeps track of zero or more _datasets_.
A Noms database can be implemented on top of any underlying storage system that provides key/value storage with at least optional optimistic concurrency. We only use optimistic concurrency to store the current value of each dataset. Chunks themselves are immutable.
We have implementations of Noms databases on top of our own file-backed store [Noms Block Store (NBS)](https://github.com/attic-labs/noms/tree/master/go/nbs) (usually used locally), our own [HTTP protocol](https://github.com/attic-labs/noms/blob/master/go/datas/database_server.go) (used for working with a remote database), [Amazon DynamoDB](https://aws.amazon.com/dynamodb/), and [memory](https://github.com/attic-labs/noms/blob/master/go/chunks/memory_store.go) (mainly used for testing).
Here's an example of creating an http-backed database using the [Go Noms SDK](go-tour.md):
```go
package main
import (
"fmt"
"os"
"github.com/attic-labs/noms/go/spec"
)
func main() {
sp, err := spec.ForDatabase("http://localhost:8000")
if err != nil {
fmt.Fprintf(os.Stderr, "Could not access database: %s\n", err)
return
}
defer sp.Close()
}
```
A dataset is nothing more than a named pointer into the DAG. Consider the following command to copy the dataset named `foo` to the dataset named `bar` within a database:
```shell
noms sync http://localhost:8000::foo http://localhost:8000::bar
```
This command is trivial and causes basically zero IO. Noms first resolves the dataset name `foo` in `http://localhost:8000`. This results in a hash. Noms then checks whether that hash exists in the destination database (which in this case is the same as the source database), finds that it does, and then adds a new dataset pointing at that chunk.
Syncs across database can be efficient by the same logic if the destination database already has all or most of the chunks required chunks.
## Time
All data in Noms is immutable. Once a piece of data is stored, it is never changed. To represent state changes, Noms uses a progression of `Commit` structures.
[TODO - diagram]
As in Git, Commits typically have one _parent_, which is the previous commit in time. But in the cases of merges, a Noms commit can have multiple parents.
### Chunks
When a value is stored in Noms, it is stored as one or more chunks of data. Chunk boundaries are typically created implicitly, as a way to store large collections efficiently (see [Prolly Trees](#prolly-trees-probabilistic-b-trees)). Programmers can also create explicit chunk boundaries using the `Ref` type (see [Types](#types )).
[TODO - Diagram]
Every chunk encodes a single logical value (which may be a component of another value and/or be composed of sub-values). Chunks are [addressed](https://en.wikipedia.org/wiki/Content-addressable_storage) in the Noms persistence layer by the hash of the value they encode.
## Types
Noms is a typed system, meaning that every Noms value is classified into one of the following _types_:
* `Boolean`
* `Number` (arbitrary precision binary)
* `String` (utf8-encoded)
* `Blob` (raw binary data)
* `Set<T>`
* `List<T>`
* `Map<K,V>`
* Unions: `T|U|V|...`
* `Ref<T>` (explicit out-of-line references)
* `Struct` (user-defined record types, e.g., `Struct Person { name: String, age?: Number })`
* `Type` (A value that stores a Noms type)
Blobs, sets, lists, and maps can be gigantic - Noms will _chunk_ these types into reasonable sized parts internally for efficient storage, searching, and updating (see [Prolly Trees](#prolly-trees-probabilistic-b-trees) below for more on this).
Strings, numbers, unions, and structs are not chunked, and should be used for "reasonably-sized" values. Use `Ref` if you need to force a particular value to be in a different chunk for some reason.
Types serve several purposes in Noms:
1. Most importantly, types make Noms data self-describing. You can use the `types.TypeOf` function on any Noms `Value`, no matter how large, and get a very precise description of the entire value and all values reachable from it. This allows software to interoperate without prior agreement or planning.
2. Users of Noms can define their own structures and publish data that uses them. This allows for ad-hoc standardization of types within communities working on similar data.
3. Types can be used _structurally_. A program can check incoming data against a required type. If the incoming root chunk matches the type, or is a superset of it, then the program can proceed with certainty of the shape of all accessible data. This enables richer interoperability between software, since schemas can be expanded over time as long as a compatible subset remains.
4. Eventually, we plan to add type restrictions to datasets, which would enforce the allowed types that can be committed to a dataset. This would allow something akin to schema validation in traditional databases.
### Refs vs Hashes
A _hash_ in Noms is just like the hashes used elsewhere in computing: a short string of bytes that uniquely identifies a larger value. Every value in Noms has a hash. Noms currently uses the [sha2-512](https://github.com/attic-labs/noms/blob/master/go/hash/hash.go#L7) hash function, but that can change in future versions of the system.
A _ref_ is different in subtle, but important ways. A `Ref` is a part of the type system - a `Ref` is a value. Anywhere you can find a Noms value, you can find a `Ref`. For example, you can commit a `Ref<T>` to a dataset, but you can't commit a bare hash.
The difference is that `Ref` carries the type of its target, along with the hash. This allows us to efficiently validate commits that include `Ref`, among other things.
### Type Accretion
Noms is an immutable database, which leads to the question: How do you change the schema? If I have a dataset containing `Set<Number>`, and I later decide that it should be `Set<String>`, what do I do?
You might say that you just commit the new type, but that would mean that users can't look at a dataset and understand what types previous versions contained, without manually exploring every one of those commits.
We call our solution to this problem _Type Accretion_.
If you construct a `Set` containing only `Number`s, its type will be `Set<Number>`. If you then insert a string into this set, the type of the resulting value is `Set<Number|String>`.
This is usually completely implicit, done based on the data you store (you can set types explicitly though, which is useful in some cases).
We do the same thing for datasets. If you commit a `Set<Number>`, the type of the commit we create for you is:
```go
Struct Commit {
Value: Set<Number>
Parents: Set<Ref<Cycle<Commit>>>
}
```
This tells you that the current and all previous commits have values of type `Set<Number>`.
But if you then commit a `Set<String>` to this same dataset, then the type of that commit will be:
```go
Struct Commit {
Value: Set<String>
Parents: Set<Ref<Cycle<Commit>> |
Ref<Struct Commit {
Value: Set<Number>
Parents: Cycle<Commit>
}>>
}
}
```
This tells you that the dataset's current commit has a value of type `Set<String>` and that previous commits are either the same, or else have a value of type `Set<Number>`.
Type accretion has a number of benefits related to schema changes:
1. You can widen the type of any container (list, set, map) without rewriting any existing data. `Set<Struct { name: String }>` becomes `Set<Struct { name: String }> | Struct { name: String, age: Number }>>` and all existing data is reused.
2. You can widen containers in ways that other databases wouldn't allow. For example, you can go from `Set<Number>` to `Set<Number|String>`. Existing data is still reused.
3. You can change the type of a dataset in either direction - either widening or narrowing it, and the dataset remains self-documenting as to its current and previous types.
## Prolly Trees: Probabilistic B-Trees
A critical invariant of Noms is [history-independence](https://arxiv.org/pdf/1501.06508.pdf): the same Noms value will be represented by the same graph of physical chunks, and the same hashes, regardless of what past sequence of logical mutations resulted in the value. This is what makes fast diff, sync, and merge possible in Noms: we can compare two values just by looking at their hash. If their hashes are identical, we know the values are identical without additional work. By modeling collections as trees of values, the same trick can be used to quickly find the differences between larges sets of values.
But Noms is also a database, and needs to do what databases do: efficiently search, scan, and mutate large collections. The classic data structures that enable these features inside databases — B-Trees and LSM Trees — can't be used by Noms because they aren't history-independent: their internal state depends upon their mutation history.
In order to model large mutable collections in Noms, of the type where B-Trees would typically be used, while preserving efficient diff, sync, and merge, Noms introduces _Prolly Trees_.
### Prolly Tree Structure
A Prolly Tree is a [search tree](https://en.wikipedia.org/wiki/Search_tree) where the number of values stored in each node is determined probabilistically, based on the data which is stored in the tree.
A Prolly Tree is similar in many ways to a B-Tree, except that the number of values in each node has a probabilistic average rather than an enforced upper and lower bound, and the set of values in each node is determined by the output of a rolling hash function over the values, rather than via split and join operations when upper and lower bounds are exceeded.
Like B-Trees, Prolly Trees can model lists, maps, tables, and sets. Below is an example of a small set of ASCII characters stored in a Prolly Tree:
### Prolly Tree Construction
Prolly Trees can be constructed from scratch using a variation of [content-slicing](https://en.wikipedia.org/wiki/Rolling_hash#Content-based_slicing_using_a_rolling_hash), as used in bup, rsync, Camlistore, and many other systems.
To start, we "chunk" the serialization of a larged sorted sequence by sliding a fixed-size window through it, one byte at a time.
At each position, we compute a hash of the bytes in the window. Any hash can be used, but in Noms a [rolling hash](https://en.wikipedia.org/wiki/Rolling_hash) is used for performance.
Within each hash, we look for a pattern that has a known probability of occuring. If the pattern is found, that position is a _boundary_. We slide the window forward to the end of the containing item, and write a new _chunk_ containing the bytes between this boundary and the previous, if any. The resulting chunk is stored in a [content-addressed storage system](https://en.wikipedia.org/wiki/Content-addressable_storage). Again, any hash can be used for this, but in Noms [we use truncated SHA-512](https://github.com/attic-labs/noms/blob/master/go/hash/hash.go).
By adjusting the pattern we look for, we can control the average size of the chunks our tree will be decomposed into.
In Noms, the pattern we look for is the [12 high bits being 1](https://github.com/attic-labs/noms/blob/master/go/types/rolling_value_hasher.go). Since this has a probability of 1/2^12, the average chunk size in Noms is 4kb.
Once we've created an initial pass of chunks this way, we build an index describing the contents of each of those chunks, and perform the chunking operation again on the serialization of that index. This continues recursively, until we are left with a node that doesn't chunk. This is the root of the tree.
Noms uses a window size of 64 bytes, so the probability of any 1 bit change moving a boundary is about 64/4kb ~= 0.016.
### Prolly Tree Mutation
To mutate a Prolly Tree, conceptually we build a new Prolly Tree from scratch, except that we can reuse everything from the previous tree that we know cannot have been affected (because it is outside the window).
In the example above, we insert the value _I_ into the set. The chunk boundary is unchanged in this case so the subtrees before and after the modified chunk can be reused as-is.
Every so often (1.6% of the time in Noms) a write will move a chunk boundary. This results in one extra chunk getting written at that level in the tree. This can happen at each level, so the expected number of operations to make a change to a prolly tree is 1.016*treedepth.
A 4-level Prolly Tree in Noms can hold 4096^4 ~= 281TB of data. We can make a single mutation to that tree with about 4 4kb writes.
### Some Properties of Prolly Trees
Operation | B-Trees | Patricia Trees† / HAMTs | Prolly Trees
--------- | ------- | ----------------------- | ------------
1 Random Read | 🎉logk(n) | 🎉logk(n) | 🎉logk(n)
1 Random Write | 🎉logk(n) | 💩2*logk(n) | 👍(1+k/w)*logk(n)
Ordered scan of one item with size z | 🎉z/k | ❌ | 🎉z/k
Calculate diff of size d | 💩n | 🎉d | 🎉d
Verification, proofs | ❌ | 🙌 | 🙌
Structured sharing | ❌ | 🙌 | 🙌
**†** assuming hashed keys, unhashed destroys perf — **n**: total leaf data in tree, **k**: average block size, **w**: window width
### Indexing and Searching with Prolly Trees
Like B-Trees, Prolly Trees are sorted. In Noms, we sort keys of type Boolean, Number, and String by their natural order. We sort other key types by their hash.
Because of this sorting, Noms collections can be used as efficient indexes, in the same manner as primary and secondary indexes in traditional databases.
For example, say you want to quickly be able to find `Person` structs by their age. You could build a map of type `Map<Number, Set<Person>>`. This would allow you to quickly (~log<sub>k</sub>(n) seeks) find all the people of an exact age. But it would _also_ allow you to find all people within a range of ages efficiently (~num_results/log<sub>k</sub>(n) seeks), even if the ages are non-integral.
Also, because Noms collections are ordered search trees, it is possible to implement set operations like union and intersect efficiently on them.
So, for example, if you wanted to find all the people of a particular age AND having a particular hair color, you could construct a second map having type `Map<String, Set<Person>>`, and intersect the two sets.
Over time, we plan to develop this basic capability into support for some kind of generalized query system.
================================================
FILE: doc/spelling.md
================================================
[Home](../README.md) »
[Technical Overview](intro.md) | [Use Cases](../README.md#use-cases) | [Command-Line Interface](cli-tour.md) | [Go bindings Tour](go-tour.md) | **Path Syntax** | [FAQ](faq.md)
<br><br>
# Spelling in Noms
Many commands and APIs in Noms accept database, dataset, or value specifications as arguments. This document describes how to construct these specifications.
## Spelling Databases
database specifications take the form:
```nohighlight
<protocol>[:<path>]
```
The `path` part of the name is interpreted differently depending on the protocol:
- **http(s)** specs describe a remote database to be accessed over HTTP. In this case, the entire database spec is a normal http(s) URL. For example: `https://dev.noms.io/aa`.
- **mem** specs describe an ephemeral memory-backed database. In this case, the path component is not used and must be empty.
- **nbs** specs describe a local [Noms Block Store (NBS)](https://github.com/attic-labs/noms/tree/master/go/nbs)-backed database. In this case, the path component should be a relative or absolute path on disk to a directory in which to store the data, e.g. `nbs:/tmp/noms-data`.
- In Go, `nbs:` can be ommitted (just `/tmp/noms-data` will work).
- **aws** specs describe a remote Noms Block Store backed directly by Amazon Web Services, specifically DynamoDB and S3. The format is a URI containing the names of the DynamoDB table to use, the S3 bucket to use, and the database to serve. For example: `aws:dynamo-table/s3-bucket/database`.
## Spelling Datasets
Dataset specifications take the form:
```nohighlight
<database>::<dataset>
```
See [spelling databases](#spelling-databases) for how to build the `database` part of the name. The `dataset` part is just any string matching the regex `^[a-zA-Z0-9\-_/]+$`.
Example datasets:
```nohighlight
/tmp/test-db::my-dataset
nbs:/tmp/test-db::my-dataset
http://localhost:8000::registered-businesses
https://demo.noms.io/aa::music
```
## Spelling Values
Value specifications take the form:
```nohighlight
<database>::<root><path>
```
See [spelling databases](#spelling-databases) for how to build the database part of the name.
The `root` part can be either a hash or a dataset name. If `root` begins with `#` it will be interpreted as a hash otherwise it is used as a dataset name. See [spelling datasets](#spelling-datasets) for how to build the dataset part of the name.
The `path` part is relative to the `root` provided.
### Specifying Struct Fields
Elements of a Noms struct can be referenced using a period `.`.
For example, if the `root` is a dataset, then one can use `.value` to get the root of the data in the dataset. In this case `.value` selects the `value` field from the `Commit` struct at the top of the dataset. One could instead use `.meta` to select the `meta` struct from the `Commit` struct. The `root` does not need to be a dataset though, so if it is a hash that references a struct, the same notation still works: `#o38hugtf3l1e8rqtj89mijj1dq57eh4m.field`.
### Specifying Collection Values
Elements of a Noms list, map, or set can be retrieved using brackets `[...]`.
For example, if the dataset is a Noms map of number to struct then one could use `.value[42]` to get the Noms struct associated with the key 42. Similarly selecting the first element from a Noms list would be `.value[0]`. If the Noms map was keyed by string, then using `.value["0000024-02-999"]` would reference the Noms struct associated with key "0000024-02-999".
Noms lists also support indexing from the back, using `.value[-1]` to mean the last element of a last, `.value[-2]` for the 2nd last, and so on.
If the key of a Noms map or set is a Noms struct or a more complex value, then indexing into the collection can be done using the hash of that more complex value. For example, if the `root` of our dataset is a Noms set of Noms structs, then if you provide the hash of the struct element then you can index into the map using the brackets as described above. e.g. http://localhost:8000::dataset.value[#o38hugtf3l1e8rqtj89mijj1dq57eh4m].field
Similarly, the key is addressable using `@key` syntax. One use for this is when you have the hash of a complex value, but want need to retrieve the key (rather than or in addition to the value) in a Noms map. The syntax is to append `@key` after the closing bracket of the index specifier. e.g. http://localhost:8000::dataset.value[#o38hugtf3l1e8rqtj89mijj1dq57eh4m]@key would retrieve the key element specified by the hash key `#o38hugtf3l1e8rqtj89mijj1dq57eh4m` from the `dataset.value` collection.
### Specifying Collection Positions
Elements of a Noms list, map, or set can be retrived _by their position_ using the `@at(index)` annotation.
For lists, this is exactly equivalent to `[index]`. For sets and maps, note that Noms has a stable ordering, so `@at(0)` will always return the smallest element, `@at(1)` the 2nd smallest, and so on. `@at(-1)` will return the largest. For maps, adding the `@key` annotation will retrieve the key of the map entry instead of the value.
### Examples
```sh
# “sf-registered-business” dataset at https://demo.noms.io/cli-tour
https://demo.noms.io/cli-tour::sf-registered-business
# value o38hugtf3l1e8rqtj89mijj1dq57eh4m at https://localhost:8000
https://localhost:8000/monkey::#o38hugtf3l1e8rqtj89mijj1dq57eh4m
# “bonk” dataset at /foo/bar
/foo/bar::bonk
# from https://demo.noms.io/cli-tour, select the "sf-registered-business" dataset,
# the root value is a Noms map, select the value of the Noms map identified by string
# key "0000024-02-999", then from that resulting struct select the Ownership_Name field
https://demo.noms.io/cli-tour::sf-registered-business.value["0000024-02-999"].Ownership_Name
```
Be careful with shell escaping. Your shell might require escaping of the double quotes and other characters or use single quotes around the entire command line argument. e.g.:
```sh
> noms show https://demo.noms.io/cli-tour::sf-registered-business.value["0000024-02-999"].Ownership_Name
error: Invalid index: 0000024-02-999
> noms show https://demo.noms.io/cli-tour::sf-registered-business.value[\"0000024-02-999\"].Ownership_Name
"EASTMAN KODAK CO"
> noms show 'https://demo.noms.io/cli-tour::sf-registered-business.value["0000024-02-999"].Ownership_Name'
"EASTMAN KODAK CO"
```
================================================
FILE: go/chunks/chunk.go
================================================
// Copyright 2016 Attic Labs, Inc. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
// Package chunks provides facilities for representing, storing, and fetching content-addressed chunks of Noms data.
package chunks
import (
"bytes"
"github.com/attic-labs/noms/go/d"
"github.com/attic-labs/noms/go/hash"
)
// Chunk is a unit of stored data in noms
type Chunk struct {
r hash.Hash
data []byte
}
var EmptyChunk = NewChunk([]byte{})
func (c Chunk) Hash() hash.Hash {
return c.r
}
func (c Chunk) Data() []byte {
return c.data
}
func (c Chunk) IsEmpty() bool {
return len(c.data) == 0
}
// NewChunk creates a new Chunk backed by data. This means that the returned Chunk has ownership of this slice of memory.
func NewChunk(data []byte) Chunk {
r := hash.Of(data)
return Chunk{r, data}
}
// NewChunkWithHash creates a new chunk with a known hash. The hash is not re-calculated or verified. This should obviously only be used in cases where the caller already knows the specified hash is correct.
func NewChunkWithHash(r hash.Hash, data []byte) Chunk {
return Chunk{r, data}
}
// ChunkWriter wraps an io.WriteCloser, additionally providing the ability to grab the resulting Chunk for all data written through the interface. Calling Chunk() or Close() on an instance disallows further writing.
type ChunkWriter struct {
buffer *bytes.Buffer
c Chunk
}
func NewChunkWriter() *ChunkWriter {
b := &bytes.Buffer{}
return &ChunkWriter{
buffer: b,
}
}
func (w *ChunkWriter) Write(data []byte) (int, error) {
if w.buffer == nil {
d.Panic("Write() cannot be called after Hash() or Close().")
}
size, err := w.buffer.Write(data)
d.Chk.NoError(err)
return size, nil
}
// Chunk() closes the writer and returns the resulting Chunk.
func (w *ChunkWriter) Chunk() Chunk {
d.Chk.NoError(w.Close())
return w.c
}
// Close() closes computes the hash and Puts it into the ChunkSink Note: The Write() method never returns an error. Instead, like other noms interfaces, errors are reported via panic.
func (w *ChunkWriter) Close() error {
if w.buffer == nil {
return nil
}
w.c = NewChunk(w.buffer.Bytes())
w.buffer = nil
return nil
}
================================================
FILE: go/chunks/chunk_serializer.go
================================================
// Copyright 2016 Attic Labs, Inc. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
package chunks
import (
"bytes"
"encoding/binary"
"io"
"github.com/attic-labs/noms/go/d"
"github.com/attic-labs/noms/go/hash"
)
/*
Chunk Serialization:
Chunk 0
Chunk 1
..
Chunk N
Chunk:
Hash // 20-byte hash
Len // 4-byte int
Data // len(Data) == Len
*/
// Serialize a single Chunk to writer.
func Serialize(chunk Chunk, writer io.Writer) {
d.PanicIfFalse(chunk.data != nil)
h := chunk.Hash()
n, err := io.Copy(writer, bytes.NewReader(h[:]))
d.Chk.NoError(err)
d.PanicIfFalse(int64(hash.ByteLen) == n)
// Because of chunking at higher levels, no chunk should never be more than 4GB
chunkSize := uint32(len(chunk.Data()))
err = binary.Write(writer, binary.BigEndian, chunkSize)
d.Chk.NoError(err)
n, err = io.Copy(writer, bytes.NewReader(chunk.Data()))
d.Chk.NoError(err)
d.PanicIfFalse(uint32(n) == chunkSize)
}
// Deserialize reads off of |reader| until EOF, sending chunks to
// chunkChan in the order they are read. Objects sent over chunkChan are
// *Chunk.
func Deserialize(reader io.Reader, chunkChan chan<- *Chunk) (err error) {
for {
var c Chunk
c, err = deserializeChunk(reader)
if err != nil {
break
}
d.Chk.NotEqual(EmptyChunk.Hash(), c.Hash())
chunkChan <- &c
}
if err == io.EOF {
err = nil
}
return
}
func deserializeChunk(reader io.Reader) (Chunk, error) {
h := hash.Hash{}
n, err := io.ReadFull(reader, h[:])
if err != nil {
return EmptyChunk, err
}
d.PanicIfFalse(int(hash.ByteLen) == n)
chunkSize := uint32(0)
if err = binary.Read(reader, binary.BigEndian, &chunkSize); err != nil {
return EmptyChunk, err
}
data := make([]byte, int(chunkSize))
if n, err = io.ReadFull(reader, data); err != nil {
return EmptyChunk, err
}
d.PanicIfFalse(int(chunkSize) == n)
c := NewChunk(data)
if h != c.Hash() {
d.Panic("%s != %s", h, c.Hash().String())
}
return c, nil
}
================================================
FILE: go/chunks/chunk_serializer_test.go
================================================
// Copyright 2016 Attic Labs, Inc. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
package chunks
import (
"bytes"
"testing"
"github.com/stretchr/testify/assert"
)
func TestSerializeRoundTrip(t *testing.T) {
assert := assert.New(t)
inputs := [][]byte{[]byte("abc"), []byte("def")}
chnx := make([]Chunk, len(inputs))
for i, data := range inputs {
chnx[i] = NewChunk(data)
}
buf := &bytes.Buffer{}
Serialize(chnx[0], buf)
Serialize(chnx[1], buf)
chunkChan := make(chan *Chunk)
go func() {
defer close(chunkChan)
err := Deserialize(bytes.NewReader(buf.Bytes()), chunkChan)
assert.NoError(err)
}()
for c := range chunkChan {
assert.Equal(chnx[0].Hash(), c.Hash())
chnx = chnx[1:]
}
assert.Len(chnx, 0)
}
func TestBadSerialization(t *testing.T) {
bad := []byte{0, 1} // Not enough bytes to read first length
ch := make(chan *Chunk)
defer close(ch)
assert.Error(t, Deserialize(bytes.NewReader(bad), ch))
}
================================================
FILE: go/chunks/chunk_store.go
================================================
// Copyright 2016 Attic Labs, Inc. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
package chunks
import (
"io"
"github.com/attic-labs/noms/go/hash"
)
// ChunkStore is the core storage abstraction in noms. We can put data
// anyplace we have a ChunkStore implementation for.
type ChunkStore interface {
// Get the Chunk for the value of the hash in the store. If the hash is
// absent from the store EmptyChunk is returned.
Get(h hash.Hash) Chunk
// GetMany gets the Chunks with |hashes| from the store. On return,
// |foundChunks| will have been fully sent all chunks which have been
// found. Any non-present chunks will silently be ignored.
GetMany(hashes hash.HashSet, foundChunks chan *Chunk)
// Returns true iff the value at the address |h| is contained in the
// store
Has(h hash.Hash) bool
// Returns a new HashSet containing any members of |hashes| that are
// absent from the store.
HasMany(hashes hash.HashSet) (absent hash.HashSet)
// Put caches c in the ChunkSource. Upon return, c must be visible to
// subsequent Get and Has calls, but must not be persistent until a call
// to Flush(). Put may be called concurrently with other calls to Put(),
// Get(), GetMany(), Has() and HasMany().
Put(c Chunk)
// Returns the NomsVersion with which this ChunkSource is compatible.
Version() string
// Rebase brings this ChunkStore into sync with the persistent storage's
// current root.
Rebase()
// Root returns the root of the database as of the time the ChunkStore
// was opened or the most recent call to Rebase.
Root() hash.Hash
// Commit atomically attempts to persist all novel Chunks and update the
// persisted root hash from last to current (or keeps it the same).
// If last doesn't match the root in persistent storage, returns false.
Commit(current, last hash.Hash) bool
// Stats may return some kind of struct that reports statistics about the
// ChunkStore instance. The type is implementation-dependent, and impls
// may return nil
Stats() interface{}
// StatsSummary may return a string containing summarized statistics for
// this ChunkStore. It must return "Unsupported" if this operation is not
// supported.
StatsSummary() string
// Close tears down any resources in use by the implementation. After
// Close(), the ChunkStore may not be used again. It is NOT SAFE to call
// Close() concurrently with any other ChunkStore method; behavior is
// undefined and probably crashy.
io.Closer
}
// Factory allows the creation of namespaced ChunkStore instances. The details
// of how namespaces are separated is left up to the particular implementation
// of Factory and ChunkStore.
type Factory interface {
CreateStore(ns string) ChunkStore
// CreateStoreFromCache allows caller to signal to the factory that it's
// willing to tolerate an out-of-date ChunkStore.
CreateStoreFromCache(ns string) ChunkStore
// Shutter shuts down the factory. Subsequent calls to CreateStore() will fail.
Shutter()
}
================================================
FILE: go/chunks/chunk_store_common_test.go
================================================
// Copyright 2016 Attic Labs, Inc. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
package chunks
import (
"github.com/stretchr/testify/suite"
"github.com/attic-labs/noms/go/constants"
"github.com/attic-labs/noms/go/hash"
)
type ChunkStoreTestSuite struct {
suite.Suite
Factory Factory
}
func (suite *ChunkStoreTestSuite) TestChunkStorePut() {
store := suite.Factory.CreateStore("ns")
input := "abc"
c := NewChunk([]byte(input))
store.Put(c)
h := c.Hash()
// Reading it via the API should work.
assertInputInStore(input, h, store, suite.Assert())
}
func (suite *ChunkStoreTestSuite) TestChunkStoreRoot() {
store := suite.Factory.CreateStore("ns")
oldRoot := store.Root()
suite.True(oldRoot.IsEmpty())
bogusRoot := hash.Parse("8habda5skfek1265pc5d5l1orptn5dr0")
newRoot := hash.Parse("8la6qjbh81v85r6q67lqbfrkmpds14lg")
// Try to update root with bogus oldRoot
result := store.Commit(newRoot, bogusRoot)
suite.False(result)
// Now do a valid root update
result = store.Commit(newRoot, oldRoot)
suite.True(result)
}
func (suite *ChunkStoreTestSuite) TestChunkStoreCommitPut() {
name := "ns"
store := suite.Factory.CreateStore(name)
input := "abc"
c := NewChunk([]byte(input))
store.Put(c)
h := c.Hash()
// Reading it via the API should work...
assertInputInStore(input, h, store, suite.Assert())
// ...but it shouldn't be persisted yet
assertInputNotInStore(input, h, suite.Factory.CreateStore(name), suite.Assert())
store.Commit(h, store.Root()) // Commit persists Chunks
assertInputInStore(input, h, store, suite.Assert())
assertInputInStore(input, h, suite.Factory.CreateStore(name), suite.Assert())
}
func (suite *ChunkStoreTestSuite) TestChunkStoreGetNonExisting() {
store := suite.Factory.CreateStore("ns")
h := hash.Parse("11111111111111111111111111111111")
c := store.Get(h)
suite.True(c.IsEmpty())
}
func (suite *ChunkStoreTestSuite) TestChunkStoreVersion() {
store := suite.Factory.CreateStore("ns")
oldRoot := store.Root()
suite.True(oldRoot.IsEmpty())
newRoot := hash.Parse("11111222223333344444555556666677")
suite.True(store.Commit(newRoot, oldRoot))
suite.Equal(constants.NomsVersion, store.Version())
}
func (suite *ChunkStoreTestSuite) TestChunkStoreCommitUnchangedRoot() {
store1, store2 := suite.Factory.CreateStore("ns"), suite.Factory.CreateStore("ns")
input := "abc"
c := NewChunk([]byte(input))
store1.Put(c)
h := c.Hash()
// Reading c from store1 via the API should work...
assertInputInStore(input, h, store1, suite.Assert())
// ...but not store2.
assertInputNotInStore(input, h, store2, suite.Assert())
store1.Commit(store1.Root(), store1.Root())
store2.Rebase()
// Now, reading c from store2 via the API should work...
assertInputInStore(input, h, store2, suite.Assert())
}
================================================
FILE: go/chunks/chunk_test.go
================================================
// Copyright 2016 Attic Labs, Inc. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
package chunks
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestChunk(t *testing.T) {
c := NewChunk([]byte("abc"))
h := c.Hash()
// See http://www.di-mgt.com.au/sha_testvectors.html
assert.Equal(t, "rmnjb8cjc5tblj21ed4qs821649eduie", h.String())
}
func TestChunkWriteAfterCloseFails(t *testing.T) {
assert := assert.New(t)
input := "abc"
w := NewChunkWriter()
_, err := w.Write([]byte(input))
assert.NoError(err)
assert.NoError(w.Close())
assert.Panics(func() { w.Write([]byte(input)) }, "Write() after Close() should barf!")
}
func TestChunkWriteAfterChunkFails(t *testing.T) {
assert := assert.New(t)
input := "abc"
w := NewChunkWriter()
_, err := w.Write([]byte(input))
assert.NoError(err)
_ = w.Chunk()
assert.Panics(func() { w.Write([]byte(input)) }, "Write() after Chunk() should barf!")
}
func TestChunkChunkCloses(t *testing.T) {
assert := assert.New(t)
input := "abc"
w := NewChunkWriter()
_, err := w.Write([]byte(input))
assert.NoError(err)
w.Chunk()
assert.Panics(func() { w.Write([]byte(input)) }, "Write() after Close() should barf!")
}
================================================
FILE: go/chunks/memory_store.go
================================================
// Copyright 2016 Attic Labs, Inc. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
package chunks
import (
"sync"
"github.com/attic-labs/noms/go/constants"
"github.com/attic-labs/noms/go/d"
"github.com/attic-labs/noms/go/hash"
)
// MemoryStorage provides a "persistent" storage layer to back multiple
// MemoryStoreViews. A MemoryStorage instance holds the ground truth for the
// root and set of chunks that are visible to all MemoryStoreViews vended by
// NewView(), allowing them to implement the transaction-style semantics that
// ChunkStore requires.
type MemoryStorage struct {
data map[hash.Hash]Chunk
rootHash hash.Hash
mu sync.RWMutex
}
// NewView vends a MemoryStoreView backed by this MemoryStorage. It's
// initialized with the currently "persisted" root.
func (ms *MemoryStorage) NewView() ChunkStore {
return &MemoryStoreView{storage: ms, rootHash: ms.rootHash}
}
// Get retrieves the Chunk with the Hash h, returning EmptyChunk if it's not
// present.
func (ms *MemoryStorage) Get(h hash.Hash) Chunk {
ms.mu.RLock()
defer ms.mu.RUnlock()
if c, ok := ms.data[h]; ok {
return c
}
return EmptyChunk
}
// Has returns true if the Chunk with the Hash h is present in ms.data, false
// if not.
func (ms *MemoryStorage) Has(r hash.Hash) bool {
ms.mu.RLock()
defer ms.mu.RUnlock()
_, ok := ms.data[r]
return ok
}
// Len returns the number of Chunks in ms.data.
func (ms *MemoryStorage) Len() int {
ms.mu.RLock()
defer ms.mu.RUnlock()
return len(ms.data)
}
// Root returns the currently "persisted" root hash of this in-memory store.
func (ms *MemoryStorage) Root() hash.Hash {
ms.mu.RLock()
defer ms.mu.RUnlock()
return ms.rootHash
}
// Update checks the "persisted" root against last and, iff it matches,
// updates the root to current, adds all of novel to ms.data, and returns
// true. Otherwise returns false.
func (ms *MemoryStorage) Update(current, last hash.Hash, novel map[hash.Hash]Chunk) bool {
ms.mu.Lock()
defer ms.mu.Unlock()
if last != ms.rootHash {
return false
}
if ms.data == nil {
ms.data = map[hash.Hash]Chunk{}
}
for h, c := range novel {
ms.data[h] = c
}
ms.rootHash = current
return true
}
// MemoryStoreView is an in-memory implementation of store.ChunkStore. Useful
// mainly for tests.
// The proper way to get one:
// storage := &MemoryStorage{}
// ms := storage.NewView()
type MemoryStoreView struct {
pending map[hash.Hash]Chunk
rootHash hash.Hash
mu sync.RWMutex
storage *MemoryStorage
}
func (ms *MemoryStoreView) Get(h hash.Hash) Chunk {
ms.mu.RLock()
defer ms.mu.RUnlock()
if c, ok := ms.pending[h]; ok {
return c
}
return ms.storage.Get(h)
}
func (ms *MemoryStoreView) GetMany(hashes hash.HashSet, foundChunks chan *Chunk) {
for h := range hashes {
c := ms.Get(h)
if !c.IsEmpty() {
foundChunks <- &c
}
}
return
}
func (ms *MemoryStoreView) Has(h hash.Hash) bool {
ms.mu.RLock()
defer ms.mu.RUnlock()
if _, ok := ms.pending[h]; ok {
return true
}
return ms.storage.Has(h)
}
func (ms *MemoryStoreView) HasMany(hashes hash.HashSet) hash.HashSet {
absent := hash.HashSet{}
for h := range hashes {
if !ms.Has(h) {
absent.Insert(h)
}
}
return absent
}
func (ms *MemoryStoreView) Version() string {
return constants.NomsVersion
}
func (ms *MemoryStoreView) Put(c Chunk) {
ms.mu.Lock()
defer ms.mu.Unlock()
if ms.pending == nil {
ms.pending = map[hash.Hash]Chunk{}
}
ms.pending[c.Hash()] = c
}
func (ms *MemoryStoreView) Len() int {
ms.mu.RLock()
defer ms.mu.RUnlock()
return len(ms.pending) + ms.storage.Len()
}
func (ms *MemoryStoreView) Rebase() {
ms.mu.Lock()
defer ms.mu.Unlock()
ms.rootHash = ms.storage.Root()
}
func (ms *MemoryStoreView) Root() hash.Hash {
ms.mu.RLock()
defer ms.mu.RUnlock()
return ms.rootHash
}
func (ms *MemoryStoreView) Commit(current, last hash.Hash) bool {
ms.mu.Lock()
defer ms.mu.Unlock()
if last != ms.rootHash {
return false
}
success := ms.storage.Update(current, last, ms.pending)
if success {
ms.pending = nil
}
ms.rootHash = ms.storage.Root()
return success
}
func (ms *MemoryStoreView) Stats() interface{} {
return nil
}
func (ms *MemoryStoreView) StatsSummary() string {
return "Unsupported"
}
func (ms *MemoryStoreView) Close() error {
return nil
}
type memoryStoreFactory struct {
stores map[string]*MemoryStorage
mu *sync.Mutex
}
func NewMemoryStoreFactory() Factory {
return &memoryStoreFactory{map[string]*MemoryStorage{}, &sync.Mutex{}}
}
func (f *memoryStoreFactory) CreateStoreFromCache(ns string) ChunkStore {
return f.CreateStore(ns)
}
func (f *memoryStoreFactory) CreateStore(ns string) ChunkStore {
f.mu.Lock()
defer f.mu.Unlock()
if f.stores == nil {
d.Panic("Cannot use memoryStoreFactory after Shutter().")
}
if ms, present := f.stores[ns]; present {
return ms.NewView()
}
f.stores[ns] = &MemoryStorage{}
return f.stores[ns].NewView()
}
func (f *memoryStoreFactory) Shutter() {
f.stores = nil
}
================================================
FILE: go/chunks/memory_store_test.go
================================================
// Copyright 2016 Attic Labs, Inc. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
package chunks
import (
"testing"
"github.com/stretchr/testify/suite"
)
func TestMemoryStoreTestSuite(t *testing.T) {
suite.Run(t, &MemoryStoreTestSuite{})
}
type MemoryStoreTestSuite struct {
ChunkStoreTestSuite
}
func (suite *MemoryStoreTestSuite) SetupTest() {
suite.Factory = NewMemoryStoreFactory()
}
func (suite *MemoryStoreTestSuite) TearDownTest() {
suite.Factory.Shutter()
}
================================================
FILE: go/chunks/put_cache.go
================================================
// Copyright 2016 Attic Labs, Inc. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
package chunks
import (
"sync"
"github.com/attic-labs/noms/go/hash"
)
func newUnwrittenPutCache() *unwrittenPutCache {
return &unwrittenPutCache{map[hash.Hash]Chunk{}, &sync.Mutex{}}
}
type unwrittenPutCache struct {
unwrittenPuts map[hash.Hash]Chunk
mu *sync.Mutex
}
func (p *unwrittenPutCache) Add(c Chunk) bool {
p.mu.Lock()
defer p.mu.Unlock()
if _, ok := p.unwrittenPuts[c.Hash()]; !ok {
p.unwrittenPuts[c.Hash()] = c
return true
}
return false
}
func (p *unwrittenPutCache) Has(c Chunk) (has bool) {
p.mu.Lock()
defer p.mu.Unlock()
_, has = p.unwrittenPuts[c.Hash()]
return
}
func (p *unwrittenPutCache) Get(r hash.Hash) Chunk {
p.mu.Lock()
defer p.mu.Unlock()
if c, ok := p.unwrittenPuts[r]; ok {
return c
}
return EmptyChunk
}
func (p *unwrittenPutCache) Clear(chunks []Chunk) {
p.mu.Lock()
defer p.mu.Unlock()
for _, c := range chunks {
delete(p.unwrittenPuts, c.Hash())
}
}
================================================
FILE: go/chunks/remote_requests.go
================================================
// Copyright 2016 Attic Labs, Inc. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
package chunks
import (
"sync"
"github.com/attic-labs/noms/go/hash"
)
type ReadRequest interface {
Hashes() hash.HashSet
Outstanding() OutstandingRequest
}
func NewGetRequest(r hash.Hash, ch chan<- *Chunk) GetRequest {
return GetRequest{hash.HashSet{r: struct{}{}}, ch}
}
type GetRequest struct {
hashes hash.HashSet
ch chan<- *Chunk
}
func NewGetManyRequest(hashes hash.HashSet, wg *sync.WaitGroup, ch chan<- *Chunk) GetManyRequest {
return GetManyRequest{hashes, wg, ch}
}
type GetManyRequest struct {
hashes hash.HashSet
wg *sync.WaitGroup
ch chan<- *Chunk
}
func NewAbsentRequest(r hash.Hash, ch chan<- bool) AbsentRequest {
return AbsentRequest{hash.HashSet{r: struct{}{}}, ch}
}
type AbsentRequest struct {
hashes hash.HashSet
ch chan<- bool
}
func NewAbsentManyRequest(hashes hash.HashSet, wg *sync.WaitGroup, ch chan<- hash.Hash) AbsentManyRequest {
return AbsentManyRequest{hashes, wg, ch}
}
type AbsentManyRequest struct {
hashes hash.HashSet
wg *sync.WaitGroup
ch chan<- hash.Hash
}
func (g GetRequest) Hashes() hash.HashSet {
return g.hashes
}
func (g GetRequest) Outstanding() OutstandingRequest {
return OutstandingGet(g.ch)
}
func (g GetManyRequest) Hashes() hash.HashSet {
return g.hashes
}
func (g GetManyRequest) Outstanding() OutstandingRequest {
return OutstandingGetMany{g.wg, g.ch}
}
func (h AbsentRequest) Hashes() hash.HashSet {
return h.hashes
}
func (h AbsentRequest) Outstanding() OutstandingRequest {
return OutstandingAbsent(h.ch)
}
func (h AbsentManyRequest) Hashes() hash.HashSet {
return h.hashes
}
func (h AbsentManyRequest) Outstanding() OutstandingRequest {
return OutstandingAbsentMany{h.wg, h.ch}
}
type OutstandingRequest interface {
Satisfy(h hash.Hash, c *Chunk)
Fail()
}
type OutstandingGet chan<- *Chunk
type OutstandingGetMany struct {
wg *sync.WaitGroup
ch chan<- *Chunk
}
type OutstandingAbsent chan<- bool
type OutstandingAbsentMany struct {
wg *sync.WaitGroup
ch chan<- hash.Hash
}
func (r OutstandingGet) Satisfy(h hash.Hash, c *Chunk) {
r <- c
}
func (r OutstandingGet) Fail() {
r <- &EmptyChunk
}
func (ogm OutstandingGetMany) Satisfy(h hash.Hash, c *Chunk) {
ogm.ch <- c
ogm.wg.Done()
}
func (ogm OutstandingGetMany) Fail() {
ogm.wg.Done()
}
func (oh OutstandingAbsent) Satisfy(h hash.Hash, c *Chunk) {
oh <- false
}
func (oh OutstandingAbsent) Fail() {
oh <- true
}
func (ohm OutstandingAbsentMany) Satisfy(h hash.Hash, c *Chunk) {
ohm.ch <- h
ohm.wg.Done()
}
func (ohm OutstandingAbsentMany) Fail() {
ohm.wg.Done()
}
// ReadBatch represents a set of queued Get/Has requests, each of which are blocking on a receive channel for a response.
type ReadBatch map[hash.Hash][]OutstandingRequest
// Close ensures that callers to Get() and Has() are failed correctly if the corresponding chunk wasn't in the response from the server (i.e. it wasn't found).
func (rb *ReadBatch) Close() error {
for _, reqs := range *rb {
for _, req := range reqs {
req.Fail()
}
}
return nil
}
================================================
FILE: go/chunks/remote_requests_test.go
================================================
// Copyright 2016 Attic Labs, Inc. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
package chunks
import (
"sync"
"testing"
"github.com/attic-labs/noms/go/hash"
"github.com/stretchr/testify/assert"
)
func TestGetRequestBatch(t *testing.T) {
assert := assert.New(t)
h0 := hash.Parse("00000000000000000000000000000000")
c1 := NewChunk([]byte("abc"))
h1 := c1.Hash()
c2 := NewChunk([]byte("123"))
h2 := c2.Hash()
tally := func(b bool, trueCnt, falseCnt *int) {
if b {
*trueCnt++
} else {
*falseCnt++
}
}
req0chan := make(chan bool, 1)
req1chan := make(chan *Chunk, 1)
req2chan := make(chan bool, 1)
req3chan := make(chan bool, 1)
req4chan := make(chan *Chunk, 1)
defer func() { close(req0chan); close(req1chan); close(req2chan); close(req3chan); close(req4chan) }()
batch := ReadBatch{
h0: []OutstandingRequest{OutstandingAbsent(req0chan), OutstandingGet(req1chan)},
h1: []OutstandingRequest{OutstandingAbsent(req2chan)},
h2: []OutstandingRequest{OutstandingAbsent(req3chan), OutstandingGet(req4chan)},
}
go func() {
for requestedHash, reqs := range batch {
for _, req := range reqs {
if requestedHash == h1 {
req.Satisfy(h1, &c1)
delete(batch, h1)
} else if requestedHash == h2 {
req.Satisfy(h2, &c2)
delete(batch, h2)
}
}
}
batch.Close()
}()
var r0True, r0False, r2True, r2False, r3True, r3False int
b := <-req0chan
tally(b, &r0True, &r0False)
c := <-req1chan
assert.EqualValues(EmptyChunk.Hash(), c.Hash())
b = <-req2chan
tally(b, &r2True, &r2False)
b = <-req3chan
tally(b, &r3True, &r3False)
c = <-req4chan
assert.EqualValues(c2.Hash(), c.Hash())
assert.Equal(1, r0True)
assert.Equal(0, r0False)
assert.Equal(0, r2True)
assert.Equal(1, r2False)
assert.Equal(0, r3True)
assert.Equal(1, r3False)
}
func TestGetManyRequestBatch(t *testing.T) {
assert := assert.New(t)
h0 := hash.Parse("00000000000000000000000000000000")
c1 := NewChunk([]byte("abc"))
h1 := c1.Hash()
c2 := NewChunk([]byte("123"))
h2 := c2.Hash()
chunks := make(chan *Chunk)
hashes := hash.NewHashSet(h0, h1, h2)
wg := &sync.WaitGroup{}
wg.Add(len(hashes))
go func() { wg.Wait(); close(chunks) }()
req := NewGetManyRequest(hashes, wg, chunks)
batch := ReadBatch{
h0: {req.Outstanding()},
h1: {req.Outstanding()},
h2: {req.Outstanding()},
}
go func() {
for reqHash, reqs := range batch {
for _, req := range reqs {
if reqHash == h1 {
req.Satisfy(h1, &c1)
delete(batch, h1)
} else if reqHash == h2 {
req.Satisfy(h2, &c2)
delete(batch, h2)
}
}
}
batch.Close()
}()
for c := range chunks {
hashes.Remove(c.Hash())
}
assert.Len(hashes, 1)
assert.True(hashes.Has(h0))
}
func TestAbsentManyRequestBatch(t *testing.T) {
assert := assert.New(t)
h0 := hash.Parse("00000000000000000000000000000000")
c1 := NewChunk([]byte("abc"))
h1 := c1.Hash()
c2 := NewChunk([]byte("123"))
h2 := c2.Hash()
found := make(chan hash.Hash)
hashes := hash.NewHashSet(h0, h1, h2)
wg := &sync.WaitGroup{}
wg.Add(len(hashes))
go func() { wg.Wait(); close(found) }()
req := NewAbsentManyRequest(hashes, wg, found)
batch := ReadBatch{}
for h := range req.Hashes() {
batch[h] = []OutstandingRequest{req.Outstanding()}
}
go func() {
for reqHash, reqs := range batch {
for _, req := range reqs {
if reqHash == h1 {
req.Satisfy(h1, &EmptyChunk)
delete(batch, h1)
} else if reqHash == h2 {
req.Satisfy(h2, &EmptyChunk)
delete(batch, h2)
}
}
}
batch.Close()
}()
for h := range found {
hashes.Remove(h)
}
assert.Len(hashes, 1)
assert.True(hashes.Has(h0))
}
================================================
FILE: go/chunks/test_utils.go
================================================
// Copyright 2016 Attic Labs, Inc. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
package chunks
import (
"github.com/attic-labs/noms/go/d"
"github.com/attic-labs/noms/go/hash"
"github.com/stretchr/testify/assert"
)
func assertInputInStore(input string, h hash.Hash, s ChunkStore, assert *assert.Assertions) {
chunk := s.Get(h)
assert.False(chunk.IsEmpty(), "Shouldn't get empty chunk for %s", h.String())
assert.Equal(input, string(chunk.Data()))
}
func assertInputNotInStore(input string, h hash.Hash, s ChunkStore, assert *assert.Assertions) {
chunk := s.Get(h)
assert.True(chunk.IsEmpty(), "Shouldn't get non-empty chunk for %s: %v", h.String(), chunk)
}
type TestStorage struct {
MemoryStorage
}
func (t *TestStorage) NewView() *TestStoreView {
return &TestStoreView{ChunkStore: t.MemoryStorage.NewView()}
}
type TestStoreView struct {
ChunkStore
Reads int
Hases int
Writes int
}
func (s *TestStoreView) Get(h hash.Hash) Chunk {
s.Reads++
return s.ChunkStore.Get(h)
}
func (s *TestStoreView) GetMany(hashes hash.HashSet, foundChunks chan *Chunk) {
s.Reads += len(hashes)
s.ChunkStore.GetMany(hashes, foundChunks)
}
func (s *TestStoreView) Has(h hash.Hash) bool {
s.Hases++
return s.ChunkStore.Has(h)
}
func (s *TestStoreView) HasMany(hashes hash.HashSet) hash.HashSet {
s.Hases += len(hashes)
return s.ChunkStore.HasMany(hashes)
}
func (s *TestStoreView) Put(c Chunk) {
s.Writes++
s.ChunkStore.Put(c)
}
type TestStoreFactory struct {
stores map[string]*TestStorage
}
func NewTestStoreFactory() *TestStoreFactory {
return &TestStoreFactory{map[string]*TestStorage{}}
}
func (f *TestStoreFactory) CreateStore(ns string) ChunkStore {
if f.stores == nil {
d.Panic("Cannot use TestStoreFactory after Shutter().")
}
if ts, present := f.stores[ns]; present {
return ts.NewView()
}
f.stores[ns] = &TestStorage{}
return f.stores[ns].NewView()
}
func (f *TestStoreFactory) Shutter() {
f.stores = nil
}
================================================
FILE: go/config/config.go
================================================
// Copyright 2016 Attic Labs, Inc. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
package config
import (
"bytes"
"errors"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"github.com/BurntSushi/toml"
"github.com/attic-labs/noms/go/spec"
)
type Config struct {
File string
Db map[string]DbConfig
}
type DbConfig struct {
Url string
}
const (
NomsConfigFile = ".nomsconfig"
DefaultDbAlias = "default"
)
var NoConfig = errors.New(fmt.Sprintf("no %s found", NomsConfigFile))
// Find the closest directory containing .nomsconfig starting
// in cwd and then searching up ancestor tree.
// Look first looking in cwd and then up through its ancestors
func FindNomsConfig() (*Config, error) {
curDir, err := os.Getwd()
if err != nil {
return nil, err
}
for {
nomsConfig := filepath.Join(curDir, NomsConfigFile)
info, err := os.Stat(nomsConfig)
if err == nil && !info.IsDir() {
// found
return ReadConfig(nomsConfig)
} else if err != nil && !os.IsNotExist(err) {
// can't read
return nil, err
}
nextDir := filepath.Dir(curDir)
if nextDir == curDir {
// stop at root
return nil, NoConfig
}
curDir = nextDir
}
}
func ReadConfig(name string) (*Config, error) {
data, err := ioutil.ReadFile(name)
if err != nil {
return nil, err
}
c, err := NewConfig(string(data))
if err != nil {
return nil, err
}
c.File = name
return qualifyPaths(name, c)
}
func NewConfig(data string) (*Config, error) {
c := new(Config)
if _, err := toml.Decode(data, c); err != nil {
return nil, err
}
return c, nil
}
func (c *Config) WriteTo(configHome string) (string, error) {
file := filepath.Join(configHome, NomsConfigFile)
if err := os.MkdirAll(filepath.Dir(file), os.ModePerm); err != nil {
return "", err
}
if err := ioutil.WriteFile(file, []byte(c.writeableString()), os.ModePerm); err != nil {
return "", err
}
return file, nil
}
// Replace relative directory in path part of spec with an absolute
// directory. Assumes the path is relative to the location of the config file
func absDbSpec(configHome string, url string) string {
dbSpec, err := spec.ForDatabase(url)
if err != nil {
return url
}
if dbSpec.Protocol != "nbs" {
return url
}
dbName := dbSpec.DatabaseName
if !filepath.IsAbs(dbName) {
dbName = filepath.Join(configHome, dbName)
}
return "nbs:" + dbName
}
func qualifyPaths(configPath string, c *Config) (*Config, error) {
file, err := filepath.Abs(configPath)
if err != nil {
return nil, err
}
dir := filepath.Dir(file)
qc := *c
qc.File = file
for k, r := range c.Db {
qc.Db[k] = DbConfig{absDbSpec(dir, r.Url)}
}
return &qc, nil
}
func (c *Config) String() string {
var buffer bytes.Buffer
if c.File != "" {
buffer.WriteString(fmt.Sprintf("file = %s\n", c.File))
}
buffer.WriteString(c.writeableString())
return buffer.String()
}
func (c *Config) writeableString() string {
var buffer bytes.Buffer
for k, r := range c.Db {
buffer.WriteString(fmt.Sprintf("[db.%s]\n", k))
buffer.WriteString(fmt.Sprintf("\t"+`url = "%s"`+"\n", r.Url))
}
return buffer.String()
}
================================================
FILE: go/config/config_test.go
================================================
// Copyright 2016 Attic Labs, Inc. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
package config
import (
"io/ioutil"
"os"
"path/filepath"
"strings"
"testing"
"github.com/attic-labs/noms/go/spec"
"github.com/stretchr/testify/assert"
)
const (
nbsSpec = "nbs:./local"
memSpec = "mem"
httpSpec = "http://test.com:8080/foo"
nbsAbsSpec = "nbs:/tmp/noms"
remoteAlias = "origin"
)
var (
ctestRoot = os.TempDir()
ldbConfig = &Config{
"",
map[string]DbConfig{
DefaultDbAlias: {nbsSpec},
remoteAlias: {httpSpec},
},
}
httpConfig = &Config{
"",
map[string]DbConfig{
DefaultDbAlias: {httpSpec},
remoteAlias: {nbsSpec},
},
}
memConfig = &Config{
"",
map[string]DbConfig{
DefaultDbAlias: {memSpec},
remoteAlias: {httpSpec},
},
}
ldbAbsConfig = &Config{
"",
map[string]DbConfig{
DefaultDbAlias: {nbsAbsSpec},
remoteAlias: {httpSpec},
},
}
)
type paths struct {
home string
config string
}
func getPaths(assert *assert.Assertions, base string) paths {
abs, err := filepath.Abs(ctestRoot)
assert.NoError(err)
abs, err = filepath.EvalSymlinks(ctestRoot)
assert.NoError(err)
home := filepath.Join(abs, base)
config := filepath.Join(home, NomsConfigFile)
return paths{home, config}
}
func qualifyFilePath(assert *assert.Assertions, path string) string {
p, err := filepath.Abs(path)
assert.NoError(err)
return p
}
func assertDbSpecsEquiv(assert *assert.Assertions, expected string, actual string) {
e, err := spec.ForDatabase(expected)
assert.NoError(err)
if e.Protocol != "nbs" {
assert.Equal(expected, actual)
} else {
a, err := spec.ForDatabase(actual)
assert.NoError(err)
assert.Equal(e.Protocol, a.Protocol, actual)
if filepath.IsAbs(e.DatabaseName) {
assert.Equal(e.DatabaseName, a.DatabaseName, actual)
} else {
// If the original path is relative, it will return as absolute.
// All we do here is ensure that the path suffix is the same.
eName := strings.TrimPrefix(e.DatabaseName, ".")
assert.True(strings.HasSuffix(a.DatabaseName, eName),
"expected: %s; actual: %s", eName, actual)
}
}
}
func validateConfig(assert *assert.Assertions, file string, e *Config, a *Config) {
assert.Equal(qualifyFilePath(assert, file), qualifyFilePath(assert, a.File))
assert.Equal(len(e.Db), len(a.Db))
for k, er := range e.Db {
ar, ok := a.Db[k]
assert.True(ok)
assertDbSpecsEquiv(assert, er.Url, ar.Url)
}
}
func writeConfig(assert *assert.Assertions, c *Config, home string) string {
file, err := c.WriteTo(home)
assert.NoError(err, home)
return file
}
func TestConfig(t *testing.T) {
assert := assert.New(t)
path := getPaths(assert, "home")
writeConfig(assert, ldbConfig, path.home)
// Test from home
assert.NoError(os.Chdir(path.home))
c, err := FindNomsConfig()
assert.NoError(err, path.config)
validateConfig(assert, path.config, ldbConfig, c)
// Test from subdir
subdir := filepath.Join(path.home, "subdir")
assert.NoError(os.MkdirAll(subdir, os.ModePerm))
assert.NoError(os.Chdir(subdir))
c, err = FindNomsConfig()
assert.NoError(err, path.config)
validateConfig(assert, path.config, ldbConfig, c)
// Test from subdir with intervening .nomsconfig directory
nomsDir := filepath.Join(subdir, NomsConfigFile)
err = os.MkdirAll(nomsDir, os.ModePerm)
assert.NoError(err, nomsDir)
assert.NoError(os.Chdir(subdir))
c, err = FindNomsConfig()
assert.NoError(err, path.config)
validateConfig(assert, path.config, ldbConfig, c)
}
func TestUnreadableConfig(t *testing.T) {
// BUG 3816
if os.Getenv("DOCKER") != "" {
t.Skip("Skipping testing in Docker environment")
}
assert := assert.New(t)
path := getPaths(assert, "home.unreadable")
writeConfig(assert, ldbConfig, path.home)
assert.NoError(os.Chmod(path.config, 0333)) // write-only
assert.NoError(os.Chdir(path.home))
_, err := FindNomsConfig()
assert.Error(err, path.config)
}
func TestNoConfig(t *testing.T) {
assert := assert.New(t)
path := getPaths(assert, "home.none")
assert.NoError(os.MkdirAll(path.home, os.ModePerm))
assert.NoError(os.Chdir(path.home))
_, err := FindNomsConfig()
assert.Equal(NoConfig, err)
}
func TestBadConfig(t *testing.T) {
assert := assert.New(t)
path := getPaths(assert, "home.bad")
cfile := writeConfig(assert, ldbConfig, path.home)
// overwrite with something invalid
assert.NoError(ioutil.WriteFile(cfile, []byte("invalid config"), os.ModePerm))
assert.NoError(os.Chdir(path.home))
_, err := FindNomsConfig()
assert.Error(err, path.config)
}
func TestQualifyingPaths(t *testing.T) {
assert := assert.New(t)
path := getPaths(assert, "home")
assert.NoError(os.Chdir(path.home))
for _, tc := range []*Config{httpConfig, memConfig, ldbAbsConfig} {
writeConfig(assert, tc, path.home)
ac, err := FindNomsConfig()
assert.NoError(err, path.config)
validateConfig(assert, path.config, tc, ac)
}
}
func TestCwd(t *testing.T) {
assert := assert.New(t)
cwd, err := os.Getwd()
assert.NoError(err)
cwd = filepath.Join(cwd, "test")
abs, err := filepath.Abs("test")
assert.NoError(err)
assert.Equal(cwd, abs)
}
================================================
FILE: go/config/resolver.go
================================================
// Copyright 2016 Attic Labs, Inc. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
package config
import (
"fmt"
"strings"
"github.com/attic-labs/noms/go/chunks"
"github.com/attic-labs/noms/go/datas"
"github.com/attic-labs/noms/go/spec"
"github.com/attic-labs/noms/go/types"
"github.com/attic-labs/noms/go/util/verbose"
)
type Resolver struct {
config *Config
dotDatapath string // set to the first datapath that was resolved
}
// A Resolver enables using db defaults, db aliases and dataset '.' replacement in command
// line arguments when a .nomsconfig file is present. To use it, create a config resolver
// before command line processing and use it to resolve each dataspec argument in
// succession.
func NewResolver() *Resolver {
c, err := FindNomsConfig()
if err != nil {
if err != NoConfig {
panic(fmt.Errorf("Failed to read .nomsconfig due to: %v", err))
}
return &Resolver{}
}
return &Resolver{c, ""}
}
// Print replacement if one occurred
func (r *Resolver) verbose(orig string, replacement string) string {
if orig != replacement {
if orig == "" {
orig = `""`
}
verbose.Log("\tresolving %s -> %s\n", orig, replacement)
}
return replacement
}
// Resolve string to database name. If config is defined:
// - replace the empty string with the default db url
// - replace any db alias with it's url
func (r *Resolver) ResolveDbSpec(str string) string {
if r.config != nil {
if str == "" {
return r.config.Db[DefaultDbAlias].Url
}
if val, ok := r.config.Db[str]; ok {
return val.Url
}
}
return str
}
// Resolve string to dataset or path name.
// - replace database name as described in ResolveDatabase
// - if this is the first call to ResolvePath, remember the
// datapath part for subsequent calls.
// - if this is not the first call and a "." is used, replace
// it with the first datapath.
func (r *Resolver) ResolvePathSpec(str string) string {
if r.config != nil {
split := strings.SplitN(str, spec.Separator, 2)
db, rest := "", split[0]
if len(split) > 1 {
db, rest = split[0], split[1]
}
if r.dotDatapath == "" {
r.dotDatapath = rest
} else if rest == "." {
rest = r.dotDatapath
}
return r.ResolveDbSpec(db) + spec.Separator + rest
}
return str
}
// Resolve string to database spec. If a config is present,
// - resolve a db alias to its db spec
// - resolve "" to the default db spec
func (r *Resolver) GetDatabase(str string) (datas.Database, error) {
sp, err := spec.ForDatabase(r.verbose(str, r.ResolveDbSpec(str)))
if err != nil {
return nil, err
}
return sp.GetDatabase(), nil
}
// Resolve string to a chunkstore. Like ResolveDatabase, but returns the underlying ChunkStore
func (r *Resolver) GetChunkStore(str string) (chunks.ChunkStore, error) {
sp, err := spec.ForDatabase(r.verbose(str, r.ResolveDbSpec(str)))
if err != nil {
return nil, err
}
return sp.NewChunkStore(), nil
}
// Resolve string to a dataset. If a config is present,
// - if no db prefix is present, assume the default db
// - if the db prefix is an alias, replace it
func (r *Resolver) GetDataset(str string) (datas.Database, datas.Dataset, error) {
sp, err := spec.ForDataset(r.verbose(str, r.ResolvePathSpec(str)))
if err != nil {
return nil, datas.Dataset{}, err
}
return sp.GetDatabase(), sp.GetDataset(), nil
}
// Resolve string to a value path. If a config is present,
// - if no db spec is present, assume the default db
// - if the db spec is an alias, replace it
func (r *Resolver) GetPath(str string) (datas.Database, types.Value, error) {
sp, err := spec.ForPath(r.verbose(str, r.ResolvePathSpec(str)))
if err != nil {
return nil, nil, err
}
return sp.GetDatabase(), sp.GetValue(), nil
}
================================================
FILE: go/config/resolver_test.go
================================================
// Copyright 2016 Attic Labs, Inc. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
package config
import (
"fmt"
"os"
"path/filepath"
"testing"
"github.com/attic-labs/noms/go/spec"
"github.com/stretchr/testify/assert"
)
const (
localSpec = nbsSpec
remoteSpec = httpSpec
testDs = "testds"
testObject = "#pckdvpvr9br1fie6c3pjudrlthe7na18"
)
type testData struct {
input string
expected string
}
var (
rtestRoot = os.TempDir()
rtestConfig = &Config{
"",
map[string]DbConfig{
DefaultDbAlias: {localSpec},
remoteAlias: {remoteSpec},
},
}
dbTestsNoAliases = []testData{
{localSpec, localSpec},
{remoteSpec, remoteSpec},
}
dbTestsWithAliases = []testData{
{"", localSpec},
{remoteAlias, remoteSpec},
}
pathTestsNoAliases = []testData{
{remoteSpec + "::" + testDs, remoteSpec + "::" + testDs},
{remoteSpec + "::" + testObject, remoteSpec + "::" + testObject},
}
pathTestsWithAliases = []testData{
{testDs, localSpec + "::" + testDs},
{remoteAlias + "::" + testDs, remoteSpec + "::" + testDs},
{testObject, localSpec + "::" + testObject},
{remoteAlias + "::" + testObject, remoteSpec + "::" + testObject},
}
)
func withConfig(t *testing.T) *Resolver {
assert := assert.New(t)
dir := filepath.Join(rtestRoot, "with-config")
_, err := rtestConfig.WriteTo(dir)
assert.NoError(err, dir)
assert.NoError(os.Chdir(dir))
r := NewResolver() // resolver must be created after changing directory
return r
}
func withoutConfig(t *testing.T) *Resolver {
assert := assert.New(t)
dir := filepath.Join(rtestRoot, "without-config")
assert.NoError(os.MkdirAll(dir, os.ModePerm), dir)
assert.NoError(os.Chdir(dir))
r := NewResolver() // resolver must be created after changing directory
return r
}
func assertPathSpecsEquiv(assert *assert.Assertions, expected string, actual string) {
e, err := spec.ForPath(expected)
assert.NoError(err)
a, err := spec.ForPath(actual)
assert.NoError(err)
databaseSpec := func(sp spec.Spec) string {
return fmt.Sprintf("%s:%s", sp.Protocol, sp.DatabaseName)
}
assertDbSpecsEquiv(assert, databaseSpec(e), databaseSpec(a))
assert.Equal(e.Path.String(), a.Path.String())
}
func TestResolveDatabaseWithConfig(t *testing.T) {
r := withConfig(t)
assert := assert.New(t)
for _, d := range append(dbTestsNoAliases, dbTestsWithAliases...) {
db := r.ResolveDbSpec(d.input)
assertDbSpecsEquiv(assert, d.expected, db)
}
}
func TestResolvePathWithConfig(t *testing.T) {
r := withConfig(t)
assert := assert.New(t)
for _, d := range append(pathTestsNoAliases, pathTestsWithAliases...) {
path := r.ResolvePathSpec(d.input)
assertPathSpecsEquiv(assert, d.expected, path)
}
}
func TestResolveDatabaseWithoutConfig(t *testing.T) {
r := withoutConfig(t)
assert := assert.New(t)
for _, d := range dbTestsNoAliases {
db := r.ResolveDbSpec(d.input)
assert.Equal(d.expected, db, d.input)
}
}
func TestResolvePathWithoutConfig(t *testing.T) {
r := withoutConfig(t)
assert := assert.New(t)
for _, d := range pathTestsNoAliases {
path := r.ResolvePathSpec(d.input)
assertPathSpecsEquiv(assert, d.expected, path)
}
}
func TestResolveDestPathWithDot(t *testing.T) {
r := withConfig(t)
assert := assert.New(t)
data := []struct {
src string
dest string
expSrc string
expDest string
}{
{testDs, remoteSpec + "::.", localSpec + "::" + testDs, remoteSpec + "::" + testDs},
{remoteSpec + "::" + testDs, ".", remoteSpec + "::" + testDs, localSpec + "::" + testDs},
}
for _, d := range data {
src := r.ResolvePathSpec(d.src)
dest := r.ResolvePathSpec(d.dest)
assertPathSpecsEquiv(assert, d.expSrc, src)
assertPathSpecsEquiv(assert, d.expDest, dest)
}
}
================================================
FILE: go/constants/http.go
================================================
// Copyright 2016 Attic Labs, Inc. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
package constants
const (
RootPath = "/root/"
GetRefsPath = "/getRefs/"
GetBlobPath = "/getBlob/"
HasRefsPath = "/hasRefs/"
WriteValuePath = "/writeValue/"
BasePath = "/"
GraphQLPath = "/graphql/"
StatsPath = "/stats/"
)
================================================
FILE: go/constants/version.go
================================================
// Copyright 2016 Attic Labs, Inc. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
// Package constants collects common constants used in Noms, such as the Noms data format version.
package constants
const NomsVersion = "7.18"
var NomsGitSHA = "<developer build>"
================================================
FILE: go/d/check_error.go
================================================
// Copyright 2016 Attic Labs, Inc. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
package d
import (
"fmt"
"os"
"github.com/attic-labs/kingpin"
"github.com/attic-labs/noms/go/util/exit"
)
func CheckError(err error) {
if err != nil {
fmt.Fprintf(os.Stderr, "error: %s\n", err)
kingpin.Usage()
exit.Fail()
}
}
func CheckErrorNoUsage(err error) {
if err != nil {
fmt.Fprintf(os.Stderr, "error: %s\n", err)
exit.Fail()
}
}
================================================
FILE: go/d/try.go
================================================
// Copyright 2016 Attic Labs, Inc. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
// Package d implements several debug, error and assertion functions used throughout Noms.
package d
import (
"errors"
"fmt"
"reflect"
"github.com/stretchr/testify/assert"
)
// d.Chk.<Method>() -- used in test cases and as assertions
var (
Chk = assert.New(&panicker{})
)
type panicker struct {
}
func (s panicker) Errorf(format string, args ...interface{}) {
panic(fmt.Sprintf(format, args...))
}
// Panic(err) creates an error using format and args and wraps it in a
// WrappedError which can be handled using Try() and TryCatch()
func Panic(format string, args ...interface{}) {
if len(args) == 0 {
err := errors.New(format)
panic(Wrap(err))
}
err := fmt.Errorf(format, args...)
panic(Wrap(err))
}
// PanicIfError(err) && PanicIfTrue(expr) can be used to panic in a way that's
// easily handled by Try() and TryCatch()
func PanicIfError(err error) {
if err != nil {
panic(Wrap(err))
}
}
// If b is true, creates a default error, wraps it and panics.
func PanicIfTrue(b bool) {
if b {
panic(Wrap(errors.New("Expected true")))
}
}
// If b is false, creates a default error, wraps it and panics.
func PanicIfFalse(b bool) {
if !b {
panic(Wrap(errors.New("Expected false")))
}
}
// If 'f' panics with a WrappedError then recover that error.
// If types is empty, return the WrappedError.
// if types is not empty and cause is not one of the listed types, re-panic.
// if types is not empty and cause is one of the types, return 'cause'
func Try(f func(), types ...interface{}) (err error) {
defer recoverWrappedTypes(&err, types)
f()
return
}
// If 'f' panics with a WrappedError then recover that error and return it.
// If types is empty, return the WrappedError.
// if types is not empty and cause is not one of the listed types, re-panic.
// if types is not empty and cause is one of the types, return 'cause'
func TryCatch(f func(), catch func(err error) error) (err error) {
defer recoverWrapped(&err, catch)
f()
return
}
type WrappedError interface {
Error() string
Cause() error
}
// Wraps an error. The enclosing error has a default Error() that contains the error msg along
// with a backtrace. The original error can be retrieved by calling err.Cause().
func Wrap(err error) WrappedError {
if err == nil {
return nil
}
if we, ok := err.(WrappedError); ok {
return we
}
st := stackTracer{}
assert := assert.New(&st)
assert.Fail(err.Error())
return wrappedError{st.stackTrace, err}
}
// If err is a WrappedError, then Cause() is returned, otherwise returns err.
func Unwrap(err error) error {
cause := err
we, ok := err.(WrappedError)
if ok {
cause = we.Cause()
}
return cause
}
func causeInTypes(err error, types ...interface{}) bool {
cause := Unwrap(err)
typ := reflect.TypeOf(cause)
for _, t := range types {
if typ == reflect.TypeOf(t) {
return true
}
}
return false
}
// Utility method, that checks type of error and panics with wrapped error not one of the listed types.
func PanicIfNotType(err error, types ...interface{}) error {
if err == nil {
return nil
}
if !causeInTypes(err, types...) {
we, ok := err.(WrappedError)
if !ok {
we = Wrap(err)
}
panic(we)
}
return Unwrap(err)
}
type wrappedError struct {
msg string
cause error
}
func (we wrappedError) Error() string { return we.msg }
func (we wrappedError) Cause() error { return we.cause }
func (we wrappedError) Unwrap() error { return we.cause }
type stackTracer struct {
stackTrace string
}
func (s *stackTracer) Errorf(format string, args ...interface{}) {
s.stackTrace = fmt.Sprintf(format, args...)
}
func recoverWrappedTypes(errp *error, types []interface{}) {
if r := recover(); r != nil {
if wrapper, ok := r.(wrappedError); !ok {
panic(r)
} else if len(types) > 0 && !causeInTypes(wrapper, types...) {
panic(r)
} else if len(types) > 0 {
*errp = wrapper.Cause()
} else {
*errp = wrapper
}
}
}
func recoverWrapped(errp *error, catch func(err error) error) {
if r := recover(); r != nil {
we, ok := r.(wrappedError)
if !ok {
panic(r)
}
if catch != nil {
*errp = catch(we)
} else {
*errp = Unwrap(we)
}
}
}
================================================
FILE: go/d/try_test.go
================================================
// Copyright 2016 Attic Labs, Inc. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
package d
import (
"errors"
"fmt"
"testing"
"github.com/stretchr/testify/assert"
)
var (
te = testError{"te"}
te2 = testError2{"te2"}
)
type testError struct {
s string
}
func (e testError) Error() string { return e.s }
type testError2 struct {
s string
}
func (e testError2) Error() string { return e.s }
func TestTry2(t *testing.T) {
assert := assert.New(t)
assert.Panics(func() {
Try(func() {
panic(te)
})
})
assert.Panics(func() {
Try(func() {
PanicIfError(te)
}, te2)
})
assert.Error(func() error {
return Try(func() {
PanicIfError(te)
})
}())
assert.Error(func() error {
return Try(func() {
PanicIfError(te)
}, testError{})
}())
assert.Nil(func() error {
return Try(func() {
PanicIfError(nil)
})
}())
}
func TestTryCatch(t *testing.T) {
assert := assert.New(t)
assert.Panics(func() {
TryCatch(func() {
panic(Wrap(te))
},
func(err error) error {
if !causeInTypes(err, testError2{}) {
panic(err)
}
return Unwrap(err)
})
})
assert.Panics(func() {
TryCatch(func() {
panic(te)
},
func(err error) error {
if !causeInTypes(err, testError{}) {
panic(err)
}
return Unwrap(err)
})
})
assert.IsType(wrappedError{}, func() error {
return TryCatch(func() {
panic(Wrap(te))
},
func(err error) error {
return err
})
}())
assert.Error(func() error {
return TryCatch(func() {
panic(Wrap(te))
},
func(err error) error {
if !causeInTypes(err, testError2{}, testError{}) {
panic(err)
}
return Unwrap(err)
})
}())
}
func TestUnwrap(t *testing.T) {
assert := assert.New(t)
err := errors.New("test")
we := wrappedError{"test msg", err}
assert.Equal(err, Unwrap(err))
assert.Equal(err, Unwrap(we))
}
func TestPanicIfTrue(t *testing.T) {
assert := assert.New(t)
arg := "arg value"
format := "could be a format: %s"
formatted := fmt.Sprintf(format, arg)
assert.Panics(func() {
PanicIfTrue(true)
})
assert.Panics(func() {
PanicIfTrue(true)
})
assert.NotPanics(func() {
PanicIfTrue(false)
})
err := Try(func() {
Panic(format)
})
assert.Equal(errors.New(format), Unwrap(err))
err = Try(func() {
Panic(format, arg)
})
assert.Equal(errors.New(formatted), Unwrap(err))
}
func TestPanicIfFalse(t *testing.T) {
assert := assert.New(t)
arg := "arg value"
format := "could be a format: %s"
formatted := fmt.Sprintf(format, arg)
assert.Panics(func() {
PanicIfFalse(false)
})
assert.Panics(func() {
PanicIfFalse(false)
})
assert.NotPanics(func() {
PanicIfFalse(true)
})
err := Try(func() {
Panic(format)
})
assert.Equal(errors.New(format), Unwrap(err))
err = Try(func() {
Panic(format, arg)
})
assert.Equal(errors.New(formatted), Unwrap(err))
}
func TestPanicIfNotType(t *testing.T) {
assert := assert.New(t)
te := testError{"te"}
te2 := testError2{"te2"}
assert.Panics(func() {
PanicIfNotType(te, te2)
})
assert.Equal(te, PanicIfNotType(te, te))
assert.Equal(te2, PanicIfNotType(te2, te, te2))
}
func TestCauseInTypes(t *testing.T) {
assert := assert.New(t)
te := testError{"te"}
te2 := testError2{"te2"}
assert.True(causeInTypes(te, te))
assert.True(causeInTypes(te, te2, te))
assert.False(causeInTypes(te, te2))
assert.False(causeInTypes(te))
}
func TestWrap(t *testing.T) {
assert := assert.New(t)
te := testError{"te"}
we := Wrap(te)
assert.Equal(te, we.Cause())
assert.Equal(te, errors.Unwrap(we))
assert.IsType(wrappedError{}, we)
assert.Equal(we, Wrap(we))
fmt.Printf("st: %s, cause: %s\n", we.Error(), we.Cause())
assert.Nil(Wrap(nil))
}
================================================
FILE: go/datas/commit.go
================================================
// Copyright 2016 Attic Labs, Inc. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
package datas
import (
"sort"
"github.com/attic-labs/noms/go/d"
"github.com/attic-labs/noms/go/hash"
"github.com/attic-labs/noms/go/nomdl"
"github.com/attic-labs/noms/go/types"
)
const (
ParentsField = "parents"
ValueField = "value"
MetaField = "meta"
commitName = "Commit"
)
var commitTemplate = types.MakeStructTemplate(commitName, []string{MetaField, ParentsField, ValueField})
var valueCommitType = nomdl.MustParseType(`Struct Commit {
meta: Struct {},
parents: Set<Ref<Cycle<Commit>>>,
value: Value,
}`)
// NewCommit creates a new commit object.
//
// A commit has the following type:
//
// ```
// struct Commit {
// meta: M,
// parents: Set<Ref<Cycle<Commit>>>,
// value: T,
// }
// ```
// where M is a struct type and T is any type.
func NewCommit(value types.Value, parents types.Set, meta types.Struct) types.Struct {
return commitTemplate.NewStruct([]types.Value{meta, parents, value})
}
// FindCommonAncestor returns the most recent common ancestor of c1 and c2, if
// one exists, setting ok to true. If there is no common ancestor, ok is set
// to false.
func FindCommonAncestor(c1, c2 types.Ref, vr types.ValueReader) (a types.Ref, ok bool) {
if !IsRefOfCommitType(types.TypeOf(c1)) {
d.Panic("FindCommonAncestor() called on %s", types.TypeOf(c1).Describe())
}
if !IsRefOfCommitType(types.TypeOf(c2)) {
d.Panic("FindCommonAncestor() called on %s", types.TypeOf(c2).Describe())
}
c1Q, c2Q := &types.RefByHeight{c1}, &types.RefByHeight{c2}
for !c1Q.Empty() && !c2Q.Empty() {
c1Ht, c2Ht := c1Q.MaxHeight(), c2Q.MaxHeight()
if c1Ht == c2Ht {
c1Parents, c2Parents := c1Q.PopRefsOfHeight(c1Ht), c2Q.PopRefsOfHeight(c2Ht)
if common, ok := findCommonRef(c1Parents, c2Parents); ok {
return common, true
}
parentsToQueue(c1Parents, c1Q, vr)
parentsToQueue(c2Parents, c2Q, vr)
} else if c1Ht > c2Ht {
parentsToQueue(c1Q.PopRefsOfHeight(c1Ht), c1Q, vr)
} else {
parentsToQueue(c2Q.PopRefsOfHeight(c2Ht), c2Q, vr)
}
}
return
}
func parentsToQueue(refs types.RefSlice, q *types.RefByHeight, vr types.ValueReader) {
for _, r := range refs {
c := r.TargetValue(vr).(types.Struct)
p := c.Get(ParentsField).(types.Set)
p.IterAll(func(v types.Value) {
q.PushBack(v.(types.Ref))
})
}
sort.Sort(q)
}
func findCommonRef(a, b types.RefSlice) (types.Ref, bool) {
toRefSet := func(s types.RefSlice) map[hash.Hash]types.Ref {
out := map[hash.Hash]types.Ref{}
for _, r := range s {
out[r.TargetHash()] = r
}
return out
}
aSet, bSet := toRefSet(a), toRefSet(b)
for s, r := range aSet {
if _, present := bSet[s]; present {
return r, true
}
}
return types.Ref{}, false
}
func makeCommitStructType(metaType, parentsType, valueType *types.Type) *types.Type {
return types.MakeStructType("Commit",
types.StructField{
Name: MetaField,
Type: metaType,
},
types.StructField{
Name: ParentsField,
Type: parentsType,
},
types.StructField{
Name: ValueField,
Type: valueType,
},
)
}
func getRefElementType(t *types.Type) *types.Type {
d.PanicIfFalse(t.TargetKind() == types.RefKind)
return t.Desc.(types.CompoundDesc).ElemTypes[0]
}
func IsCommitType(t *types.Type) bool {
return types.IsSubtype(valueCommitType, t)
}
func IsCommit(v types.Value) bool {
return types.IsValueSubtypeOf(v, valueCommitType)
}
func IsRefOfCommitType(t *types.Type) bool {
return t.TargetKind() == types.RefKind && IsCommitType(getRefElementType(t))
}
================================================
FILE: go/datas/commit_options.go
================================================
// Copyright 2016 Attic Labs, Inc. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
package datas
import (
"github.com/attic-labs/noms/go/merge"
"github.com/attic-labs/noms/go/types"
)
// CommitOptions is used to pass options into Commit.
type CommitOptions struct {
// Parents, if provided is the parent commits of the commit we are
// creating.
Parents types.Set
// Meta is a Struct that describes arbitrary metadata about this Commit,
// e.g. a timestamp or descriptive text.
Meta types.Struct
// Policy will be called to attempt to merge this Commit with the current
// Head, if this is not a fast-forward. If Policy is nil, no merging will
// be attempted. Note that because Commit() retries in some cases, Policy
// might also be called multiple times with different values.
Policy merge.Policy
}
================================================
FILE: go/datas/commit_test.go
================================================
// Copyright 2016 Attic Labs, Inc. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
package datas
import (
"fmt"
"strings"
"testing"
"github.com/attic-labs/noms/go/chunks"
"github.com/attic-labs/noms/go/nomdl"
"github.com/attic-labs/noms/go/types"
"github.com/stretchr/testify/assert"
)
func TestNewCommit(t *testing.T) {
assert := assert.New(t)
assertTypeEquals := func(e, a *types.Type) {
assert.True(a.Equals(e), "Actual: %s\nExpected %s", a.Describe(), e.Describe())
}
storage := &chunks.TestStorage{}
db := NewDatabase(storage.NewView())
defer db.Close()
commit := NewCommit(types.Number(1), types.NewSet(db), types.EmptyStruct)
at := types.TypeOf(commit)
et := makeCommitStructType(
types.EmptyStructType,
types.MakeSetType(types.MakeUnionType()),
types.NumberType,
)
assertTypeEquals(et, at)
// Committing another Number
commit2 := NewCommit(types.Number(2), types.NewSet(db, types.NewRef(commit)), types.EmptyStruct)
at2 := types.TypeOf(commit2)
et2 := nomdl.MustParseType(`Struct Commit {
meta: Struct {},
parents: Set<Ref<Cycle<Commit>>>,
value: Number,
}`)
assertTypeEquals(et2, at2)
// Now commit a String
commit3 := NewCommit(types.String("Hi"), types.NewSet(db, types.NewRef(commit2)), types.EmptyStruct)
at3 := types.TypeOf(commit3)
et3 := nomdl.MustParseType(`Struct Commit {
meta: Struct {},
parents: Set<Ref<Cycle<Commit>>>,
value: Number | String,
}`)
assertTypeEquals(et3, at3)
// Now commit a String with MetaInfo
meta := types.NewStruct("Meta", types.StructData{"date": types.String("some date"), "number": types.Number(9)})
metaType := nomdl.MustParseType(`Struct Meta {
date: String,
number: Number,
}`)
assertTypeEquals(metaType, types.TypeOf(meta))
commit4 := NewCommit(types.String("Hi"), types.NewSet(db, types.NewRef(commit2)), meta)
at4 := types.TypeOf(commit4)
et4 := nomdl.MustParseType(`Struct Commit {
meta: Struct {} | Struct Meta {
date: String,
number: Number,
},
parents: Set<Ref<Cycle<Commit>>>,
value: Number | String,
}`)
assertTypeEquals(et4, at4)
// Merge-commit with different parent types
commit5 := NewCommit(types.String("Hi"), types.NewSet(db, types.NewRef(commit2), types.NewRef(commit3)), types.EmptyStruct)
at5 := types.TypeOf(commit5)
et5 := nomdl.MustParseType(`Struct Commit {
meta: Struct {},
parents: Set<Ref<Cycle<Commit>>>,
value: Number | String,
}`)
assertTypeEquals(et5, at5)
}
func TestCommitWithoutMetaField(t *testing.T) {
assert := assert.New(t)
storage := &chunks.TestStorage{}
db := NewDatabase(storage.NewView())
defer db.Close()
metaCommit := types.NewStruct("Commit", types.StructData{
"value": types.Number(9),
"parents": types.NewSet(db),
"meta": types.EmptyStruct,
})
assert.True(IsCommit(metaCommit))
assert.True(IsCommitType(types.TypeOf(metaCommit)))
noMetaCommit := types.NewStruct("Commit", types.StructData{
"value": types.Number(9),
"parents": types.NewSet(db),
})
assert.False(IsCommit(noMetaCommit))
assert.False(IsCommitType(types.TypeOf(noMetaCommit)))
}
// Convert list of Struct's to Set<Ref>
func toRefSet(vrw types.ValueReadWriter, commits ...types.Struct) types.Set {
set := types.NewSet(vrw).Edit()
for _, p := range commits {
set.Insert(types.NewRef(p))
}
return set.Set()
}
// Convert Set<Ref<Struct>> to a string of Struct.Get("value")'s
func toValuesString(refSet types.Set, vr types.ValueReader) string {
values := []string{}
refSet.IterAll(func(v types.Value) {
values = append(values, fmt.Sprintf("%v", v.(types.Ref).TargetValue(vr).(types.Struct).Get("value")))
})
return strings.Join(values, ",")
}
func TestFindCommonAncestor(t *testing.T) {
assert := assert.New(t)
storage := &chunks.TestStorage{}
db := NewDatabase(storage.NewView())
defer db.Close()
// Add a commit and return it
addCommit := func(datasetID string, val string, parents ...types.Struct) types.Struct {
ds := db.GetDataset(datasetID)
var err error
ds, err = db.Commit(ds, types.String(val), CommitOptions{Parents: toRefSet(db, parents...)})
assert.NoError(err)
return ds.Head()
}
// Assert that c is the common ancestor of a and b
assertCommonAncestor := func(expected, a, b types.Struct) {
if found, ok := FindCommonAncestor(types.NewRef(a), types.NewRef(b), db); assert.True(ok) {
ancestor := found.TargetValue(db).(types.Struct)
assert.True(
expected.Equals(ancestor),
"%s should be common ancestor of %s, %s. Got %s",
expected.Get(ValueField),
a.Get(ValueField),
b.Get(ValueField),
ancestor.Get(ValueField),
)
}
}
// Build commit DAG
//
// ds-a: a1<-a2<-a3<-a4<-a5<-a6
// ^ ^ ^ |
// | \ \----\ /-/
// | \ \V
// ds-b: \ b3<-b4<-b5
// \
// \
// ds-c: c2<-c3
// /
// /
// V
// ds-d: d1<-d2
//
a, b, c, d := "ds-a", "ds-b", "ds-c", "ds-d"
a1 := addCommit(a, "a1")
d1 := addCommit(d, "d1")
a2 := addCommit(a, "a2", a1)
c2 := addCommit(c, "c2", a1)
d2 := addCommit(d, "d2", d1)
a3 := addCommit(a, "a3", a2)
b3 := addCommit(b, "b3", a2)
c3 := addCommit(c, "c3", c2, d2)
a4 := addCommit(a, "a4", a3)
b4 := addCommit(b, "b4", b3)
a5 := addCommit(a, "a5", a4)
b5 := addCommit(b, "b5", b4, a3)
a6 := addCommit(a, "a6", a5, b5)
assertCommonAncestor(a1, a1, a1) // All self
assertCommonAncestor(a1, a1, a2) // One side self
assertCommonAncestor(a2, a3, b3) // Common parent
assertCommonAncestor(a2, a4, b4) // Common grandparent
assertCommonAncestor(a1, a6, c3) // Traversing multiple parents on both sides
// No common ancestor
if found, ok := FindCommonAncestor(types.NewRef(d2), types.NewRef(a6), db); !assert.False(ok) {
assert.Fail(
"Unexpected common ancestor!",
"Should be no common ancestor of %s, %s. Got %s",
d2.Get(ValueField),
a6.Get(ValueField),
found.TargetValue(db).(types.Struct).Get(ValueField),
)
}
}
func TestNewCommitRegressionTest(t *testing.T) {
storage := &chunks.TestStorage{}
db := NewDatabase(storage.NewView())
defer db.Close()
c1 := NewCommit(types.String("one"), types.NewSet(db), types.EmptyStruct)
cx := NewCommit(types.Bool(true), types.NewSet(db), types.EmptyStruct)
value := types.String("two")
parents := types.NewSet(db, types.NewRef(c1))
meta := types.NewStruct("", types.StructData{
"basis": cx,
})
// Used to fail
NewCommit(value, parents, meta)
}
================================================
FILE: go/datas/database.go
================================================
// Copyright 2016 Attic Labs, Inc. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
// Package datas defines and implements the database layer used in Noms.
package datas
import (
"io"
"github.com/attic-labs/noms/go/chunks"
"github.com/attic-labs/noms/go/types"
)
// Database provides versioned storage for noms values. While Values can be
// directly read and written from a Database, it is generally more appropriate
// to read data by inspecting the Head of a Dataset and write new data by
// updating the Head of a Dataset via Commit() or similar. Particularly, new
// data is not guaranteed to be persistent until after a Commit (Delete,
// SetHead, or FastForward) operation completes.
// The Database API is stateful, meaning that calls to GetDataset() or
// Datasets() occurring after a call to Commit() (et al) will represent the
// result of the Commit().
type Database interface {
// To implement types.ValueWriter, Database implementations provide
// WriteValue(). WriteValue() writes v to this Database, though v is not
// guaranteed to be be persistent until after a subsequent Commit(). The
// return value is the Ref of v.
// Written values won't be persisted until a commit-alike
types.ValueReadWriter
// Close must have no side-effects
io.Closer
// Datasets returns the root of the database which is a
// Map<String, Ref<Commit>> where string is a datasetID.
Datasets() types.Map
// GetDataset returns a Dataset struct containing the current mapping of
// datasetID in the above Datasets Map.
GetDataset(datasetID string) Dataset
// Rebase brings this Database's view of the world inline with upstream.
Rebase()
// Commit updates the Commit that ds.ID() in this database points at. All
// Values that have been written to this Database are guaranteed to be
// persistent after Commit() returns.
// The new Commit struct is constructed using v, opts.Parents, and
// opts.Meta. If opts.Parents is the zero value (types.Set{}) then
// the current head is used. If opts.Meta is the zero value
// (types.Struct{}) then a fully initialized empty Struct is passed to
// NewCommit.
// The returned Dataset is always the newest snapshot, regardless of
// success or failure, and Datasets() is updated to match backing storage
// upon return as well. If the update cannot be performed, e.g., because
// of a conflict, Commit returns an 'ErrMergeNeeded' error.
Commit(ds Dataset, v types.Value, opts CommitOptions) (Dataset, error)
// CommitValue updates the Commit that ds.ID() in this database points at.
// All Values that have been written to this Database are guaranteed to be
// persistent after Commit().
// The new Commit struct is constructed using `v`, and the current Head of
// `ds` as the lone Parent.
// The returned Dataset is always the newest snapshot, regardless of
// success or failure, and Datasets() is updated to match backing storage
// upon return as well. If the update cannot be performed, e.g., because
// of a conflict, Commit returns an 'ErrMergeNeeded' error.
CommitValue(ds Dataset, v types.Value) (Dataset, error)
// Delete removes the Dataset named ds.ID() from the map at the root of
// the Database. The Dataset data is not necessarily cleaned up at this
// time, but may be garbage collected in the future.
// The returned Dataset is always the newest snapshot, regardless of
// success or failure, and Datasets() is updated to match backing storage
// upon return as well. If the update cannot be performed, e.g., because
// of a conflict, Delete returns an 'ErrMergeNeeded' error.
Delete(ds Dataset) (Dataset, error)
// SetHead ignores any lineage constraints (e.g. the current Head being in
// commit’s Parent set) and force-sets a mapping from datasetID: commit in
// this database.
// All Values that have been written to this Database are guaranteed to be
// persistent after SetHead(). If the update cannot be performed, e.g.,
// because another process moved the current Head out from under you,
// error will be non-nil.
// The newest snapshot of the Dataset is always returned, so the caller an
// easily retry using the latest.
// Regardless, Datasets() is updated to match backing storage upon return.
SetHead(ds Dataset, newHeadRef types.Ref) (Dataset, error)
// FastForward takes a types.Ref to a Commit object and makes it the new
// Head of ds iff it is a descendant of the current Head. Intended to be
// used e.g. after a call to Pull(). If the update cannot be performed,
// e.g., because another process moved the current Head out from under
// you, err will be non-nil.
// The newest snapshot of the Dataset is always returned, so the caller
// can easily retry using the latest.
// Regardless, Datasets() is updated to match backing storage upon return.
FastForward(ds Dataset, newHeadRef types.Ref) (Dataset, error)
// Stats may return some kind of struct that reports statistics about the
// ChunkStore that backs this Database instance. The type is
// implementation-dependent, and impls may return nil
Stats() interface{}
// StatsSummary may return a string containing summarized statistics for
// the ChunkStore that backs this Database. It must return "Unsupported"
// if this operation is not supported.
StatsSummary() string
Flush()
// chunkStore returns the ChunkStore used to read and write
// groups of values to the database efficiently. This interface is a low-
// level detail of the database that should infrequently be needed by
// clients.
chunkStore() chunks.ChunkStore
}
func NewDatabase(cs chunks.ChunkStore) Database {
return newDatabase(cs)
}
================================================
FILE: go/datas/database_common.go
================================================
// Copyright 2016 Attic Labs, Inc. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
package datas
import (
"errors"
"fmt"
"github.com/attic-labs/noms/go/chunks"
"github.com/attic-labs/noms/go/d"
"github.com/attic-labs/noms/go/hash"
"github.com/attic-labs/noms/go/merge"
"github.com/attic-labs/noms/go/types"
"github.com/attic-labs/noms/go/util/random"
)
type database struct {
*types.ValueStore
rt rootTracker
}
var (
ErrOptimisticLockFailed = errors.New("Optimistic lock failed on database Root update")
ErrMergeNeeded = errors.New("Dataset head is not ancestor of commit")
)
// rootTracker is a narrowing of the ChunkStore interface, to keep Database disciplined about working directly with Chunks
type rootTracker interface {
Rebase()
Root() hash.Hash
Commit(current, last hash.Hash) bool
}
func newDatabase(cs chunks.ChunkStore) *database {
vs := types.NewValueStore(cs)
if _, ok := cs.(*httpChunkStore); ok {
vs.SetEnforceCompleteness(false)
}
return &database{
ValueStore: vs, // ValueStore is responsible for closing |cs|
rt: vs,
}
}
func (db *database) chunkStore() chunks.ChunkStore {
return db.ChunkStore()
}
func (db *database) Stats() interface{} {
return db.ChunkStore().Stats()
}
func (db *database) StatsSummary() string {
return db.ChunkStore().StatsSummary()
}
func (db *database) Flush() {
// TODO: This is a pretty ghetto hack - do better.
// See: https://github.com/attic-labs/noms/issues/3530
ds := db.GetDataset(fmt.Sprintf("-/flush/%s", random.Id()))
r := db.WriteValue(types.Bool(true))
ds, err := db.CommitValue(ds, r)
d.PanicIfError(err)
_, err = db.Delete(ds)
d.PanicIfError(err)
}
func (db *database) Datasets() types.Map {
rootHash := db.rt.Root()
if rootHash.IsEmpty() {
return types.NewMap(db)
}
return db.ReadValue(rootHash).(types.Map)
}
func (db *database) GetDataset(datasetID string) Dataset {
if !DatasetFullRe.MatchString(datasetID) {
d.Panic("Invalid dataset ID: %s", datasetID)
}
var head types.Value
if r, ok := db.Datasets().MaybeGet(types.String(datasetID)); ok {
head = r.(types.Ref).TargetValue(db)
}
return newDataset(db, datasetID, head)
}
func (db *database) Rebase() {
db.rt.Rebase()
}
func (db *database) Close() error {
return db.ValueStore.Close()
}
func (db *database) SetHead(ds Dataset, newHeadRef types.Ref) (Dataset, error) {
return db.doHeadUpdate(ds, func(ds Dataset) error { return db.doSetHead(ds, newHeadRef) })
}
func (db *database) doSetHead(ds Dataset, newHeadRef types.Ref) error {
if currentHeadRef, ok := ds.MaybeHeadRef(); ok && newHeadRef.Equals(currentHeadRef) {
return nil
}
commit := db.validateRefAsCommit(newHeadRef)
currentRootHash, currentDatasets := db.rt.Root(), db.Datasets()
commitRef := db.WriteValue(commit) // will be orphaned if the tryCommitChunks() below fails
currentDatasets = currentDatasets.Edit().Set(types.String(ds.ID()), types.ToRefOfValue(commitRef)).Map()
return db.tryCommitChunks(currentDatasets, currentRootHash)
}
func (db *database) FastForward(ds Dataset, newHeadRef types.Ref) (Dataset, error) {
return db.doHeadUpdate(ds, func(ds Dataset) error { return db.doFastForward(ds, newHeadRef) })
}
func (db *database) doFastForward(ds Dataset, newHeadRef types.Ref) error {
currentHeadRef, ok := ds.MaybeHeadRef()
if ok && newHeadRef.Equals(currentHeadRef) {
return nil
}
if ok && newHeadRef.Height() <= currentHeadRef.Height() {
return ErrMergeNeeded
}
commit := db.validateRefAsCommit(newHeadRef)
return db.doCommit(ds.ID(), commit, nil)
}
func (db *database) Commit(ds Dataset, v types.Value, opts CommitOptions) (Dataset, error) {
return db.doHeadUpdate(
ds,
func(ds Dataset) error { return db.doCommit(ds.ID(), buildNewCommit(ds, v, opts), opts.Policy) },
)
}
func (db *database) CommitValue(ds Dataset, v types.Value) (Dataset, error) {
return db.Commit(ds, v, CommitOptions{})
}
// doCommit manages concurrent access the single logical piece of mutable state: the current Root. doCommit is optimistic in that it is attempting to update head making the assumption that currentRootHash is the hash of the current head. The call to Commit below will return an 'ErrOptimisticLockFailed' error if that assumption fails (e.g. because of a race with another writer) and the entire algorithm must be tried again. This method will also fail and return an 'ErrMergeNeeded' error if the |commit| is not a descendent of the current dataset head
func (db *database) doCommit(datasetID string, commit types.Struct, mergePolicy merge.Policy) error {
if !IsCommit(commit) {
d.Panic("Can't commit a non-Commit struct to dataset %s", datasetID)
}
// This could loop forever, given enough simultaneous committers. BUG 2565
var err error
for err = ErrOptimisticLockFailed; err == ErrOptimisticLockFailed; {
currentRootHash, currentDatasets := db.rt.Root(), db.Datasets()
commitRef := db.WriteValue(commit) // will be orphaned if the tryCommitChunks() below fails
// If there's nothing in the DB yet, skip all this logic.
if !currentRootHash.IsEmpty() {
r, hasHead := currentDatasets.MaybeGet(types.String(datasetID))
// First commit in dataset is always fast-forward, so go through all this iff there's already a Head for datasetID.
if hasHead {
head := r.(types.Ref).TargetValue(db)
currentHeadRef := types.NewRef(head)
ancestorRef, found := FindCommonAncestor(commitRef, currentHeadRef, db)
if !found {
return ErrMergeNeeded
}
// This covers all cases where currentHeadRef is not an ancestor of commit, including the following edge cases:
// - commit is a duplicate of currentHead.
// - we hit an ErrOptimisticLockFailed and looped back around because some other process changed the Head out from under us.
if currentHeadRef.TargetHash() != ancestorRef.TargetHash() || currentHeadRef.TargetHash() == commitRef.TargetHash() {
if mergePolicy == nil {
return ErrMergeNeeded
}
ancestor, currentHead := db.validateRefAsCommit(ancestorRef), db.validateRefAsCommit(currentHeadRef)
merged, err := mergePolicy(commit.Get(ValueField), currentHead.Get(ValueField), ancestor.Get(ValueField), db, nil)
if err != nil {
return err
}
commitRef = db.WriteValue(NewCommit(merged, types.NewSet(db, commitRef, currentHeadRef), types.EmptyStruct))
}
}
}
currentDatasets = currentDatasets.Edit().Set(types.String(datasetID), types.ToRefOfValue(commitRef)).Map()
err = db.tryCommitChunks(currentDatasets, currentRootHash)
}
return err
}
func (db *database) Delete(ds Dataset) (Dataset, error) {
return db.doHeadUpdate(ds, func(ds Dataset) error { return db.doDelete(ds.ID()) })
}
// doDelete manages concurrent access the single logical piece of mutable state: the current Root. doDelete is optimistic in that it is attempting to update head making the assumption that currentRootHash is the hash of the current head. The call to Commit below will return an 'ErrOptimisticLockFailed' error if that assumption fails (e.g. because of a race with another writer) and the entire algorithm must be tried again.
func (db *database) doDelete(datasetIDstr string) error {
datasetID := types.String(datasetIDstr)
currentRootHash, currentDatasets := db.rt.Root(), db.Datasets()
var initialHead types.Ref
if r, hasHead := currentDatasets.MaybeGet(datasetID); !hasHead {
return nil
} else {
initialHead = r.(types.Ref)
}
var err error
for {
currentDatasets = currentDatasets.Edit().Remove(datasetID).Map()
err = db.tryCommitChunks(currentDatasets, currentRootHash)
if err != ErrOptimisticLockFailed {
break
}
// If the optimistic lock failed because someone changed the Head of datasetID, then return ErrMergeNeeded. If it failed because someone changed a different Dataset, we should try again.
currentRootHash, currentDatasets = db.rt.Root(), db.Datasets()
if r, hasHead := currentDatasets.MaybeGet(datasetID); !hasHead || (hasHead && !initialHead.Equals(r)) {
err = ErrMergeNeeded
break
}
}
return err
}
func (db *database) tryCommitChunks(currentDatasets types.Map, currentRootHash hash.Hash) (err error) {
newRootHash := db.WriteValue(currentDatasets).TargetHash()
if !db.rt.Commit(newRootHash, currentRootHash) {
err = ErrOptimisticLockFailed
}
return
}
func (db *database) validateRefAsCommit(r types.Ref) types.Struct {
v := db.ReadValue(r.TargetHash())
if v == nil {
panic(r.TargetHash().String() + " not found")
}
if !IsCommit(v) {
panic("Not a commit: " + types.EncodedValueMaxLines(v, 10) + " ...\n")
}
return v.(types.Struct)
}
func buildNewCommit(ds Dataset, v types.Value, opts CommitOptions) types.Struct {
parents := opts.Parents
if (parents == types.Set{}) {
parents = types.NewSet(ds.Database())
if headRef, ok := ds.MaybeHeadRef(); ok {
parents = parents.Edit().Insert(headRef).Set()
}
}
meta := opts.Meta
if meta.IsZeroValue() {
meta = types.EmptyStruct
}
return NewCommit(v, parents, meta)
}
func (db *database) doHeadUpdate(ds Dataset, updateFunc func(ds Dataset) error) (Dataset, error) {
err := updateFunc(ds)
return db.GetDataset(ds.ID()), err
}
================================================
FILE: go/datas/database_server.go
================================================
// Copyright 2016 Attic Labs, Inc. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
package datas
import (
"fmt"
"log"
"net"
"net/http"
"strconv"
"github.com/attic-labs/noms/go/chunks"
"github.com/attic-labs/noms/go/constants"
"github.com/attic-labs/noms/go/d"
"github.com/julienschmidt/httprouter"
)
type connectionState struct {
c net.Conn
cs http.ConnState
}
type RemoteDatabaseServer struct {
cs chunks.ChunkStore
address string
port int
l *net.Listener
csChan chan *connectionState
closing bool
// Called just before the server is started.
Ready func()
}
func NewRemoteDatabaseServer(cs chunks.ChunkStore, address string, port int) *RemoteDatabaseServer {
dataVersion := cs.Version()
if constants.NomsVersion != dataVersion {
d.Panic("SDK version %s is incompatible with data of version %s", constants.NomsVersion, dataVersion)
}
return &RemoteDatabaseServer{
cs, address, port, nil, make(chan *connectionState, 16), false, func() {},
}
}
// Port is the actual port used. This may be different than the port passed in to NewRemoteDatabaseServer.
func (s *RemoteDatabaseServer) Port() int {
return s.port
}
func Router(cs chunks.ChunkStore, prefix string) *httprouter.Router {
router := httprouter.New()
router.POST(prefix+constants.GetRefsPath, corsHandle(makeHandle(HandleGetRefs, cs)))
router.GET(prefix+constants.GetBlobPath, corsHandle(makeHandle(HandleGetBlob, cs)))
router.OPTIONS(prefix+constants.GetRefsPath, corsHandle(noopHandle))
router.POST(prefix+constants.HasRefsPath, corsHandle(makeHandle(HandleHasRefs, cs)))
router.OPTIONS(prefix+constants.HasRefsPath, corsHandle(noopHandle))
router.GET(prefix+constants.RootPath, corsHandle(makeHandle(HandleRootGet, cs)))
router.POST(prefix+constants.RootPath, corsHandle(makeHandle(HandleRootPost, cs)))
router.OPTIONS(prefix+constants.RootPath, corsHandle(noopHandle))
router.POST(prefix+constants.WriteValuePath, corsHandle(makeHandle(HandleWriteValue, cs)))
router.OPTIONS(prefix+constants.WriteValuePath, corsHandle(noopHandle))
router.GET(prefix+constants.BasePath, corsHandle(makeHandle(HandleBaseGet, cs)))
router.GET(prefix+constants.GraphQLPath, corsHandle(makeHandle(HandleGraphQL, cs)))
router.POST(prefix+constants.GraphQLPath, corsHandle(makeHandle(HandleGraphQL, cs)))
router.OPTIONS(prefix+constants.GraphQLPath, corsHandle(noopHandle))
router.GET(prefix+constants.StatsPath, corsHandle(makeHandle(HandleStats, cs)))
router.OPTIONS(prefix+constants.StatsPath, corsHandle(noopHandle))
return router
}
// Run blocks while the RemoteDatabaseServer is listening. Running on a separate go routine is supported.
func (s *RemoteDatabaseServer) Run() {
l, err := net.Listen("tcp", fmt.Sprintf("%s:%d", s.address, s.port))
d.Chk.NoError(err)
s.l = &l
_, port, err := net.SplitHostPort(l.Addr().String())
d.Chk.NoError(err)
s.port, err = strconv.Atoi(port)
d.Chk.NoError(err)
log.Printf("Listening on %s:%d...\n", s.address, s.port)
router := Router(s.cs, "")
srv := &http.Server{
Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
router.ServeHTTP(w, req)
}),
ConnState: s.connState,
}
go func() {
m := map[net.Conn]http.ConnState{}
for connState := range s.csChan {
switch connState.cs {
case http.StateNew, http.StateActive, http.StateIdle:
m[connState.c] = connState.cs
default:
delete(m, connState.c)
}
}
for c := range m {
c.Close()
}
}()
go s.Ready()
srv.Serve(l)
}
func makeHandle(hndlr Handler, cs chunks.ChunkStore) httprouter.Handle {
return func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
hndlr(w, req, ps, cs)
}
}
func noopHandle(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
}
func corsHandle(f httprouter.Handle) httprouter.Handle {
// TODO: Implement full pre-flighting?
// See: http://www.html5rocks.com/static/images/cors_server_flowchart.png
return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
// Can't use * when clients are using cookies.
w.Header().Add("Access-Control-Allow-Origin", r.Header.Get("Origin"))
w.Header().Add("Access-Control-Allow-Methods", "GET, POST")
w.Header().Add("Access-Control-Allow-Headers", "*")
w.Header().Add("Access-Control-Expose-Headers", NomsVersionHeader)
w.Header().Add(NomsVersionHeader, constants.NomsVersion)
f(w, r, ps)
}
}
func (s *RemoteDatabaseServer) connState(c net.Conn, cs http.ConnState) {
if s.closing {
d.PanicIfFalse(cs == http.StateClosed)
return
}
s.csChan <- &connectionState{c, cs}
}
// Will cause the RemoteDatabaseServer to stop listening and an existing call to Run() to continue.
func (s *RemoteDatabaseServer) Stop() {
s.closing = true
(*s.l).Close()
(s.cs).Close()
close(s.csChan)
}
================================================
FILE: go/datas/database_test.go
================================================
// Copyright 2016 Attic Labs, Inc. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
package datas
import (
"testing"
"github.com/attic-labs/noms/go/chunks"
"github.com/attic-labs/noms/go/hash"
"github.com/attic-labs/noms/go/merge"
"github.com/attic-labs/noms/go/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
)
func TestLocalDatabase(t *testing.T) {
suite.Run(t, &LocalDatabaseSuite{})
}
func TestRemoteDatabase(t *testing.T) {
suite.Run(t, &RemoteDatabaseSuite{})
}
func TestValidateRef(t *testing.T) {
st := &chunks.TestStorage{}
db := NewDatabase(st.NewView()).(*database)
defer db.Close()
b := types.Bool(true)
r := db.WriteValue(b)
assert.Panics(t, func() { db.validateRefAsCommit(r) })
assert.Panics(t, func() { db.validateRefAsCommit(types.NewRef(b)) })
}
type DatabaseSuite struct {
suite.Suite
storage *chunks.TestStorage
db Database
makeDb func(chunks.ChunkStore) Database
}
type LocalDatabaseSuite struct {
DatabaseSuite
}
func (suite *LocalDatabaseSuite) SetupTest() {
suite.storage = &chunks.TestStorage{}
suite.makeDb = NewDatabase
suite.db = suite.makeDb(suite.storage.NewView())
}
type RemoteDatabaseSuite struct {
DatabaseSuite
}
func (suite *RemoteDatabaseSuite) SetupTest() {
suite.storage = &chunks.TestStorage{}
suite.makeDb = func(cs chunks.ChunkStore) Database {
return NewDatabase(newHTTPChunkStoreForTest(cs))
}
suite.db = suite.makeDb(suite.storage.NewView())
}
func (suite *DatabaseSuite) TearDownTest() {
suite.db.Close()
}
func (suite *RemoteDatabaseSuite) TestWriteRefToNonexistentValue() {
ds := suite.db.GetDataset("foo")
r := types.NewRef(types.Bool(true))
suite.Panics(func() { suite.db.CommitValue(ds, r) })
}
func (suite *DatabaseSuite) TestTolerateUngettableRefs() {
suite.Nil(suite.db.ReadValue(hash.Hash{}))
}
func (suite *DatabaseSuite) TestCompletenessCheck() {
datasetID := "ds1"
ds1 := suite.db.GetDataset(datasetID)
se := types.NewSet(suite.db).Edit()
for i := 0; i < 100; i++ {
se.Insert(suite.db.WriteValue(types.Number(100)))
}
s := se.Set()
ds1, err := suite.db.CommitValue(ds1, s)
suite.NoError(err)
s = ds1.HeadValue().(types.Set)
s = s.Edit().Insert(types.NewRef(types.Number(1000))).Set() // danging ref
suite.Panics(func() {
ds1, err = suite.db.CommitValue(ds1, s)
})
}
func (suite *DatabaseSuite) TestRebase() {
datasetID := "ds1"
ds1 := suite.db.GetDataset(datasetID)
var err error
// Setup:
// ds1: |a| <- |b|
ds1, err = suite.db.CommitValue(ds1, types.String("a"))
b := types.String("b")
ds1, err = suite.db.CommitValue(ds1, b)
suite.NoError(err)
suite.True(ds1.HeadValue().Equals(b))
interloper := suite.makeDb(suite.storage.NewView())
defer interloper.Close()
// Concurrent change, to move root out from under my feet:
// ds1: |a| <- |b| <- |e|
e := types.String("e")
iDS, concErr := interloper.CommitValue(interloper.GetDataset(datasetID), e)
suite.NoError(concErr)
suite.True(iDS.HeadValue().Equals(e))
// suite.ds shouldn't see the above change yet
suite.True(suite.db.GetDataset(datasetID).HeadValue().Equals(b))
suite.db.Rebase()
suite.True(suite.db.GetDataset(datasetID).HeadValue().Equals(e))
cs := suite.storage.NewView()
noChangeDB := suite.makeDb(cs)
noChangeDB.Datasets()
cs.Reads = 0 // New baseline
noChangeDB.Rebase()
suite.Zero(cs.Reads)
}
func (suite *DatabaseSuite) TestCommitProperlyTracksRoot() {
id1, id2 := "testdataset", "othertestdataset"
db1 := suite.makeDb(suite.storage.NewView())
defer db1.Close()
ds1 := db1.GetDataset(id1)
ds1HeadVal := types.String("Commit value for " + id1)
ds1, err := db1.CommitValue(ds1, ds1HeadVal)
suite.NoError(err)
db2 := suite.makeDb(suite.storage.NewView())
defer db2.Close()
ds2 := db2.GetDataset(id2)
ds2HeadVal := types.String("Commit value for " + id2)
ds2, err = db2.CommitValue(ds2, ds2HeadVal)
suite.NoError(err)
suite.EqualValues(ds1HeadVal, ds1.HeadValue())
suite.EqualValues(ds2HeadVal, ds2.HeadValue())
suite.False(ds2.HeadValue().Equals(ds1HeadVal))
suite.False(ds1.HeadValue().Equals(ds2HeadVal))
}
func (suite *DatabaseSuite) TestDatabaseCommit() {
datasetID := "ds1"
datasets := suite.db.Datasets()
suite.Zero(datasets.Len())
// |a|
ds := suite.db.GetDataset(datasetID)
a := types.String("a")
ds2, err := suite.db.CommitValue(ds, a)
suite.NoError(err)
// ds2 matches the Datasets Map in suite.db
suite.True(ds2.HeadRef().Equals(suite.db.GetDataset(datasetID).HeadRef()))
// ds2 has |a| at its head
h, ok := ds2.MaybeHeadValue()
suite.True(ok)
suite.True(h.Equals(a))
suite.Equal(uint64(1), ds2.HeadRef().Height())
ds = ds2
aCommitRef := ds.HeadRef() // to be used to test disallowing of non-fastforward commits below
// |a| <- |b|
b := types.String("b")
ds, err = suite.db.CommitValue(ds, b)
suite.NoError(err)
suite.True(ds.HeadValue().Equals(b))
suite.Equal(uint64(2), ds.HeadRef().Height())
// |a| <- |b|
// \----|c|
// Should be disallowed.
c := types.String("c")
ds, err = suite.db.Commit(ds, c, newOpts(suite.db, aCommitRef))
suite.Error(err)
suite.True(ds.HeadValue().Equals(b))
// |a| <- |b| <- |d|
d := types.String("d")
ds, err = suite.db.CommitValue(ds, d)
suite.NoError(err)
suite.True(ds.HeadValue().Equals(d))
suite.Equal(uint64(3), ds.HeadRef().Height())
// Attempt to recommit |b| with |a| as parent.
// Should be disallowed.
ds, err = suite.db.Commit(ds, b, newOpts(suite.db, aCommitRef))
suite.Error(err)
suite.True(ds.HeadValue().Equals(d))
// Add a commit to a different datasetId
_, err = suite.db.CommitValue(suite.db.GetDataset("otherDS"), a)
suite.NoError(err)
// Get a fresh database, and verify that both datasets are present
newDB := suite.makeDb(suite.storage.NewView())
defer newDB.Close()
datasets2 := newDB.Datasets()
suite.Equal(uint64(2), datasets2.Len())
}
func (suite *DatabaseSuite) TestDatasetsMapType() {
dsID1, dsID2 := "ds1", "ds2"
datasets := suite.db.Datasets()
ds, err := suite.db.CommitValue(suite.db.GetDataset(dsID1), types.String("a"))
suite.NoError(err)
suite.NotPanics(func() { assertMapOfStringToRefOfCommit(suite.db.Datasets(), datasets, suite.db) })
datasets = suite.db.Datasets()
_, err = suite.db.CommitValue(suite.db.GetDataset(dsID2), types.Number(42))
suite.NoError(err)
suite.NotPanics(func() { assertMapOfStringToRefOfCommit(suite.db.Datasets(), datasets, suite.db) })
datasets = suite.db.Datasets()
_, err = suite.db.Delete(ds)
suite.NoError(err)
suite.NotPanics(func() { assertMapOfStringToRefOfCommit(suite.db.Datasets(), datasets, suite.db) })
}
func newOpts(vrw types.ValueReadWriter, parents ...types.Value) CommitOptions {
return CommitOptions{Parents: types.NewSet(vrw, parents...)}
}
func (suite *DatabaseSuite) TestDatabaseDuplicateCommit() {
datasetID := "ds1"
ds := suite.db.GetDataset(datasetID)
datasets := suite.db.Datasets()
suite.Zero(datasets.Len())
v := types.String("Hello")
_, err := suite.db.CommitValue(ds, v)
suite.NoError(err)
_, err = suite.db.CommitValue(ds, v)
suite.IsType(ErrMergeNeeded, err)
}
func (suite *DatabaseSuite) TestDatabaseCommitMerge() {
datasetID1, datasetID2 := "ds1", "ds2"
ds1, ds2 := suite.db.GetDataset(datasetID1), suite.db.GetDataset(datasetID2)
var err error
v := types.NewMap(suite.db, types.String("Hello"), types.Number(42))
ds1, err = suite.db.CommitValue(ds1, v)
ds1First := ds1
suite.NoError(err)
ds1, err = suite.db.CommitValue(ds1, v.Edit().Set(types.String("Friends"), types.Bool(true)).Map())
suite.NoError(err)
ds2, err = suite.db.CommitValue(ds2, types.String("Goodbye"))
suite.NoError(err)
// No common ancestor
_, err = suite.db.Commit(ds1, types.Number(47), newOpts(suite.db, ds2.HeadRef()))
suite.IsType(ErrMergeNeeded, err, "%s", err)
// Unmergeable
_, err = suite.db.Commit(ds1, types.Number(47), newOptsWithMerge(suite.db, merge.None, ds1First.HeadRef()))
suite.IsType(&merge.ErrMergeConflict{}, err, "%s", err)
// Merge policies
newV := v.Edit().Set(types.String("Friends"), types.Bool(false)).Map()
_, err = suite.db.Commit(ds1, newV, newOptsWithMerge(suite.db, merge.None, ds1First.HeadRef()))
suite.IsType(&merge.ErrMergeConflict{}, err, "%s", err)
theirs, err := suite.db.Commit(ds1, newV, newOptsWithMerge(suite.db, merge.Theirs, ds1First.HeadRef()))
suite.NoError(err)
suite.True(types.Bool(true).Equals(theirs.HeadValue().(types.Map).Get(types.String("Friends"))))
newV = v.Edit().Set(types.String("Friends"), types.Number(47)).Map()
ours, err := suite.db.Commit(ds1First, newV, newOptsWithMerge(suite.db, merge.Ours, ds1First.HeadRef()))
suite.NoError(err)
suite.True(types.Number(47).Equals(ours.HeadValue().(types.Map).Get(types.String("Friends"))))
}
func newOptsWithMerge(vrw types.ValueReadWriter, policy merge.ResolveFunc, parents ...types.Value) CommitOptions {
return CommitOptions{Parents: types.NewSet(vrw, parents...), Policy: merge.NewThreeWay(policy)}
}
func (suite *DatabaseSuite) TestDatabaseDelete() {
datasetID1, datasetID2 := "ds1", "ds2"
ds1, ds2 := suite.db.GetDataset(datasetID1), suite.db.GetDataset(datasetID2)
datasets := suite.db.Datasets()
suite.Zero(datasets.Len())
// ds1: |a|
var err error
a := types.String("a")
ds1, err = suite.db.CommitValue(ds1, a)
suite.NoError(err)
suite.True(ds1.HeadValue().Equals(a))
// ds1: |a|, ds2: |b|
b := types.String("b")
ds2, err = suite.db.CommitValue(ds2, b)
suite.NoError(err)
suite.True(ds2.HeadValue().Equals(b))
ds1, err = suite.db.Delete(ds1)
suite.NoError(err)
suite.True(suite.db.GetDataset(datasetID2).HeadValue().Equals(b))
_, present := suite.db.GetDataset(datasetID1).MaybeHead()
suite.False(present, "Dataset %s should not be present", datasetID1)
// Get a fresh database, and verify that only ds2 is present
newDB := suite.makeDb(suite.storage.NewView())
defer newDB.Close()
datasets = newDB.Datasets()
suite.Equal(uint64(1), datasets.Len())
_, present = newDB.GetDataset(datasetID2).MaybeHeadRef()
suite.True(present, "Dataset %s should be present", datasetID2)
}
type waitDuringUpdateRootChunkStore struct {
chunks.ChunkStore
preUpdateRootHook func()
}
func (w *waitDuringUpdateRootChunkStore) Commit(current, last hash.Hash) bool {
if w.preUpdateRootHook != nil {
w.preUpdateRootHook()
}
return w.ChunkStore.Commit(current, last)
}
func (suite *DatabaseSuite) TestCommitWithConcurrentChunkStoreUse() {
datasetID := "ds1"
ds1 := suite.db.GetDataset(datasetID)
var err error
// Setup:
// ds1: |a| <- |b|
ds1, err = suite.db.CommitValue(ds1, types.String("a"))
b := types.String("b")
ds1, err = suite.db.CommitValue(ds1, b)
suite.NoError(err)
suite.True(ds1.HeadValue().Equals(b))
// Craft DB that will allow me to move the backing ChunkStore while suite.db isn't looking
interloper := suite.makeDb(suite.storage.NewView())
defer interloper.Close()
// Change ds2 behind suite.db's back. This shouldn't block changes to ds1 via suite.db below.
// ds1: |a| <- |b|
// ds2: |stuff|
stf := types.String("stuff")
ds2, concErr := interloper.CommitValue(interloper.GetDataset("ds2"), stf)
suite.NoError(concErr)
suite.True(ds2.HeadValue().Equals(stf))
// Change ds1 via suite.db, which should proceed without a problem
c := types.String("c")
ds1, err = suite.db.CommitValue(ds1, c)
suite.NoError(err)
suite.True(ds1.HeadValue().Equals(c))
// Change ds1 behind suite.db's back. Will block changes to ds1 below.
// ds1: |a| <- |b| <- |c| <- |e|
e := types.String("e")
interloper.Rebase()
iDS, concErr := interloper.CommitValue(interloper.GetDataset("ds1"), e)
suite.NoError(concErr)
suite.True(iDS.HeadValue().Equals(e))
// Attempted Concurrent change, which should fail due to the above
nope := types.String("nope")
ds1, err = suite.db.CommitValue(ds1, nope)
suite.Error(err)
v := ds1.HeadValue()
suite.True(v.Equals(e), "%s", v.(types.String))
}
func (suite *DatabaseSuite) TestDeleteWithConcurrentChunkStoreUse() {
datasetID := "ds1"
ds1 := suite.db.GetDataset(datasetID)
var err error
// Setup:
// ds1: |a| <- |b|
ds1, err = suite.db.CommitValue(ds1, types.String("a"))
b := types.String("b")
ds1, err = suite.db.CommitValue(ds1, b)
suite.NoError(err)
suite.True(ds1.HeadValue().Equals(b))
// Craft DB that will allow me to move the backing ChunkStore while suite.db isn't looking
interloper := suite.makeDb(suite.storage.NewView())
defer interloper.Close()
// Concurrent change, to move root out from under my feet:
// ds1: |a| <- |b| <- |e|
e := types.String("e")
iDS, concErr := interloper.CommitValue(interloper.GetDataset(datasetID), e)
suite.NoError(concErr)
suite.True(iDS.HeadValue().Equals(e))
// Attempt to delete ds1 via suite.db, which should fail due to the above
ds1, err = suite.db.Delete(ds1)
suite.Error(err)
suite.True(ds1.HeadValue().Equals(e))
// Concurrent change, but to some other dataset. This shouldn't stop changes to ds1.
// ds1: |a| <- |b| <- |e|
// ds2: |stuff|
stf := types.String("stuff")
iDS, concErr = interloper.CommitValue(suite.db.GetDataset("other"), stf)
suite.NoError(concErr)
suite.True(iDS.HeadValue().Equals(stf))
// Attempted concurrent delete, which should proceed without a problem
ds1, err = suite.db.Delete(ds1)
suite.NoError(err)
_, present := ds1.MaybeHeadRef()
suite.False(present, "Dataset %s should not be present", datasetID)
}
func (suite *DatabaseSuite) TestSetHead() {
var err error
datasetID := "ds1"
// |a| <- |b|
ds := suite.db.GetDataset(datasetID)
a := types.String("a")
ds, err = suite.db.CommitValue(ds, a)
suite.NoError(err)
aCommitRef := ds.HeadRef() // To use in non-FF SetHead() below.
b := types.String("b")
ds, err = suite.db.CommitValue(ds, b)
suite.NoError(err)
suite.True(ds.HeadValue().Equals(b))
bCommitRef := ds.HeadRef() // To use in FF SetHead() below.
ds, err = suite.db.SetHead(ds, aCommitRef)
suite.NoError(err)
suite.True(ds.HeadValue().Equals(a))
ds, err = suite.db.SetHead(ds, bCommitRef)
suite.NoError(err)
suite.True(ds.HeadValue().Equals(b))
}
func (suite *DatabaseSuite) TestFastForward() {
var err error
datasetID := "ds1"
// |a| <- |b| <- |c|
ds := suite.db.GetDataset(datasetID)
a := types.String("a")
ds, err = suite.db.CommitValue(ds, a)
suite.NoError(err)
aCommitRef := ds.HeadRef() // To use in non-FF cases below.
b := types.String("b")
ds, err = suite.db.CommitValue(ds, b)
suite.NoError(err)
suite.True(ds.HeadValue().Equals(b))
c := types.String("c")
ds, err = suite.db.CommitValue(ds, c)
suite.NoError(err)
suite.True(ds.HeadValue().Equals(c))
cCommitRef := ds.HeadRef() // To use in FastForward() below.
// FastForward should disallow this, as |a| is not a descendant of |c|
ds, err = suite.db.FastForward(ds, aCommitRef)
suite.Error(err)
suite.True(ds.HeadValue().Equals(c))
// Move Head back to something earlier in the lineage, so we can test FastForward
ds, err = suite.db.SetHead(ds, aCommitRef)
suite.NoError(err)
suite.True(ds.HeadValue().Equals(a))
// This should succeed, because while |a| is not a direct parent of |c|, it is an ancestor.
ds, err = suite.db.FastForward(ds, cCommitRef)
suite.NoError(err)
suite.True(ds.HeadValue().Equals(c))
}
func (suite *DatabaseSuite) TestDatabaseHeightOfRefs() {
r1 := suite.db.WriteValue(types.String("hello"))
suite.Equal(uint64(1), r1.Height())
r2 := suite.db.WriteValue(r1)
suite.Equal(uint64(2), r2.Height())
suite.Equal(uint64(3), suite.db.WriteValue(r2).Height())
}
func (suite *DatabaseSuite) TestDatabaseHeightOfCollections() {
setOfStringType := types.MakeSetType(types.StringType)
setOfRefOfStringType := types.MakeSetType(types.MakeRefType(types.StringType))
// Set<String>
v1 := types.String("hello")
v2 := types.String("world")
s1 := types.NewSet(suite.db, v1, v2)
suite.Equal(uint64(1), suite.db.WriteValue(s1).Height())
// Set<Ref<String>>
s2 := types.NewSet(suite.db, suite.db.WriteValue(v1), suite.db.WriteValue(v2))
suite.Equal(uint64(2), suite.db.WriteValue(s2).Height())
// List<Set<String>>
v3 := types.String("foo")
v4 := types.String("bar")
s3 := types.NewSet(suite.db, v3, v4)
l1 := types.NewList(suite.db, s1, s3)
suite.Equal(uint64(1), suite.db.WriteValue(l1).Height())
// List<Ref<Set<String>>
l2 := types.NewList(suite.db, suite.db.WriteValue(s1), suite.db.WriteValue(s3))
suite.Equal(uint64(2), suite.db.WriteValue(l2).Height())
// List<Ref<Set<Ref<String>>>
s4 := types.NewSet(suite.db, suite.db.WriteValue(v3), suite.db.WriteValue(v4))
l3 := types.NewList(suite.db, suite.db.WriteValue(s4))
suite.Equal(uint64(3), suite.db.WriteValue(l3).Height())
// List<Set<String> | RefValue<Set<String>>>
l4 := types.NewList(suite.db, s1, suite.db.WriteValue(s3))
suite.Equal(uint64(2), suite.db.WriteValue(l4).Height())
l5 := types.NewList(suite.db, suite.db.WriteValue(s1), s3)
suite.Equal(uint64(2), suite.db.WriteValue(l5).Height())
// Familiar with the "New Jersey Turnpike" drink? Here's the noms version of that...
everything := []types.Value{v1, v2, s1, s2, v3, v4, s3, l1, l2, s4, l3, l4, l5}
andMore := make([]types.Value, 0, len(everything)*3+2)
for _, v := range everything {
andMore = append(andMore, v, types.TypeOf(v), suite.db.WriteValue(v))
}
andMore = append(andMore, setOfStringType, setOfRefOfStringType)
suite.db.WriteValue(types.NewList(suite.db, andMore...))
}
func (suite *DatabaseSuite) TestMetaOption() {
ds := suite.db.GetDataset("ds1")
m := types.NewStruct("M", types.StructData{
"author": types.String("arv"),
})
ds, err := suite.db.Commit(ds, types.String("a"), CommitOptions{Meta: m})
suite.NoError(err)
c := ds.Head()
suite.Equal(types.String("arv"), c.Get("meta").(types.Struct).Get("author"))
}
================================================
FILE: go/datas/dataset.go
================================================
// Copyright 2016 Attic Labs, Inc. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
package datas
import (
"regexp"
"github.com/attic-labs/noms/go/d"
"github.com/attic-labs/noms/go/types"
)
// DatasetRe is a regexp that matches a legal Dataset name anywhere within the
// target string.
var DatasetRe = regexp.MustCompile(`[a-zA-Z0-9\-_/]+`)
// DatasetFullRe is a regexp that matches a only a target string that is
// entirely legal Dataset name.
var DatasetFullRe = regexp.MustCompile("^" + DatasetRe.String() + "$")
// Dataset is a named Commit within a Database.
type Dataset struct {
db Database
id string
head types.Value
}
func newDataset(db Database, id string, head types.Value) Dataset {
d.PanicIfFalse(head == nil || IsCommit(head))
return Dataset{db, id, head}
}
// Database returns the Database object in which this Dataset is stored.
// WARNING: This method is under consideration for deprecation.
func (ds Dataset) Database() Database {
return ds.db
}
// ID returns the name of this Dataset.
func (ds Dataset) ID() string {
return ds.id
}
// MaybeHead returns the current Head Commit of this Dataset, which contains
// the current root of the Dataset's value tree, if available. If not, it
// returns a new Commit and 'false'.
func (ds Dataset) MaybeHead() (types.Struct, bool) {
if ds.head == nil {
return types.Struct{}, false
}
return ds.head.(types.Struct), true
}
// Head returns the current head Commit, which contains the current root of
// the Dataset's value tree.
func (ds Dataset) Head() types.Struct {
c, ok := ds.MaybeHead()
if !ok {
d.Panic("Dataset \"%s\" does not exist", ds.id)
}
return c
}
// MaybeHeadRef returns the Ref of the current Head Commit of this Dataset,
// which contains the current root of the Dataset's value tree, if available.
// If not, it returns an empty Ref and 'false'.
func (ds Dataset) MaybeHeadRef() (types.Ref, bool) {
if ds.head == nil {
return types.Ref{}, false
}
return types.NewRef(ds.head), true
}
// HasHead() returns 'true' if this dataset has a Head Commit, false otherwise.
func (ds Dataset) HasHead() bool {
return ds.head != nil
}
// HeadRef returns the Ref of the current head Commit, which contains the
// current root of the Dataset's value tree.
func (ds Dataset) HeadRef() types.Ref {
r, ok := ds.MaybeHeadRef()
if !ok {
d.Panic("Dataset \"%s\" does not exist", ds.id)
}
return r
}
// MaybeHeadValue returns the Value field of the current head Commit, if
// available. If not it returns nil and 'false'.
func (ds Dataset) MaybeHeadValue() (types.Value, bool) {
if c, ok := ds.MaybeHead(); ok {
return c.Get(ValueField), true
}
return nil, false
}
// HeadValue returns the Value field of the current head Commit.
func (ds Dataset) HeadValue() types.Value {
c := ds.Head()
return c.Get(ValueField)
}
func IsValidDatasetName(name string) bool {
return DatasetFullRe.MatchString(name)
}
================================================
FILE: go/datas/dataset_test.go
================================================
// Copyright 2016 Attic Labs, Inc. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
package datas
import (
"testing"
"github.com/attic-labs/noms/go/chunks"
"github.com/attic-labs/noms/go/types"
"github.com/stretchr/testify/assert"
)
func TestExplicitBranchUsingDatasets(t *testing.T) {
assert := assert.New(t)
id1 := "testdataset"
id2 := "othertestdataset"
stg := &chunks.MemoryStorage{}
store := NewDatabase(stg.NewView())
defer store.Close()
ds1 := store.GetDataset(id1)
// ds1: |a|
a := types.String("a")
ds1, err := store.CommitValue(ds1, a)
assert.NoError(err)
assert.True(ds1.Head().Get(ValueField).Equals(a))
// ds1: |a|
// \ds2
ds2 := store.GetDataset(id2)
ds2, err = store.Commit(ds2, ds1.HeadValue(), CommitOptions{Parents: types.NewSet(store, ds1.HeadRef())})
assert.NoError(err)
assert.True(ds2.Head().Get(ValueField).Equals(a))
// ds1: |a| <- |b|
b := types.String("b")
ds1, err = store.CommitValue(ds1, b)
assert.NoError(err)
assert.True(ds1.Head().Get(ValueField).Equals(b))
// ds1: |a| <- |b|
// \ds2 <- |c|
c := types.String("c")
ds2, err = store.CommitValue(ds2, c)
assert.NoError(err)
assert.True(ds2.Head().Get(ValueField).Equals(c))
// ds1: |a| <- |b| <--|d|
// \ds2 <- |c| <--/
mergeParents := types.NewSet(store, types.NewRef(ds1.Head()), types.NewRef(ds2.Head()))
d := types.String("d")
ds2, err = store.Commit(ds2, d, CommitOptions{Parents: mergeParents})
assert.NoError(err)
assert.True(ds2.Head().Get(ValueField).Equals(d))
ds1, err = store.Commit(ds1, d, CommitOptions{Parents: mergeParents})
assert.NoError(err)
assert.True(ds1.Head().Get(ValueField).Equals(d))
}
func TestTwoClientsWithEmptyDataset(t *testing.T) {
assert := assert.New(t)
id1 := "testdataset"
stg := &chunks.MemoryStorage{}
store := NewDatabase(stg.NewView())
defer store.Close()
dsx := store.GetDataset(id1)
dsy := store.GetDataset(id1)
// dsx: || -> |a|
a := types.String("a")
dsx, err := store.CommitValue(dsx, a)
assert.NoError(err)
assert.True(dsx.Head().Get(ValueField).Equals(a))
// dsy: || -> |b|
_, ok := dsy.MaybeHead()
assert.False(ok)
b := types.String("b")
dsy, err = store.CommitValue(dsy, b)
assert.Error(err)
// Commit failed, but dsy now has latest head, so we should be able to just try again.
// dsy: |a| -> |b|
dsy, err = store.CommitValue(dsy, b)
assert.NoError(err)
assert.True(dsy.Head().Get(ValueField).Equals(b))
}
func TestTwoClientsWithNonEmptyDataset(t *testing.T) {
assert := assert.New(t)
id1 := "testdataset"
stg := &chunks.MemoryStorage{}
store := NewDatabase(stg.NewView())
defer store.Close()
a := types.String("a")
{
// ds1: || -> |a|
ds1 := store.GetDataset(id1)
ds1, err := store.CommitValue(ds1, a)
assert.NoError(err)
assert.True(ds1.Head().Get(ValueField).Equals(a))
}
dsx := store.GetDataset(id1)
dsy := store.GetDataset(id1)
// dsx: |a| -> |b|
assert.True(dsx.Head().Get(ValueField).Equals(a))
b := types.String("b")
dsx, err := store.CommitValue(dsx, b)
assert.NoError(err)
assert.True(dsx.Head().Get(ValueField).Equals(b))
// dsy: |a| -> |c|
assert.True(dsy.Head().Get(ValueField).Equals(a))
c := types.String("c")
dsy, err = store.CommitValue(dsy, c)
assert.Error(err)
assert.True(dsy.Head().Get(ValueField).Equals(b))
// Commit failed, but dsy now has latest head, so we should be able to just try again.
// dsy: |b| -> |c|
dsy, err = store.CommitValue(dsy, c)
assert.NoError(err)
assert.True(dsy.Head().Get(ValueField).Equals(c))
}
func TestIdValidation(t *testing.T) {
assert := assert.New(t)
stg := &chunks.MemoryStorage{}
store := NewDatabase(stg.NewView())
invalidDatasetNames := []string{" ", "", "a ", " a", "$", "#", ":", "\n", "💩"}
for _, id := range invalidDatasetNames {
assert.Panics(func() {
store.GetDataset(id)
})
}
}
func TestHeadValueFunctions(t *testing.T) {
assert := assert.New(t)
id1 := "testdataset"
id2 := "otherdataset"
stg := &chunks.MemoryStorage{}
store := NewDatabase(stg.NewView())
defer store.Close()
ds1 := store.GetDataset(id1)
assert.False(ds1.HasHead())
// ds1: |a|
a := types.String("a")
ds1, err := store.CommitValue(ds1, a)
assert.NoError(err)
as
gitextract_tbm5d7az/
├── .dockerignore
├── .gitignore
├── .travis.yml
├── CONTRIBUTING.md
├── Dockerfile
├── HACKING.md
├── LICENSE
├── README.md
├── cmd/
│ └── util/
│ └── kingpin_command.go
├── codecov.yml
├── doc/
│ ├── cli-tour.md
│ ├── decent/
│ │ ├── about.md
│ │ ├── architectures.md
│ │ ├── demo-ipfs-chat.md
│ │ ├── demo-p2p-chat.md
│ │ └── quickstart.md
│ ├── faq.md
│ ├── go-tour.md
│ ├── intro.md
│ └── spelling.md
├── go/
│ ├── chunks/
│ │ ├── chunk.go
│ │ ├── chunk_serializer.go
│ │ ├── chunk_serializer_test.go
│ │ ├── chunk_store.go
│ │ ├── chunk_store_common_test.go
│ │ ├── chunk_test.go
│ │ ├── memory_store.go
│ │ ├── memory_store_test.go
│ │ ├── put_cache.go
│ │ ├── remote_requests.go
│ │ ├── remote_requests_test.go
│ │ └── test_utils.go
│ ├── config/
│ │ ├── config.go
│ │ ├── config_test.go
│ │ ├── resolver.go
│ │ └── resolver_test.go
│ ├── constants/
│ │ ├── http.go
│ │ └── version.go
│ ├── d/
│ │ ├── check_error.go
│ │ ├── try.go
│ │ └── try_test.go
│ ├── datas/
│ │ ├── commit.go
│ │ ├── commit_options.go
│ │ ├── commit_test.go
│ │ ├── database.go
│ │ ├── database_common.go
│ │ ├── database_server.go
│ │ ├── database_test.go
│ │ ├── dataset.go
│ │ ├── dataset_test.go
│ │ ├── http_chunk_store.go
│ │ ├── http_chunk_store_test.go
│ │ ├── pull.go
│ │ ├── pull_test.go
│ │ ├── pulling.md
│ │ ├── remote_database_handlers.go
│ │ ├── remote_database_handlers_test.go
│ │ ├── serialize_hashes.go
│ │ └── serialize_hashes_test.go
│ ├── diff/
│ │ ├── apply_patch.go
│ │ ├── apply_patch_test.go
│ │ ├── diff.go
│ │ ├── diff_test.go
│ │ ├── patch.go
│ │ ├── patch_test.go
│ │ ├── print_diff.go
│ │ └── summary.go
│ ├── hash/
│ │ ├── base32.go
│ │ ├── base32_test.go
│ │ ├── hash.go
│ │ ├── hash_slice.go
│ │ ├── hash_slice_test.go
│ │ └── hash_test.go
│ ├── marshal/
│ │ ├── decode.go
│ │ ├── decode_test.go
│ │ ├── encode.go
│ │ ├── encode_test.go
│ │ ├── encode_type.go
│ │ └── encode_type_test.go
│ ├── merge/
│ │ ├── candidate.go
│ │ ├── three_way.go
│ │ ├── three_way_keyval_test.go
│ │ ├── three_way_list.go
│ │ ├── three_way_list_test.go
│ │ ├── three_way_ordered_sequence.go
│ │ ├── three_way_set_test.go
│ │ └── three_way_test.go
│ ├── metrics/
│ │ ├── histogram.go
│ │ └── histogram_test.go
│ ├── nbs/
│ │ ├── NBS-on-AWS.md
│ │ ├── README.md
│ │ ├── aws_chunk_source.go
│ │ ├── aws_chunk_source_test.go
│ │ ├── aws_table_persister.go
│ │ ├── aws_table_persister_test.go
│ │ ├── block_store_test.go
│ │ ├── cache.go
│ │ ├── conjoiner.go
│ │ ├── conjoiner_test.go
│ │ ├── dynamo_fake_test.go
│ │ ├── dynamo_manifest.go
│ │ ├── dynamo_manifest_test.go
│ │ ├── dynamo_table_reader.go
│ │ ├── dynamo_table_reader_test.go
│ │ ├── factory.go
│ │ ├── factory_test.go
│ │ ├── fd_cache.go
│ │ ├── fd_cache_test.go
│ │ ├── file_manifest.go
│ │ ├── file_manifest_test.go
│ │ ├── file_table_persister.go
│ │ ├── file_table_persister_test.go
│ │ ├── fs_table_cache.go
│ │ ├── fs_table_cache_test.go
│ │ ├── manifest.go
│ │ ├── manifest_cache.go
│ │ ├── manifest_cache_test.go
│ │ ├── mem_table.go
│ │ ├── mem_table_test.go
│ │ ├── mmap_table_reader.go
│ │ ├── mmap_table_reader_test.go
│ │ ├── persisting_chunk_source.go
│ │ ├── persisting_chunk_source_test.go
│ │ ├── root_tracker_test.go
│ │ ├── s3_fake_test.go
│ │ ├── s3_table_reader.go
│ │ ├── s3_table_reader_test.go
│ │ ├── stats.go
│ │ ├── stats_test.go
│ │ ├── store.go
│ │ ├── table.go
│ │ ├── table_persister.go
│ │ ├── table_persister_test.go
│ │ ├── table_reader.go
│ │ ├── table_set.go
│ │ ├── table_set_test.go
│ │ ├── table_test.go
│ │ ├── table_writer.go
│ │ └── test/
│ │ └── manifest_clobber.go
│ ├── ngql/
│ │ ├── README.md
│ │ ├── query.go
│ │ ├── query_test.go
│ │ └── types.go
│ ├── nomdl/
│ │ ├── lexer.go
│ │ ├── parser.go
│ │ └── parser_test.go
│ ├── perf/
│ │ ├── hash-perf-rig/
│ │ │ ├── README.md
│ │ │ └── main.go
│ │ └── suite/
│ │ ├── suite.go
│ │ └── suite_test.go
│ ├── sloppy/
│ │ ├── sloppy.go
│ │ └── sloppy_test.go
│ ├── spec/
│ │ ├── absolute_path.go
│ │ ├── absolute_path_test.go
│ │ ├── commit_meta.go
│ │ ├── commit_meta_test.go
│ │ ├── spec.go
│ │ ├── spec_test.go
│ │ └── util.go
│ ├── types/
│ │ ├── blob.go
│ │ ├── blob_editor.go
│ │ ├── blob_editor_test.go
│ │ ├── blob_leaf_sequence.go
│ │ ├── blob_test.go
│ │ ├── bool.go
│ │ ├── codec.go
│ │ ├── codec_test.go
│ │ ├── collection.go
│ │ ├── collection_test.go
│ │ ├── common_supertype.go
│ │ ├── common_supertype_test.go
│ │ ├── compare_test.go
│ │ ├── edit_distance.go
│ │ ├── edit_distance_test.go
│ │ ├── encode_human_readable.go
│ │ ├── encode_human_readable_test.go
│ │ ├── encoding_test.go
│ │ ├── equals_test.go
│ │ ├── get_hash.go
│ │ ├── graph_builder.go
│ │ ├── graph_builder_test.go
│ │ ├── incremental_test.go
│ │ ├── indexed_sequence_diff.go
│ │ ├── indexed_sequences.go
│ │ ├── leaf_sequence.go
│ │ ├── less.go
│ │ ├── list.go
│ │ ├── list_editor.go
│ │ ├── list_editor_test.go
│ │ ├── list_iterator.go
│ │ ├── list_iterator_test.go
│ │ ├── list_leaf_sequence.go
│ │ ├── list_test.go
│ │ ├── make_type.go
│ │ ├── map.go
│ │ ├── map_editor.go
│ │ ├── map_iterator.go
│ │ ├── map_iterator_test.go
│ │ ├── map_leaf_sequence.go
│ │ ├── map_test.go
│ │ ├── meta_sequence.go
│ │ ├── noms_kind.go
│ │ ├── number.go
│ │ ├── number_util.go
│ │ ├── opcache.go
│ │ ├── opcache_compare.go
│ │ ├── opcache_test.go
│ │ ├── ordered_sequences.go
│ │ ├── ordered_sequences_diff.go
│ │ ├── ordered_sequences_diff_test.go
│ │ ├── path.go
│ │ ├── path_test.go
│ │ ├── perf/
│ │ │ ├── dummy.go
│ │ │ └── perf_test.go
│ │ ├── primitives_test.go
│ │ ├── ref.go
│ │ ├── ref_heap.go
│ │ ├── ref_heap_test.go
│ │ ├── ref_test.go
│ │ ├── rolling_value_hasher.go
│ │ ├── rungen.go
│ │ ├── sequence.go
│ │ ├── sequence_chunker.go
│ │ ├── sequence_concat.go
│ │ ├── sequence_cursor.go
│ │ ├── sequence_cursor_test.go
│ │ ├── set.go
│ │ ├── set_editor.go
│ │ ├── set_iterator.go
│ │ ├── set_iterator_test.go
│ │ ├── set_leaf_sequence.go
│ │ ├── set_test.go
│ │ ├── simplify.go
│ │ ├── simplify_test.go
│ │ ├── string.go
│ │ ├── string_test.go
│ │ ├── struct.go
│ │ ├── struct_test.go
│ │ ├── subtype.go
│ │ ├── subtype_test.go
│ │ ├── type.go
│ │ ├── type_desc.go
│ │ ├── type_test.go
│ │ ├── util_test.go
│ │ ├── validate_type.go
│ │ ├── validating_decoder.go
│ │ ├── validating_decoder_test.go
│ │ ├── value.go
│ │ ├── value_decoder.go
│ │ ├── value_stats.go
│ │ ├── value_store.go
│ │ ├── value_store_test.go
│ │ ├── walk.go
│ │ ├── walk_refs.go
│ │ ├── walk_refs_test.go
│ │ └── walk_test.go
│ └── util/
│ ├── clienttest/
│ │ └── client_test_suite.go
│ ├── datetime/
│ │ ├── date_time.go
│ │ └── date_time_test.go
│ ├── exit/
│ │ └── exit.go
│ ├── functions/
│ │ ├── all.go
│ │ └── all_test.go
│ ├── json/
│ │ ├── from_json.go
│ │ ├── from_json_test.go
│ │ ├── to_json.go
│ │ └── to_json_test.go
│ ├── math/
│ │ └── minmax.go
│ ├── outputpager/
│ │ └── page_output.go
│ ├── profile/
│ │ └── profile.go
│ ├── progressreader/
│ │ └── reader.go
│ ├── random/
│ │ ├── id.go
│ │ └── id_test.go
│ ├── sizecache/
│ │ ├── size_cache.go
│ │ └── size_cache_test.go
│ ├── status/
│ │ └── status.go
│ ├── test/
│ │ └── equals_ignore_hashes.go
│ ├── verbose/
│ │ └── verbose.go
│ └── writers/
│ ├── max_line_writer.go
│ ├── prefix_writer.go
│ └── writers_test.go
├── go.mod
├── go.sum
├── release.sh
├── samples/
│ ├── cli/
│ │ └── nomsconfig/
│ │ └── README.md
│ └── go/
│ ├── csv/
│ │ ├── README.md
│ │ ├── common.go
│ │ ├── csv_reader.go
│ │ ├── csv_reader_test.go
│ │ ├── kind_slice.go
│ │ ├── kind_slice_test.go
│ │ ├── read.go
│ │ ├── read_test.go
│ │ ├── schema.go
│ │ ├── schema_test.go
│ │ ├── write.go
│ │ └── write_test.go
│ ├── decent/
│ │ ├── README.md
│ │ ├── data/
│ │ │ ├── godfather.html
│ │ │ ├── godfather2.html
│ │ │ └── godfather3.html
│ │ ├── dbg/
│ │ │ └── debug.go
│ │ ├── ipfs-chat/
│ │ │ └── main.go
│ │ ├── lib/
│ │ │ ├── datapager.go
│ │ │ ├── event.go
│ │ │ ├── importer.go
│ │ │ ├── logger.go
│ │ │ ├── model.go
│ │ │ ├── model_test.go
│ │ │ ├── pubsub.go
│ │ │ ├── term_index.go
│ │ │ ├── term_index_test.go
│ │ │ └── termui.go
│ │ └── p2p-chat/
│ │ ├── README.md
│ │ └── main.go
│ └── nomdex/
│ ├── Readme.md
│ ├── expr.go
│ ├── nomdex.go
│ ├── nomdex_find.go
│ ├── nomdex_test.go
│ ├── nomdex_update.go
│ ├── parser.go
│ ├── parser_test.go
│ ├── query_range.go
│ └── query_range_test.go
└── tools/
├── file/
│ ├── file.go
│ └── file_test.go
├── licensify.py
├── loadtest/
│ └── loadtest.go
├── noms/
│ ├── README.md
│ ├── __init__.py
│ ├── copy.py
│ ├── copy_test.py
│ ├── pushd.py
│ ├── staging.py
│ ├── staging_test.py
│ ├── symlink.py
│ └── symlink_test.py
└── runner/
├── serial.go
└── serial_test.go
Showing preview only (344K chars total). Download the full file or copy to clipboard to get everything.
SYMBOL INDEX (3933 symbols across 300 files)
FILE: cmd/util/kingpin_command.go
type KingpinHandler (line 9) | type KingpinHandler
type KingpinCommand (line 10) | type KingpinCommand
FILE: go/chunks/chunk.go
type Chunk (line 16) | type Chunk struct
method Hash (line 23) | func (c Chunk) Hash() hash.Hash {
method Data (line 27) | func (c Chunk) Data() []byte {
method IsEmpty (line 31) | func (c Chunk) IsEmpty() bool {
function NewChunk (line 36) | func NewChunk(data []byte) Chunk {
function NewChunkWithHash (line 42) | func NewChunkWithHash(r hash.Hash, data []byte) Chunk {
type ChunkWriter (line 47) | type ChunkWriter struct
method Write (line 59) | func (w *ChunkWriter) Write(data []byte) (int, error) {
method Chunk (line 69) | func (w *ChunkWriter) Chunk() Chunk {
method Close (line 75) | func (w *ChunkWriter) Close() error {
function NewChunkWriter (line 52) | func NewChunkWriter() *ChunkWriter {
FILE: go/chunks/chunk_serializer.go
function Serialize (line 30) | func Serialize(chunk Chunk, writer io.Writer) {
function Deserialize (line 51) | func Deserialize(reader io.Reader, chunkChan chan<- *Chunk) (err error) {
function deserializeChunk (line 67) | func deserializeChunk(reader io.Reader) (Chunk, error) {
FILE: go/chunks/chunk_serializer_test.go
function TestSerializeRoundTrip (line 14) | func TestSerializeRoundTrip(t *testing.T) {
function TestBadSerialization (line 40) | func TestBadSerialization(t *testing.T) {
FILE: go/chunks/chunk_store.go
type ChunkStore (line 15) | type ChunkStore interface
type Factory (line 75) | type Factory interface
FILE: go/chunks/chunk_store_common_test.go
type ChunkStoreTestSuite (line 14) | type ChunkStoreTestSuite struct
method TestChunkStorePut (line 19) | func (suite *ChunkStoreTestSuite) TestChunkStorePut() {
method TestChunkStoreRoot (line 30) | func (suite *ChunkStoreTestSuite) TestChunkStoreRoot() {
method TestChunkStoreCommitPut (line 47) | func (suite *ChunkStoreTestSuite) TestChunkStoreCommitPut() {
method TestChunkStoreGetNonExisting (line 65) | func (suite *ChunkStoreTestSuite) TestChunkStoreGetNonExisting() {
method TestChunkStoreVersion (line 72) | func (suite *ChunkStoreTestSuite) TestChunkStoreVersion() {
method TestChunkStoreCommitUnchangedRoot (line 82) | func (suite *ChunkStoreTestSuite) TestChunkStoreCommitUnchangedRoot() {
FILE: go/chunks/chunk_test.go
function TestChunk (line 13) | func TestChunk(t *testing.T) {
function TestChunkWriteAfterCloseFails (line 20) | func TestChunkWriteAfterCloseFails(t *testing.T) {
function TestChunkWriteAfterChunkFails (line 31) | func TestChunkWriteAfterChunkFails(t *testing.T) {
function TestChunkChunkCloses (line 42) | func TestChunkChunkCloses(t *testing.T) {
FILE: go/chunks/memory_store.go
type MemoryStorage (line 20) | type MemoryStorage struct
method NewView (line 28) | func (ms *MemoryStorage) NewView() ChunkStore {
method Get (line 34) | func (ms *MemoryStorage) Get(h hash.Hash) Chunk {
method Has (line 45) | func (ms *MemoryStorage) Has(r hash.Hash) bool {
method Len (line 53) | func (ms *MemoryStorage) Len() int {
method Root (line 60) | func (ms *MemoryStorage) Root() hash.Hash {
method Update (line 69) | func (ms *MemoryStorage) Update(current, last hash.Hash, novel map[has...
type MemoryStoreView (line 90) | type MemoryStoreView struct
method Get (line 98) | func (ms *MemoryStoreView) Get(h hash.Hash) Chunk {
method GetMany (line 107) | func (ms *MemoryStoreView) GetMany(hashes hash.HashSet, foundChunks ch...
method Has (line 117) | func (ms *MemoryStoreView) Has(h hash.Hash) bool {
method HasMany (line 126) | func (ms *MemoryStoreView) HasMany(hashes hash.HashSet) hash.HashSet {
method Version (line 136) | func (ms *MemoryStoreView) Version() string {
method Put (line 140) | func (ms *MemoryStoreView) Put(c Chunk) {
method Len (line 149) | func (ms *MemoryStoreView) Len() int {
method Rebase (line 155) | func (ms *MemoryStoreView) Rebase() {
method Root (line 161) | func (ms *MemoryStoreView) Root() hash.Hash {
method Commit (line 167) | func (ms *MemoryStoreView) Commit(current, last hash.Hash) bool {
method Stats (line 182) | func (ms *MemoryStoreView) Stats() interface{} {
method StatsSummary (line 186) | func (ms *MemoryStoreView) StatsSummary() string {
method Close (line 190) | func (ms *MemoryStoreView) Close() error {
type memoryStoreFactory (line 194) | type memoryStoreFactory struct
method CreateStoreFromCache (line 203) | func (f *memoryStoreFactory) CreateStoreFromCache(ns string) ChunkStore {
method CreateStore (line 207) | func (f *memoryStoreFactory) CreateStore(ns string) ChunkStore {
method Shutter (line 221) | func (f *memoryStoreFactory) Shutter() {
function NewMemoryStoreFactory (line 199) | func NewMemoryStoreFactory() Factory {
FILE: go/chunks/memory_store_test.go
function TestMemoryStoreTestSuite (line 13) | func TestMemoryStoreTestSuite(t *testing.T) {
type MemoryStoreTestSuite (line 17) | type MemoryStoreTestSuite struct
method SetupTest (line 21) | func (suite *MemoryStoreTestSuite) SetupTest() {
method TearDownTest (line 25) | func (suite *MemoryStoreTestSuite) TearDownTest() {
FILE: go/chunks/put_cache.go
function newUnwrittenPutCache (line 13) | func newUnwrittenPutCache() *unwrittenPutCache {
type unwrittenPutCache (line 17) | type unwrittenPutCache struct
method Add (line 22) | func (p *unwrittenPutCache) Add(c Chunk) bool {
method Has (line 33) | func (p *unwrittenPutCache) Has(c Chunk) (has bool) {
method Get (line 40) | func (p *unwrittenPutCache) Get(r hash.Hash) Chunk {
method Clear (line 49) | func (p *unwrittenPutCache) Clear(chunks []Chunk) {
FILE: go/chunks/remote_requests.go
type ReadRequest (line 13) | type ReadRequest interface
function NewGetRequest (line 18) | func NewGetRequest(r hash.Hash, ch chan<- *Chunk) GetRequest {
type GetRequest (line 22) | type GetRequest struct
method Hashes (line 56) | func (g GetRequest) Hashes() hash.HashSet {
method Outstanding (line 60) | func (g GetRequest) Outstanding() OutstandingRequest {
function NewGetManyRequest (line 27) | func NewGetManyRequest(hashes hash.HashSet, wg *sync.WaitGroup, ch chan<...
type GetManyRequest (line 31) | type GetManyRequest struct
method Hashes (line 64) | func (g GetManyRequest) Hashes() hash.HashSet {
method Outstanding (line 68) | func (g GetManyRequest) Outstanding() OutstandingRequest {
function NewAbsentRequest (line 37) | func NewAbsentRequest(r hash.Hash, ch chan<- bool) AbsentRequest {
type AbsentRequest (line 41) | type AbsentRequest struct
method Hashes (line 72) | func (h AbsentRequest) Hashes() hash.HashSet {
method Outstanding (line 76) | func (h AbsentRequest) Outstanding() OutstandingRequest {
function NewAbsentManyRequest (line 46) | func NewAbsentManyRequest(hashes hash.HashSet, wg *sync.WaitGroup, ch ch...
type AbsentManyRequest (line 50) | type AbsentManyRequest struct
method Hashes (line 80) | func (h AbsentManyRequest) Hashes() hash.HashSet {
method Outstanding (line 84) | func (h AbsentManyRequest) Outstanding() OutstandingRequest {
type OutstandingRequest (line 88) | type OutstandingRequest interface
type OutstandingGet (line 93) | type OutstandingGet
method Satisfy (line 104) | func (r OutstandingGet) Satisfy(h hash.Hash, c *Chunk) {
method Fail (line 108) | func (r OutstandingGet) Fail() {
type OutstandingGetMany (line 94) | type OutstandingGetMany struct
method Satisfy (line 112) | func (ogm OutstandingGetMany) Satisfy(h hash.Hash, c *Chunk) {
method Fail (line 117) | func (ogm OutstandingGetMany) Fail() {
type OutstandingAbsent (line 98) | type OutstandingAbsent
method Satisfy (line 121) | func (oh OutstandingAbsent) Satisfy(h hash.Hash, c *Chunk) {
method Fail (line 125) | func (oh OutstandingAbsent) Fail() {
type OutstandingAbsentMany (line 99) | type OutstandingAbsentMany struct
method Satisfy (line 129) | func (ohm OutstandingAbsentMany) Satisfy(h hash.Hash, c *Chunk) {
method Fail (line 134) | func (ohm OutstandingAbsentMany) Fail() {
type ReadBatch (line 139) | type ReadBatch
method Close (line 142) | func (rb *ReadBatch) Close() error {
FILE: go/chunks/remote_requests_test.go
function TestGetRequestBatch (line 15) | func TestGetRequestBatch(t *testing.T) {
function TestGetManyRequestBatch (line 78) | func TestGetManyRequestBatch(t *testing.T) {
function TestAbsentManyRequestBatch (line 120) | func TestAbsentManyRequestBatch(t *testing.T) {
FILE: go/chunks/test_utils.go
function assertInputInStore (line 13) | func assertInputInStore(input string, h hash.Hash, s ChunkStore, assert ...
function assertInputNotInStore (line 19) | func assertInputNotInStore(input string, h hash.Hash, s ChunkStore, asse...
type TestStorage (line 24) | type TestStorage struct
method NewView (line 28) | func (t *TestStorage) NewView() *TestStoreView {
type TestStoreView (line 32) | type TestStoreView struct
method Get (line 39) | func (s *TestStoreView) Get(h hash.Hash) Chunk {
method GetMany (line 44) | func (s *TestStoreView) GetMany(hashes hash.HashSet, foundChunks chan ...
method Has (line 49) | func (s *TestStoreView) Has(h hash.Hash) bool {
method HasMany (line 54) | func (s *TestStoreView) HasMany(hashes hash.HashSet) hash.HashSet {
method Put (line 59) | func (s *TestStoreView) Put(c Chunk) {
type TestStoreFactory (line 64) | type TestStoreFactory struct
method CreateStore (line 72) | func (f *TestStoreFactory) CreateStore(ns string) ChunkStore {
method Shutter (line 83) | func (f *TestStoreFactory) Shutter() {
function NewTestStoreFactory (line 68) | func NewTestStoreFactory() *TestStoreFactory {
FILE: go/config/config.go
type Config (line 19) | type Config struct
method WriteTo (line 83) | func (c *Config) WriteTo(configHome string) (string, error) {
method String (line 125) | func (c *Config) String() string {
method writeableString (line 134) | func (c *Config) writeableString() string {
type DbConfig (line 24) | type DbConfig struct
constant NomsConfigFile (line 29) | NomsConfigFile = ".nomsconfig"
constant DefaultDbAlias (line 30) | DefaultDbAlias = "default"
function FindNomsConfig (line 38) | func FindNomsConfig() (*Config, error) {
function ReadConfig (line 62) | func ReadConfig(name string) (*Config, error) {
function NewConfig (line 75) | func NewConfig(data string) (*Config, error) {
function absDbSpec (line 96) | func absDbSpec(configHome string, url string) string {
function qualifyPaths (line 111) | func qualifyPaths(configPath string, c *Config) (*Config, error) {
FILE: go/config/config_test.go
constant nbsSpec (line 19) | nbsSpec = "nbs:./local"
constant memSpec (line 20) | memSpec = "mem"
constant httpSpec (line 21) | httpSpec = "http://test.com:8080/foo"
constant nbsAbsSpec (line 22) | nbsAbsSpec = "nbs:/tmp/noms"
constant remoteAlias (line 23) | remoteAlias = "origin"
type paths (line 62) | type paths struct
function getPaths (line 67) | func getPaths(assert *assert.Assertions, base string) paths {
function qualifyFilePath (line 77) | func qualifyFilePath(assert *assert.Assertions, path string) string {
function assertDbSpecsEquiv (line 83) | func assertDbSpecsEquiv(assert *assert.Assertions, expected string, actu...
function validateConfig (line 104) | func validateConfig(assert *assert.Assertions, file string, e *Config, a...
function writeConfig (line 114) | func writeConfig(assert *assert.Assertions, c *Config, home string) stri...
function TestConfig (line 120) | func TestConfig(t *testing.T) {
function TestUnreadableConfig (line 149) | func TestUnreadableConfig(t *testing.T) {
function TestNoConfig (line 164) | func TestNoConfig(t *testing.T) {
function TestBadConfig (line 173) | func TestBadConfig(t *testing.T) {
function TestQualifyingPaths (line 184) | func TestQualifyingPaths(t *testing.T) {
function TestCwd (line 197) | func TestCwd(t *testing.T) {
FILE: go/config/resolver.go
type Resolver (line 18) | type Resolver struct
method verbose (line 39) | func (r *Resolver) verbose(orig string, replacement string) string {
method ResolveDbSpec (line 52) | func (r *Resolver) ResolveDbSpec(str string) string {
method ResolvePathSpec (line 70) | func (r *Resolver) ResolvePathSpec(str string) string {
method GetDatabase (line 90) | func (r *Resolver) GetDatabase(str string) (datas.Database, error) {
method GetChunkStore (line 99) | func (r *Resolver) GetChunkStore(str string) (chunks.ChunkStore, error) {
method GetDataset (line 110) | func (r *Resolver) GetDataset(str string) (datas.Database, datas.Datas...
method GetPath (line 121) | func (r *Resolver) GetPath(str string) (datas.Database, types.Value, e...
function NewResolver (line 27) | func NewResolver() *Resolver {
FILE: go/config/resolver_test.go
constant localSpec (line 18) | localSpec = nbsSpec
constant remoteSpec (line 19) | remoteSpec = httpSpec
constant testDs (line 20) | testDs = "testds"
constant testObject (line 21) | testObject = "#pckdvpvr9br1fie6c3pjudrlthe7na18"
type testData (line 24) | type testData struct
function withConfig (line 63) | func withConfig(t *testing.T) *Resolver {
function withoutConfig (line 74) | func withoutConfig(t *testing.T) *Resolver {
function assertPathSpecsEquiv (line 83) | func assertPathSpecsEquiv(assert *assert.Assertions, expected string, ac...
function TestResolveDatabaseWithConfig (line 97) | func TestResolveDatabaseWithConfig(t *testing.T) {
function TestResolvePathWithConfig (line 106) | func TestResolvePathWithConfig(t *testing.T) {
function TestResolveDatabaseWithoutConfig (line 115) | func TestResolveDatabaseWithoutConfig(t *testing.T) {
function TestResolvePathWithoutConfig (line 124) | func TestResolvePathWithoutConfig(t *testing.T) {
function TestResolveDestPathWithDot (line 134) | func TestResolveDestPathWithDot(t *testing.T) {
FILE: go/constants/http.go
constant RootPath (line 8) | RootPath = "/root/"
constant GetRefsPath (line 9) | GetRefsPath = "/getRefs/"
constant GetBlobPath (line 10) | GetBlobPath = "/getBlob/"
constant HasRefsPath (line 11) | HasRefsPath = "/hasRefs/"
constant WriteValuePath (line 12) | WriteValuePath = "/writeValue/"
constant BasePath (line 13) | BasePath = "/"
constant GraphQLPath (line 15) | GraphQLPath = "/graphql/"
constant StatsPath (line 16) | StatsPath = "/stats/"
FILE: go/constants/version.go
constant NomsVersion (line 8) | NomsVersion = "7.18"
FILE: go/d/check_error.go
function CheckError (line 15) | func CheckError(err error) {
function CheckErrorNoUsage (line 23) | func CheckErrorNoUsage(err error) {
FILE: go/d/try.go
type panicker (line 21) | type panicker struct
method Errorf (line 24) | func (s panicker) Errorf(format string, args ...interface{}) {
function Panic (line 30) | func Panic(format string, args ...interface{}) {
function PanicIfError (line 42) | func PanicIfError(err error) {
function PanicIfTrue (line 49) | func PanicIfTrue(b bool) {
function PanicIfFalse (line 56) | func PanicIfFalse(b bool) {
function Try (line 66) | func Try(f func(), types ...interface{}) (err error) {
function TryCatch (line 76) | func TryCatch(f func(), catch func(err error) error) (err error) {
type WrappedError (line 82) | type WrappedError interface
function Wrap (line 89) | func Wrap(err error) WrappedError {
function Unwrap (line 105) | func Unwrap(err error) error {
function causeInTypes (line 114) | func causeInTypes(err error, types ...interface{}) bool {
function PanicIfNotType (line 126) | func PanicIfNotType(err error, types ...interface{}) error {
type wrappedError (line 140) | type wrappedError struct
method Error (line 145) | func (we wrappedError) Error() string { return we.msg }
method Cause (line 146) | func (we wrappedError) Cause() error { return we.cause }
method Unwrap (line 147) | func (we wrappedError) Unwrap() error { return we.cause }
type stackTracer (line 149) | type stackTracer struct
method Errorf (line 153) | func (s *stackTracer) Errorf(format string, args ...interface{}) {
function recoverWrappedTypes (line 157) | func recoverWrappedTypes(errp *error, types []interface{}) {
function recoverWrapped (line 171) | func recoverWrapped(errp *error, catch func(err error) error) {
FILE: go/d/try_test.go
type testError (line 20) | type testError struct
method Error (line 24) | func (e testError) Error() string { return e.s }
type testError2 (line 26) | type testError2 struct
method Error (line 30) | func (e testError2) Error() string { return e.s }
function TestTry2 (line 32) | func TestTry2(t *testing.T) {
function TestTryCatch (line 66) | func TestTryCatch(t *testing.T) {
function TestUnwrap (line 115) | func TestUnwrap(t *testing.T) {
function TestPanicIfTrue (line 124) | func TestPanicIfTrue(t *testing.T) {
function TestPanicIfFalse (line 154) | func TestPanicIfFalse(t *testing.T) {
function TestPanicIfNotType (line 184) | func TestPanicIfNotType(t *testing.T) {
function TestCauseInTypes (line 198) | func TestCauseInTypes(t *testing.T) {
function TestWrap (line 210) | func TestWrap(t *testing.T) {
FILE: go/datas/commit.go
constant ParentsField (line 17) | ParentsField = "parents"
constant ValueField (line 18) | ValueField = "value"
constant MetaField (line 19) | MetaField = "meta"
constant commitName (line 20) | commitName = "Commit"
function NewCommit (line 43) | func NewCommit(value types.Value, parents types.Set, meta types.Struct) ...
function FindCommonAncestor (line 50) | func FindCommonAncestor(c1, c2 types.Ref, vr types.ValueReader) (a types...
function parentsToQueue (line 77) | func parentsToQueue(refs types.RefSlice, q *types.RefByHeight, vr types....
function findCommonRef (line 88) | func findCommonRef(a, b types.RefSlice) (types.Ref, bool) {
function makeCommitStructType (line 106) | func makeCommitStructType(metaType, parentsType, valueType *types.Type) ...
function getRefElementType (line 123) | func getRefElementType(t *types.Type) *types.Type {
function IsCommitType (line 128) | func IsCommitType(t *types.Type) bool {
function IsCommit (line 132) | func IsCommit(v types.Value) bool {
function IsRefOfCommitType (line 136) | func IsRefOfCommitType(t *types.Type) bool {
FILE: go/datas/commit_options.go
type CommitOptions (line 13) | type CommitOptions struct
FILE: go/datas/commit_test.go
function TestNewCommit (line 18) | func TestNewCommit(t *testing.T) {
function TestCommitWithoutMetaField (line 88) | func TestCommitWithoutMetaField(t *testing.T) {
function toRefSet (line 112) | func toRefSet(vrw types.ValueReadWriter, commits ...types.Struct) types....
function toValuesString (line 121) | func toValuesString(refSet types.Set, vr types.ValueReader) string {
function TestFindCommonAncestor (line 129) | func TestFindCommonAncestor(t *testing.T) {
function TestNewCommitRegressionTest (line 207) | func TestNewCommitRegressionTest(t *testing.T) {
FILE: go/datas/database.go
type Database (line 24) | type Database interface
function NewDatabase (line 121) | func NewDatabase(cs chunks.ChunkStore) Database {
FILE: go/datas/database_common.go
type database (line 19) | type database struct
method chunkStore (line 48) | func (db *database) chunkStore() chunks.ChunkStore {
method Stats (line 52) | func (db *database) Stats() interface{} {
method StatsSummary (line 56) | func (db *database) StatsSummary() string {
method Flush (line 60) | func (db *database) Flush() {
method Datasets (line 71) | func (db *database) Datasets() types.Map {
method GetDataset (line 80) | func (db *database) GetDataset(datasetID string) Dataset {
method Rebase (line 92) | func (db *database) Rebase() {
method Close (line 96) | func (db *database) Close() error {
method SetHead (line 100) | func (db *database) SetHead(ds Dataset, newHeadRef types.Ref) (Dataset...
method doSetHead (line 104) | func (db *database) doSetHead(ds Dataset, newHeadRef types.Ref) error {
method FastForward (line 117) | func (db *database) FastForward(ds Dataset, newHeadRef types.Ref) (Dat...
method doFastForward (line 121) | func (db *database) doFastForward(ds Dataset, newHeadRef types.Ref) er...
method Commit (line 135) | func (db *database) Commit(ds Dataset, v types.Value, opts CommitOptio...
method CommitValue (line 142) | func (db *database) CommitValue(ds Dataset, v types.Value) (Dataset, e...
method doCommit (line 147) | func (db *database) doCommit(datasetID string, commit types.Struct, me...
method Delete (line 194) | func (db *database) Delete(ds Dataset) (Dataset, error) {
method doDelete (line 199) | func (db *database) doDelete(datasetIDstr string) error {
method tryCommitChunks (line 226) | func (db *database) tryCommitChunks(currentDatasets types.Map, current...
method validateRefAsCommit (line 235) | func (db *database) validateRefAsCommit(r types.Ref) types.Struct {
method doHeadUpdate (line 263) | func (db *database) doHeadUpdate(ds Dataset, updateFunc func(ds Datase...
type rootTracker (line 30) | type rootTracker interface
function newDatabase (line 36) | func newDatabase(cs chunks.ChunkStore) *database {
function buildNewCommit (line 247) | func buildNewCommit(ds Dataset, v types.Value, opts CommitOptions) types...
FILE: go/datas/database_server.go
type connectionState (line 20) | type connectionState struct
type RemoteDatabaseServer (line 25) | type RemoteDatabaseServer struct
method Port (line 47) | func (s *RemoteDatabaseServer) Port() int {
method Run (line 77) | func (s *RemoteDatabaseServer) Run() {
method connState (line 139) | func (s *RemoteDatabaseServer) connState(c net.Conn, cs http.ConnState) {
method Stop (line 148) | func (s *RemoteDatabaseServer) Stop() {
function NewRemoteDatabaseServer (line 36) | func NewRemoteDatabaseServer(cs chunks.ChunkStore, address string, port ...
function Router (line 51) | func Router(cs chunks.ChunkStore, prefix string) *httprouter.Router {
function makeHandle (line 116) | func makeHandle(hndlr Handler, cs chunks.ChunkStore) httprouter.Handle {
function noopHandle (line 122) | func noopHandle(w http.ResponseWriter, r *http.Request, ps httprouter.Pa...
function corsHandle (line 125) | func corsHandle(f httprouter.Handle) httprouter.Handle {
FILE: go/datas/database_test.go
function TestLocalDatabase (line 18) | func TestLocalDatabase(t *testing.T) {
function TestRemoteDatabase (line 22) | func TestRemoteDatabase(t *testing.T) {
function TestValidateRef (line 26) | func TestValidateRef(t *testing.T) {
type DatabaseSuite (line 37) | type DatabaseSuite struct
method TearDownTest (line 66) | func (suite *DatabaseSuite) TearDownTest() {
method TestTolerateUngettableRefs (line 76) | func (suite *DatabaseSuite) TestTolerateUngettableRefs() {
method TestCompletenessCheck (line 80) | func (suite *DatabaseSuite) TestCompletenessCheck() {
method TestRebase (line 100) | func (suite *DatabaseSuite) TestRebase() {
method TestCommitProperlyTracksRoot (line 138) | func (suite *DatabaseSuite) TestCommitProperlyTracksRoot() {
method TestDatabaseCommit (line 161) | func (suite *DatabaseSuite) TestDatabaseCommit() {
method TestDatasetsMapType (line 223) | func (suite *DatabaseSuite) TestDatasetsMapType() {
method TestDatabaseDuplicateCommit (line 246) | func (suite *DatabaseSuite) TestDatabaseDuplicateCommit() {
method TestDatabaseCommitMerge (line 260) | func (suite *DatabaseSuite) TestDatabaseCommitMerge() {
method TestDatabaseDelete (line 302) | func (suite *DatabaseSuite) TestDatabaseDelete() {
method TestCommitWithConcurrentChunkStoreUse (line 348) | func (suite *DatabaseSuite) TestCommitWithConcurrentChunkStoreUse() {
method TestDeleteWithConcurrentChunkStoreUse (line 395) | func (suite *DatabaseSuite) TestDeleteWithConcurrentChunkStoreUse() {
method TestSetHead (line 439) | func (suite *DatabaseSuite) TestSetHead() {
method TestFastForward (line 465) | func (suite *DatabaseSuite) TestFastForward() {
method TestDatabaseHeightOfRefs (line 503) | func (suite *DatabaseSuite) TestDatabaseHeightOfRefs() {
method TestDatabaseHeightOfCollections (line 512) | func (suite *DatabaseSuite) TestDatabaseHeightOfCollections() {
method TestMetaOption (line 559) | func (suite *DatabaseSuite) TestMetaOption() {
type LocalDatabaseSuite (line 44) | type LocalDatabaseSuite struct
method SetupTest (line 48) | func (suite *LocalDatabaseSuite) SetupTest() {
type RemoteDatabaseSuite (line 54) | type RemoteDatabaseSuite struct
method SetupTest (line 58) | func (suite *RemoteDatabaseSuite) SetupTest() {
method TestWriteRefToNonexistentValue (line 70) | func (suite *RemoteDatabaseSuite) TestWriteRefToNonexistentValue() {
function newOpts (line 242) | func newOpts(vrw types.ValueReadWriter, parents ...types.Value) CommitOp...
function newOptsWithMerge (line 298) | func newOptsWithMerge(vrw types.ValueReadWriter, policy merge.ResolveFun...
type waitDuringUpdateRootChunkStore (line 336) | type waitDuringUpdateRootChunkStore struct
method Commit (line 341) | func (w *waitDuringUpdateRootChunkStore) Commit(current, last hash.Has...
FILE: go/datas/dataset.go
type Dataset (line 23) | type Dataset struct
method Database (line 36) | func (ds Dataset) Database() Database {
method ID (line 41) | func (ds Dataset) ID() string {
method MaybeHead (line 48) | func (ds Dataset) MaybeHead() (types.Struct, bool) {
method Head (line 57) | func (ds Dataset) Head() types.Struct {
method MaybeHeadRef (line 68) | func (ds Dataset) MaybeHeadRef() (types.Ref, bool) {
method HasHead (line 76) | func (ds Dataset) HasHead() bool {
method HeadRef (line 82) | func (ds Dataset) HeadRef() types.Ref {
method MaybeHeadValue (line 92) | func (ds Dataset) MaybeHeadValue() (types.Value, bool) {
method HeadValue (line 100) | func (ds Dataset) HeadValue() types.Value {
function newDataset (line 29) | func newDataset(db Database, id string, head types.Value) Dataset {
function IsValidDatasetName (line 105) | func IsValidDatasetName(name string) bool {
FILE: go/datas/dataset_test.go
function TestExplicitBranchUsingDatasets (line 15) | func TestExplicitBranchUsingDatasets(t *testing.T) {
function TestTwoClientsWithEmptyDataset (line 64) | func TestTwoClientsWithEmptyDataset(t *testing.T) {
function TestTwoClientsWithNonEmptyDataset (line 93) | func TestTwoClientsWithNonEmptyDataset(t *testing.T) {
function TestIdValidation (line 132) | func TestIdValidation(t *testing.T) {
function TestHeadValueFunctions (line 145) | func TestHeadValueFunctions(t *testing.T) {
function TestIsValidDatasetName (line 179) | func TestIsValidDatasetName(t *testing.T) {
FILE: go/datas/http_chunk_store.go
constant httpChunkStoreConcurrency (line 31) | httpChunkStoreConcurrency = 6
constant readThreshold (line 32) | readThreshold = 1 << 12
type httpChunkStore (line 42) | type httpChunkStore struct
method Version (line 94) | func (hcs *httpChunkStore) Version() string {
method Close (line 98) | func (hcs *httpChunkStore) Close() (e error) {
method Stats (line 115) | func (hcs *httpChunkStore) Stats() interface{} {
method StatsSummary (line 127) | func (hcs *httpChunkStore) StatsSummary() string {
method Get (line 142) | func (hcs *httpChunkStore) Get(h hash.Hash) chunks.Chunk {
method GetMany (line 164) | func (hcs *httpChunkStore) GetMany(hashes hash.HashSet, foundChunks ch...
method batchGetRequests (line 194) | func (hcs *httpChunkStore) batchGetRequests() {
method Has (line 198) | func (hcs *httpChunkStore) Has(h hash.Hash) bool {
method HasMany (line 219) | func (hcs *httpChunkStore) HasMany(hashes hash.HashSet) (absent hash.H...
method batchHasRequests (line 247) | func (hcs *httpChunkStore) batchHasRequests() {
method batchReadRequests (line 253) | func (hcs *httpChunkStore) batchReadRequests(queue <-chan chunks.ReadR...
method sendReadRequests (line 268) | func (hcs *httpChunkStore) sendReadRequests(req chunks.ReadRequest, qu...
method getRefs (line 296) | func (hcs *httpChunkStore) getRefs(batch chunks.ReadBatch) {
method hasRefs (line 334) | func (hcs *httpChunkStore) hasRefs(batch chunks.ReadBatch) {
method Put (line 377) | func (hcs *httpChunkStore) Put(c chunks.Chunk) {
method Root (line 420) | func (hcs *httpChunkStore) Root() hash.Hash {
method Rebase (line 426) | func (hcs *httpChunkStore) Rebase() {
method getRoot (line 433) | func (hcs *httpChunkStore) getRoot(checkVers bool) (root hash.Hash, ve...
method Commit (line 448) | func (hcs *httpChunkStore) Commit(current, last hash.Hash) bool {
method requestRoot (line 498) | func (hcs *httpChunkStore) requestRoot(method string, current, last ha...
function NewHTTPChunkStore (line 60) | func NewHTTPChunkStore(baseURL, auth string) chunks.ChunkStore {
function newHTTPChunkStoreWithClient (line 65) | func newHTTPChunkStoreWithClient(baseURL, auth string, client httpDoer) ...
type httpDoer (line 90) | type httpDoer interface
function checkStatus (line 119) | func checkStatus(status int, res *http.Response, body io.Reader) {
type batchGetter (line 251) | type batchGetter
function resBodyReader (line 364) | func resBodyReader(res *http.Response) (reader io.ReadCloser) {
function sendWriteRequest (line 388) | func sendWriteRequest(u url.URL, auth, vers string, p *nbs.NomsBlockCach...
function newRequest (line 516) | func newRequest(method, auth, url string, body io.Reader, header http.He...
function expectVersion (line 531) | func expectVersion(expected string, res *http.Response) {
function closeResponse (line 546) | func closeResponse(rc io.ReadCloser) error {
FILE: go/datas/http_chunk_store_test.go
constant testAuthToken (line 24) | testAuthToken = "aToken123"
function TestHTTPChunkStore (line 26) | func TestHTTPChunkStore(t *testing.T) {
type HTTPChunkStoreSuite (line 30) | type HTTPChunkStoreSuite struct
method SetupTest (line 52) | func (suite *HTTPChunkStoreSuite) SetupTest() {
method TearDownTest (line 150) | func (suite *HTTPChunkStoreSuite) TearDownTest() {
method TestPutChunk (line 155) | func (suite *HTTPChunkStoreSuite) TestPutChunk() {
method TestPutChunksInOrder (line 164) | func (suite *HTTPChunkStoreSuite) TestPutChunksInOrder() {
method TestStats (line 182) | func (suite *HTTPChunkStoreSuite) TestStats() {
method TestRebase (line 191) | func (suite *HTTPChunkStoreSuite) TestRebase() {
method TestRoot (line 204) | func (suite *HTTPChunkStoreSuite) TestRoot() {
method TestVersionMismatch (line 213) | func (suite *HTTPChunkStoreSuite) TestVersionMismatch() {
method TestCommit (line 222) | func (suite *HTTPChunkStoreSuite) TestCommit() {
method TestEmptyHashCommit (line 231) | func (suite *HTTPChunkStoreSuite) TestEmptyHashCommit() {
method TestCommitWithParams (line 236) | func (suite *HTTPChunkStoreSuite) TestCommitWithParams() {
method TestGet (line 247) | func (suite *HTTPChunkStoreSuite) TestGet() {
method TestGetMany (line 261) | func (suite *HTTPChunkStoreSuite) TestGetMany() {
method TestOverGetThreshold_Issue3589 (line 283) | func (suite *HTTPChunkStoreSuite) TestOverGetThreshold_Issue3589() {
method TestGetManyAllCached (line 312) | func (suite *HTTPChunkStoreSuite) TestGetManyAllCached() {
method TestGetManySomeCached (line 331) | func (suite *HTTPChunkStoreSuite) TestGetManySomeCached() {
method TestGetSame (line 353) | func (suite *HTTPChunkStoreSuite) TestGetSame() {
method TestGetWithRoot (line 367) | func (suite *HTTPChunkStoreSuite) TestGetWithRoot() {
method TestHas (line 400) | func (suite *HTTPChunkStoreSuite) TestHas() {
method TestHasMany (line 412) | func (suite *HTTPChunkStoreSuite) TestHasMany() {
method TestHasManyAllCached (line 433) | func (suite *HTTPChunkStoreSuite) TestHasManyAllCached() {
method TestHasManySomeCached (line 452) | func (suite *HTTPChunkStoreSuite) TestHasManySomeCached() {
type inlineServer (line 36) | type inlineServer struct
method Do (line 40) | func (serv inlineServer) Do(req *http.Request) (resp *http.Response, e...
function newHTTPChunkStoreForTest (line 58) | func newHTTPChunkStoreForTest(cs chunks.ChunkStore) *httpChunkStore {
function newAuthenticatingHTTPChunkStoreForTest (line 106) | func newAuthenticatingHTTPChunkStoreForTest(assert *assert.Assertions, c...
function newBadVersionHTTPChunkStoreForTest (line 130) | func newBadVersionHTTPChunkStoreForTest(cs chunks.ChunkStore) *httpChunk...
FILE: go/datas/pull.go
type PullProgress (line 18) | type PullProgress struct
constant bytesWrittenSampleRate (line 23) | bytesWrittenSampleRate = .10
constant batchSize (line 24) | batchSize = 1 << 12
function Pull (line 28) | func Pull(srcDB, sinkDB Database, sourceRef types.Ref, progressCh chan P...
FILE: go/datas/pull_test.go
constant datasetID (line 15) | datasetID = "ds1"
function TestLocalToLocalPulls (line 17) | func TestLocalToLocalPulls(t *testing.T) {
function TestRemoteToLocalPulls (line 21) | func TestRemoteToLocalPulls(t *testing.T) {
function TestLocalToRemotePulls (line 25) | func TestLocalToRemotePulls(t *testing.T) {
function TestRemoteToRemotePulls (line 29) | func TestRemoteToRemotePulls(t *testing.T) {
type PullSuite (line 33) | type PullSuite struct
method TearDownTest (line 93) | func (suite *PullSuite) TearDownTest() {
method TestPullEverything (line 148) | func (suite *PullSuite) TestPullEverything() {
method TestPullMultiGeneration (line 183) | func (suite *PullSuite) TestPullMultiGeneration() {
method TestPullDivergentHistory (line 229) | func (suite *PullSuite) TestPullDivergentHistory() {
method TestPullUpdates (line 269) | func (suite *PullSuite) TestPullUpdates() {
method commitToSource (line 295) | func (suite *PullSuite) commitToSource(v types.Value, p types.Set) typ...
method commitToSink (line 302) | func (suite *PullSuite) commitToSink(v types.Value, p types.Set) types...
function makeTestStoreViews (line 42) | func makeTestStoreViews() (ts1, ts2 *chunks.TestStoreView) {
type LocalToLocalSuite (line 47) | type LocalToLocalSuite struct
method SetupTest (line 51) | func (suite *LocalToLocalSuite) SetupTest() {
type RemoteToLocalSuite (line 57) | type RemoteToLocalSuite struct
method SetupTest (line 61) | func (suite *RemoteToLocalSuite) SetupTest() {
type LocalToRemoteSuite (line 67) | type LocalToRemoteSuite struct
method SetupTest (line 71) | func (suite *LocalToRemoteSuite) SetupTest() {
type RemoteToRemoteSuite (line 78) | type RemoteToRemoteSuite struct
method SetupTest (line 82) | func (suite *RemoteToRemoteSuite) SetupTest() {
function makeRemoteDb (line 89) | func makeRemoteDb(cs chunks.ChunkStore) Database {
type progressTracker (line 100) | type progressTracker struct
method Validate (line 117) | func (pt *progressTracker) Validate(suite *PullSuite) {
function startProgressTracker (line 105) | func startProgressTracker() *progressTracker {
function buildListOfHeight (line 309) | func buildListOfHeight(height int, vrw types.ValueReadWriter) types.List {
FILE: go/datas/remote_database_handlers.go
type URLParams (line 31) | type URLParams interface
type Handler (line 35) | type Handler
constant NomsVersionHeader (line 40) | NomsVersionHeader = "x-noms-vers"
constant nomsBaseHTML (line 41) | nomsBaseHTML = "<html><head></head><body><p>Hi. This is a Noms HTTP...
constant maxGetBatchSize (line 42) | maxGetBatchSize = 1 << 14
function createHandler (line 99) | func createHandler(hndlr Handler, versionCheck bool) Handler {
function handleWriteValue (line 123) | func handleWriteValue(w http.ResponseWriter, req *http.Request, ps URLPa...
function buildWriteValueRequest (line 201) | func buildWriteValueRequest(chunkChan chan *chunks.Chunk) io.ReadCloser {
function checkClose (line 216) | func checkClose(c io.Closer) {
function bodyReader (line 220) | func bodyReader(req *http.Request) (reader io.ReadCloser) {
function respWriter (line 233) | func respWriter(req *http.Request, w http.ResponseWriter) (writer io.Wri...
type wc (line 247) | type wc struct
method Close (line 251) | func (wc wc) Close() error {
function persistChunks (line 255) | func persistChunks(cs chunks.ChunkStore) {
function handleGetRefs (line 260) | func handleGetRefs(w http.ResponseWriter, req *http.Request, ps URLParam...
function handleGetBlob (line 301) | func handleGetBlob(w http.ResponseWriter, req *http.Request, ps URLParam...
function extractHashes (line 326) | func extractHashes(req *http.Request) hash.HashSlice {
function BuildHashesRequestForTest (line 333) | func BuildHashesRequestForTest(hashes hash.HashSet) io.ReadCloser {
function buildHashesRequest (line 341) | func buildHashesRequest(batch chunks.ReadBatch) io.ReadCloser {
function handleHasRefs (line 350) | func handleHasRefs(w http.ResponseWriter, req *http.Request, ps URLParam...
function handleRootGet (line 367) | func handleRootGet(w http.ResponseWriter, req *http.Request, ps URLParam...
function handleStats (line 375) | func handleStats(w http.ResponseWriter, req *http.Request, ps URLParams,...
function handleRootPost (line 383) | func handleRootPost(w http.ResponseWriter, req *http.Request, ps URLPara...
function validateLast (line 449) | func validateLast(last hash.Hash, vrw types.ValueReadWriter) types.Map {
function validateProposed (line 460) | func validateProposed(proposed, last hash.Hash, vrw types.ValueReadWrite...
function assertMapOfStringToRefOfCommit (line 478) | func assertMapOfStringToRefOfCommit(proposed, datasets types.Map, vr typ...
function mergeDatasetMaps (line 503) | func mergeDatasetMaps(a, b, parent types.Map, vrw types.ValueReadWriter)...
function handleGraphQL (line 576) | func handleGraphQL(w http.ResponseWriter, req *http.Request, ps URLParam...
function handleBaseGet (line 635) | func handleBaseGet(w http.ResponseWriter, req *http.Request, ps URLParam...
FILE: go/datas/remote_database_handlers_test.go
function TestHandleWriteValue (line 26) | func TestHandleWriteValue(t *testing.T) {
function TestHandleWriteValuePanic (line 61) | func TestHandleWriteValuePanic(t *testing.T) {
function TestHandleWriteValueDupChunks (line 74) | func TestHandleWriteValueDupChunks(t *testing.T) {
function TestBuildWriteValueRequest (line 101) | func TestBuildWriteValueRequest(t *testing.T) {
function serializeChunks (line 128) | func serializeChunks(chnx []chunks.Chunk, assert *assert.Assertions) io....
function TestBuildHashesRequest (line 138) | func TestBuildHashesRequest(t *testing.T) {
function TestHandleGetRefs (line 154) | func TestHandleGetRefs(t *testing.T) {
function TestHandleGetBlob (line 198) | func TestHandleGetBlob(t *testing.T) {
function TestHandleHasRefs (line 261) | func TestHandleHasRefs(t *testing.T) {
function TestHandleGetRoot (line 306) | func TestHandleGetRoot(t *testing.T) {
function TestHandleGetBase (line 323) | func TestHandleGetBase(t *testing.T) {
function TestHandlePostRoot (line 335) | func TestHandlePostRoot(t *testing.T) {
function buildPostRootURL (line 376) | func buildPostRootURL(current, last hash.Hash) string {
function buildTestCommit (line 385) | func buildTestCommit(vrw types.ValueReadWriter, v types.Value, parents ....
function TestRejectPostRoot (line 389) | func TestRejectPostRoot(t *testing.T) {
type params (line 426) | type params
method ByName (line 428) | func (p params) ByName(k string) string {
FILE: go/datas/serialize_hashes.go
function serializedLength (line 16) | func serializedLength(batch chunks.ReadBatch) uint32 {
function serializeHashes (line 20) | func serializeHashes(w io.Writer, batch chunks.ReadBatch) {
function serializeHash (line 28) | func serializeHash(w io.Writer, h hash.Hash) {
function deserializeHashes (line 33) | func deserializeHashes(reader io.Reader) hash.HashSlice {
function deserializeHash (line 45) | func deserializeHash(reader io.Reader) hash.Hash {
FILE: go/datas/serialize_hashes_test.go
function TestHashRoundTrip (line 16) | func TestHashRoundTrip(t *testing.T) {
FILE: go/diff/apply_patch.go
function Apply (line 28) | func Apply(root types.Value, patch Patch) types.Value {
function commonPrefixCount (line 199) | func commonPrefixCount(p1, p2 types.Path) int {
type stackElem (line 215) | type stackElem struct
method newestValue (line 228) | func (se stackElem) newestValue() types.Value {
type patchStack (line 235) | type patchStack struct
method updateNode (line 109) | func (stack *patchStack) updateNode(top *stackElem, parent types.Value...
method push (line 242) | func (stack *patchStack) push(p types.Path, pp types.PathPart, changeT...
method top (line 246) | func (stack *patchStack) top() *stackElem {
method pop (line 252) | func (stack *patchStack) pop() stackElem {
method Len (line 263) | func (stack *patchStack) Len() int {
method adjustIndexOffset (line 277) | func (stack *patchStack) adjustIndexOffset(p types.Path, changeType ty...
FILE: go/diff/apply_patch_test.go
function TestCommonPrefixCount (line 17) | func TestCommonPrefixCount(t *testing.T) {
type testFunc (line 50) | type testFunc
type testKey (line 51) | type testKey struct
function vfk (line 59) | func vfk(keys ...string) []types.Value {
function testValues (line 67) | func testValues(vrw types.ValueReadWriter) map[string]types.Value {
function newTestValueStore (line 117) | func newTestValueStore() *types.ValueStore {
function getPatch (line 122) | func getPatch(g1, g2 types.Value) Patch {
function checkApplyPatch (line 137) | func checkApplyPatch(assert *assert.Assertions, g1, expectedG2 types.Val...
function TestPatches (line 143) | func TestPatches(t *testing.T) {
function TestNestedLists (line 160) | func TestNestedLists(t *testing.T) {
function TestUpdateNode (line 176) | func TestUpdateNode(t *testing.T) {
function checkApplyDiffs (line 236) | func checkApplyDiffs(a *assert.Assertions, n1, n2 types.Value, leftRight...
function tryApplyDiff (line 253) | func tryApplyDiff(a *assert.Assertions, a1, a2 interface{}) {
function TestUpdateList (line 263) | func TestUpdateList(t *testing.T) {
function TestUpdateMap (line 307) | func TestUpdateMap(t *testing.T) {
function TestUpdateStruct (line 321) | func TestUpdateStruct(t *testing.T) {
function TestUpdateSet (line 347) | func TestUpdateSet(t *testing.T) {
function mustMarshal (line 362) | func mustMarshal(v interface{}) types.Value {
FILE: go/diff/diff.go
type diffFunc (line 12) | type diffFunc
type pathPartFunc (line 13) | type pathPartFunc
type valueFunc (line 14) | type valueFunc
type Difference (line 18) | type Difference struct
method IsEmpty (line 32) | func (dif Difference) IsEmpty() bool {
type differ (line 37) | type differ struct
method diff (line 88) | func (d differ) diff(p types.Path, v1, v2 types.Value) bool {
method diffLists (line 103) | func (d differ) diffLists(p types.Path, v1, v2 types.List) (stop bool) {
method diffMaps (line 155) | func (d differ) diffMaps(p types.Path, v1, v2 types.Map) bool {
method diffStructs (line 177) | func (d differ) diffStructs(p types.Path, v1, v2 types.Struct) bool {
method diffSets (line 192) | func (d differ) diffSets(p types.Path, v1, v2 types.Set) bool {
method diffOrdered (line 213) | func (d differ) diffOrdered(p types.Path, ppf pathPartFunc, df diffFun...
method sendDiff (line 266) | func (d differ) sendDiff(dif Difference) bool {
function Diff (line 77) | func Diff(v1, v2 types.Value, dChan chan<- Difference, stopChan chan str...
function shouldDescend (line 260) | func shouldDescend(v1, v2 types.Value) bool {
FILE: go/diff/diff_test.go
function valToTypesValue (line 30) | func valToTypesValue(v interface{}) types.Value {
function valsToTypesValues (line 43) | func valsToTypesValues(kv ...interface{}) []types.Value {
function createMap (line 52) | func createMap(kv ...interface{}) types.Map {
function createSet (line 59) | func createSet(kv ...interface{}) types.Set {
function createList (line 66) | func createList(kv ...interface{}) types.List {
function createStruct (line 73) | func createStruct(name string, kv ...interface{}) types.Struct {
function pathsFromDiff (line 81) | func pathsFromDiff(v1, v2 types.Value, leftRight bool) []string {
function mustParsePath (line 97) | func mustParsePath(assert *assert.Assertions, s string) types.Path {
function TestNomsDiffPrintMap (line 106) | func TestNomsDiffPrintMap(t *testing.T) {
function TestNomsDiffPrintSet (line 137) | func TestNomsDiffPrintSet(t *testing.T) {
function TestNomsDiffPrintStop (line 206) | func TestNomsDiffPrintStop(t *testing.T) {
function TestNomsDiffPrintStruct (line 238) | func TestNomsDiffPrintStruct(t *testing.T) {
function TestNomsDiffPrintMapWithStructKeys (line 314) | func TestNomsDiffPrintMapWithStructKeys(t *testing.T) {
function TestNomsDiffPrintList (line 345) | func TestNomsDiffPrintList(t *testing.T) {
function TestNomsDiffPrintBlob (line 417) | func TestNomsDiffPrintBlob(t *testing.T) {
function TestNomsDiffPrintType (line 441) | func TestNomsDiffPrintType(t *testing.T) {
function TestNomsDiffPrintRef (line 474) | func TestNomsDiffPrintRef(t *testing.T) {
FILE: go/diff/patch.go
type Patch (line 16) | type Patch
method Swap (line 18) | func (r Patch) Swap(i, j int) {
method Len (line 22) | func (r Patch) Len() int {
method Less (line 28) | func (r Patch) Less(i, j int) bool {
function pathIsLess (line 37) | func pathIsLess(p1, p2 types.Path) bool {
function fieldPathCompare (line 53) | func fieldPathCompare(pp types.FieldPath, o types.PathPart) int {
function indexPathCompare (line 71) | func indexPathCompare(pp types.IndexPath, o types.PathPart) int {
function hashIndexPathCompare (line 95) | func hashIndexPathCompare(pp types.HashIndexPath, o types.PathPart) int {
function pathPartCompare (line 120) | func pathPartCompare(pp, pp2 types.PathPart) int {
FILE: go/diff/patch_test.go
function TestPatchPathPartCompare (line 16) | func TestPatchPathPartCompare(t *testing.T) {
function TestPatchPathIsLess (line 51) | func TestPatchPathIsLess(t *testing.T) {
function TestPatchSort (line 74) | func TestPatchSort(t *testing.T) {
FILE: go/diff/print_diff.go
type prefixOp (line 15) | type prefixOp
constant ADD (line 18) | ADD = "+ "
constant DEL (line 19) | DEL = "- "
type printFunc (line 23) | type printFunc
function PrintDiff (line 29) | func PrintDiff(w io.Writer, v1, v2 types.Value, leftRight bool) (err err...
function writeHeader (line 112) | func writeHeader(w io.Writer, p types.Path, wroteHdr *bool) error {
function writeFooter (line 124) | func writeFooter(w io.Writer, wroteHdr *bool) error {
function line (line 132) | func line(w io.Writer, op prefixOp, key, val types.Value) error {
function field (line 145) | func field(w io.Writer, op prefixOp, name, val types.Value) error {
function writeEncodedValue (line 156) | func writeEncodedValue(w io.Writer, v types.Value) error {
function write (line 165) | func write(w io.Writer, b []byte) error {
FILE: go/diff/summary.go
function Summary (line 17) | func Summary(value1, value2 types.Value) {
type diffSummaryProgress (line 60) | type diffSummaryProgress struct
function diffSummary (line 64) | func diffSummary(ch chan diffSummaryProgress, v1, v2 types.Value) {
function diffSummaryList (line 85) | func diffSummaryList(ch chan<- diffSummaryProgress, v1, v2 types.List) {
function diffSummaryMap (line 105) | func diffSummaryMap(ch chan<- diffSummaryProgress, v1, v2 types.Map) {
function diffSummarySet (line 111) | func diffSummarySet(ch chan<- diffSummaryProgress, v1, v2 types.Set) {
function diffSummaryStructs (line 117) | func diffSummaryStructs(ch chan<- diffSummaryProgress, v1, v2 types.Stru...
function diffSummaryValueChanged (line 126) | func diffSummaryValueChanged(ch chan<- diffSummaryProgress, oldSize, new...
function reportChanges (line 139) | func reportChanges(ch chan<- diffSummaryProgress, changeChan chan types....
function formatStatus (line 154) | func formatStatus(acc diffSummaryProgress, singular, plural string) {
FILE: go/hash/base32.go
function encode (line 11) | func encode(data []byte) string {
function decode (line 15) | func decode(s string) []byte {
FILE: go/hash/base32_test.go
function TestBase32Encode (line 13) | func TestBase32Encode(t *testing.T) {
function TestBase32Decode (line 40) | func TestBase32Decode(t *testing.T) {
FILE: go/hash/hash.go
constant ByteLen (line 43) | ByteLen = 20
constant StringLen (line 46) | StringLen = 32
type Hash (line 55) | type Hash
method IsEmpty (line 58) | func (h Hash) IsEmpty() bool {
method String (line 63) | func (h Hash) String() string {
method Less (line 104) | func (h Hash) Less(other Hash) bool {
method Greater (line 109) | func (h Hash) Greater(other Hash) bool {
function Of (line 68) | func Of(data []byte) Hash {
function New (line 76) | func New(data []byte) Hash {
function MaybeParse (line 85) | func MaybeParse(s string) (Hash, bool) {
function Parse (line 95) | func Parse(s string) Hash {
type HashSet (line 115) | type HashSet
method Insert (line 126) | func (hs HashSet) Insert(hash Hash) {
method Has (line 131) | func (hs HashSet) Has(hash Hash) (has bool) {
method Remove (line 137) | func (hs HashSet) Remove(hash Hash) {
function NewHashSet (line 117) | func NewHashSet(hashes ...Hash) HashSet {
FILE: go/hash/hash_slice.go
type HashSlice (line 7) | type HashSlice
method Len (line 9) | func (rs HashSlice) Len() int {
method Less (line 13) | func (rs HashSlice) Less(i, j int) bool {
method Swap (line 17) | func (rs HashSlice) Swap(i, j int) {
method Equals (line 21) | func (rs HashSlice) Equals(other HashSlice) bool {
method HashSet (line 33) | func (rs HashSlice) HashSet() HashSet {
FILE: go/hash/hash_slice_test.go
function TestHashSliceSort (line 14) | func TestHashSliceSort(t *testing.T) {
FILE: go/hash/hash_test.go
function TestParseError (line 14) | func TestParseError(t *testing.T) {
function TestMaybeParse (line 42) | func TestMaybeParse(t *testing.T) {
function TestEquals (line 63) | func TestEquals(t *testing.T) {
function TestString (line 76) | func TestString(t *testing.T) {
function TestOf (line 82) | func TestOf(t *testing.T) {
function TestIsEmpty (line 87) | func TestIsEmpty(t *testing.T) {
function TestLess (line 98) | func TestLess(t *testing.T) {
function TestGreater (line 115) | func TestGreater(t *testing.T) {
FILE: go/marshal/decode.go
function Unmarshal (line 63) | func Unmarshal(v types.Value, out interface{}) (err error) {
function UnmarshalOpt (line 68) | func UnmarshalOpt(v types.Value, opt Opt, out interface{}) (err error) {
function MustUnmarshal (line 88) | func MustUnmarshal(v types.Value, out interface{}) {
function MustUnmarshalOpt (line 93) | func MustUnmarshalOpt(v types.Value, opt Opt, out interface{}) {
type Unmarshaler (line 114) | type Unmarshaler interface
type InvalidUnmarshalError (line 123) | type InvalidUnmarshalError struct
method Error (line 127) | func (e *InvalidUnmarshalError) Error() string {
type UnmarshalTypeMismatchError (line 140) | type UnmarshalTypeMismatchError struct
method Error (line 146) | func (e *UnmarshalTypeMismatchError) Error() string {
function overflowError (line 156) | func overflowError(v types.Number, t reflect.Type) *UnmarshalTypeMismatc...
type unmarshalNomsError (line 162) | type unmarshalNomsError struct
method Error (line 166) | func (e *unmarshalNomsError) Error() string {
type decoderFunc (line 170) | type decoderFunc
function typeDecoder (line 172) | func typeDecoder(t reflect.Type, tags nomsTags) decoderFunc {
function boolDecoder (line 212) | func boolDecoder(v types.Value, rv reflect.Value) {
function stringDecoder (line 220) | func stringDecoder(v types.Value, rv reflect.Value) {
function floatDecoder (line 228) | func floatDecoder(v types.Value, rv reflect.Value) {
function intDecoder (line 236) | func intDecoder(v types.Value, rv reflect.Value) {
function uintDecoder (line 248) | func uintDecoder(v types.Value, rv reflect.Value) {
type decoderCacheT (line 260) | type decoderCacheT struct
method get (line 271) | func (c *decoderCacheT) get(t reflect.Type) decoderFunc {
method set (line 277) | func (c *decoderCacheT) set(t reflect.Type, d decoderFunc) {
type decField (line 286) | type decField struct
function structDecoderFields (line 294) | func structDecoderFields(t reflect.Type) []decField {
function structDecoder (line 327) | func structDecoder(t reflect.Type) decoderFunc {
function nomsValueDecoder (line 367) | func nomsValueDecoder(v types.Value, rv reflect.Value) {
function marshalerDecoder (line 374) | func marshalerDecoder(t reflect.Type) decoderFunc {
function iterListOrSlice (line 385) | func iterListOrSlice(v types.Value, t reflect.Type, f func(c types.Value...
function sliceDecoder (line 400) | func sliceDecoder(t reflect.Type) decoderFunc {
function arrayDecoder (line 432) | func arrayDecoder(t reflect.Type) decoderFunc {
function mapFromSetDecoder (line 465) | func mapFromSetDecoder(t reflect.Type) decoderFunc {
function mapDecoder (line 501) | func mapDecoder(t reflect.Type, tags nomsTags) decoderFunc {
function interfaceDecoder (line 547) | func interfaceDecoder(t reflect.Type) decoderFunc {
function getGoTypeForNomsType (line 565) | func getGoTypeForNomsType(nt *types.Type, rt reflect.Type, v types.Value...
function shouldMapDecodeFromSet (line 593) | func shouldMapDecodeFromSet(rt reflect.Type, tags nomsTags) bool {
FILE: go/marshal/decode_test.go
function TestDecode (line 23) | func TestDecode(tt *testing.T) {
function TestDecodeStructWithNomsValue (line 212) | func TestDecodeStructWithNomsValue(t *testing.T) {
function TestDecodeNilPointer (line 243) | func TestDecodeNilPointer(t *testing.T) {
function TestDecodeNonPointer (line 248) | func TestDecodeNonPointer(t *testing.T) {
function TestDecodeNil (line 253) | func TestDecodeNil(t *testing.T) {
function newTestValueStore (line 259) | func newTestValueStore() *types.ValueStore {
function TestDecodeTypeMismatch (line 264) | func TestDecodeTypeMismatch(t *testing.T) {
function assertDecodeErrorMessage (line 284) | func assertDecodeErrorMessage(t *testing.T, v types.Value, ptr interface...
function TestDecodeInvalidTypes (line 291) | func TestDecodeInvalidTypes(tt *testing.T) {
function TestDecodeOverflows (line 309) | func TestDecodeOverflows(tt *testing.T) {
function TestDecodeMissingField (line 339) | func TestDecodeMissingField(t *testing.T) {
function TestDecodeEmbeddedStruct (line 350) | func TestDecodeEmbeddedStruct(tt *testing.T) {
function TestDecodeEmbeddedStructSkip (line 379) | func TestDecodeEmbeddedStructSkip(tt *testing.T) {
function TestDecodeEmbeddedStructNamed (line 397) | func TestDecodeEmbeddedStructNamed(tt *testing.T) {
function TestDecodeEmbeddedStructOriginal (line 418) | func TestDecodeEmbeddedStructOriginal(tt *testing.T) {
function TestDecodeNonExportedField (line 443) | func TestDecodeNonExportedField(tt *testing.T) {
function TestDecodeTaggingSkip (line 451) | func TestDecodeTaggingSkip(t *testing.T) {
function TestDecodeNamedFields (line 481) | func TestDecodeNamedFields(t *testing.T) {
function TestDecodeInvalidNamedFields (line 499) | func TestDecodeInvalidNamedFields(t *testing.T) {
function TestDecodeInvalidNomsType (line 509) | func TestDecodeInvalidNomsType(t *testing.T) {
function TestDecodeNomsTypePtr (line 522) | func TestDecodeNomsTypePtr(t *testing.T) {
function ExampleUnmarshal (line 546) | func ExampleUnmarshal() {
function TestDecodeSlice (line 565) | func TestDecodeSlice(t *testing.T) {
function TestDecodeSliceEmpty (line 582) | func TestDecodeSliceEmpty(t *testing.T) {
function TestDecodeSliceReuse (line 608) | func TestDecodeSliceReuse(t *testing.T) {
function TestDecodeArray (line 627) | func TestDecodeArray(t *testing.T) {
function TestDecodeArrayEmpty (line 644) | func TestDecodeArrayEmpty(t *testing.T) {
function TestDecodeStructWithSlice (line 661) | func TestDecodeStructWithSlice(t *testing.T) {
function TestDecodeStructWithArrayOfNomsValue (line 684) | func TestDecodeStructWithArrayOfNomsValue(t *testing.T) {
function TestDecodeWrongArrayLength (line 701) | func TestDecodeWrongArrayLength(t *testing.T) {
function TestDecodeWrongArrayType (line 709) | func TestDecodeWrongArrayType(t *testing.T) {
function TestDecodeWrongSliceType (line 717) | func TestDecodeWrongSliceType(t *testing.T) {
function TestDecodeSliceWrongNomsType (line 725) | func TestDecodeSliceWrongNomsType(t *testing.T) {
function TestDecodeArrayWrongNomsType (line 733) | func TestDecodeArrayWrongNomsType(t *testing.T) {
function TestDecodeRecursive (line 741) | func TestDecodeRecursive(t *testing.T) {
function TestDecodeMap (line 779) | func TestDecodeMap(t *testing.T) {
function TestDecodeMapEmpty (line 818) | func TestDecodeMapEmpty(t *testing.T) {
function TestDecodeMapWrongNomsType (line 835) | func TestDecodeMapWrongNomsType(t *testing.T) {
function TestDecodeOntoInterface (line 843) | func TestDecodeOntoInterface(t *testing.T) {
function TestDecodeOntoNonSupportedInterface (line 879) | func TestDecodeOntoNonSupportedInterface(t *testing.T) {
function TestDecodeOntoInterfaceStruct (line 887) | func TestDecodeOntoInterfaceStruct(t *testing.T) {
function TestDecodeSet (line 893) | func TestDecodeSet(t *testing.T) {
function TestDecodeOpt (line 950) | func TestDecodeOpt(t *testing.T) {
function TestDecodeNamedSet (line 1004) | func TestDecodeNamedSet(t *testing.T) {
function TestDecodeSetWrongMapType (line 1026) | func TestDecodeSetWrongMapType(t *testing.T) {
function TestDecodeOmitEmpty (line 1063) | func TestDecodeOmitEmpty(t *testing.T) {
function TestDecodeOriginal (line 1091) | func TestDecodeOriginal(t *testing.T) {
function TestDecodeOriginalReceiveTypeError (line 1113) | func TestDecodeOriginalReceiveTypeError(t *testing.T) {
function TestDecodeCanSkipUnexportedField (line 1126) | func TestDecodeCanSkipUnexportedField(t *testing.T) {
method UnmarshalNoms (line 1141) | func (u *primitiveType) UnmarshalNoms(v types.Value) error {
function TestUnmarshalerPrimitiveType (line 1146) | func TestUnmarshalerPrimitiveType(t *testing.T) {
method UnmarshalNoms (line 1155) | func (u *primitiveSliceType) UnmarshalNoms(v types.Value) error {
function TestUnmarshalerPrimitiveSliceType (line 1165) | func TestUnmarshalerPrimitiveSliceType(t *testing.T) {
method UnmarshalNoms (line 1174) | func (u *primitiveMapType) UnmarshalNoms(v types.Value) error {
function TestUnmarshalerPrimitiveMapType (line 1185) | func TestUnmarshalerPrimitiveMapType(t *testing.T) {
method UnmarshalNoms (line 1200) | func (u *primitiveStructType) UnmarshalNoms(v types.Value) error {
function TestUnmarshalerPrimitiveStructType (line 1207) | func TestUnmarshalerPrimitiveStructType(t *testing.T) {
method UnmarshalNoms (line 1216) | func (u *builtinType) UnmarshalNoms(v types.Value) error {
function TestUnmarshalerBuiltinType (line 1222) | func TestUnmarshalerBuiltinType(t *testing.T) {
method UnmarshalNoms (line 1233) | func (u *wrappedMarshalerType) UnmarshalNoms(v types.Value) error {
function TestUnmarshalerWrappedMarshalerType (line 1239) | func TestUnmarshalerWrappedMarshalerType(t *testing.T) {
function TestUnmarshalerComplexStructType (line 1248) | func TestUnmarshalerComplexStructType(t *testing.T) {
method UnmarshalNoms (line 1284) | func (u *returnsMarshalerError) UnmarshalNoms(v types.Value) error {
method UnmarshalNoms (line 1290) | func (u panicsMarshaler) UnmarshalNoms(v types.Value) error {
function TestUnmarshalerError (line 1294) | func TestUnmarshalerError(t *testing.T) {
type notPointer (line 1305) | type notPointer struct
method UnmarshalNoms (line 1309) | func (u notPointer) UnmarshalNoms(v types.Value) error {
function TestUnmarshalNomsNotPointerDoesNotShareState (line 1314) | func TestUnmarshalNomsNotPointerDoesNotShareState(t *testing.T) {
function TestUnmarshalMustUnmarshal (line 1324) | func TestUnmarshalMustUnmarshal(t *testing.T) {
FILE: go/marshal/encode.go
function Marshal (line 94) | func Marshal(vrw types.ValueReadWriter, v interface{}) (types.Value, err...
function MarshalOpt (line 99) | func MarshalOpt(vrw types.ValueReadWriter, v interface{}, opt Opt) (noms...
function MustMarshal (line 118) | func MustMarshal(vrw types.ValueReadWriter, v interface{}) types.Value {
function MustMarshalOpt (line 123) | func MustMarshalOpt(vrw types.ValueReadWriter, v interface{}, opt Opt) t...
type Marshaler (line 133) | type Marshaler interface
type StructNameMarshaler (line 142) | type StructNameMarshaler interface
type UnsupportedTypeError (line 148) | type UnsupportedTypeError struct
method Error (line 153) | func (e *UnsupportedTypeError) Error() string {
type InvalidTagError (line 163) | type InvalidTagError struct
method Error (line 167) | func (e *InvalidTagError) Error() string {
type marshalNomsError (line 173) | type marshalNomsError struct
method Error (line 177) | func (e *marshalNomsError) Error() string {
type Opt (line 181) | type Opt struct
type nomsTags (line 186) | type nomsTags struct
type encoderFunc (line 200) | type encoderFunc
function boolEncoder (line 202) | func boolEncoder(v reflect.Value, vrw types.ValueReadWriter) types.Value {
function float64Encoder (line 206) | func float64Encoder(v reflect.Value, vrw types.ValueReadWriter) types.Va...
function intEncoder (line 210) | func intEncoder(v reflect.Value, vrw types.ValueReadWriter) types.Value {
function uintEncoder (line 214) | func uintEncoder(v reflect.Value, vrw types.ValueReadWriter) types.Value {
function stringEncoder (line 218) | func stringEncoder(v reflect.Value, vrw types.ValueReadWriter) types.Val...
function nomsValueEncoder (line 222) | func nomsValueEncoder(v reflect.Value, vrw types.ValueReadWriter) types....
function marshalerEncoder (line 226) | func marshalerEncoder(t reflect.Type) encoderFunc {
function typeEncoder (line 239) | func typeEncoder(t reflect.Type, seenStructs map[string]reflect.Type, ta...
function getStructName (line 284) | func getStructName(t reflect.Type) string {
function structEncoder (line 292) | func structEncoder(t reflect.Type, seenStructs map[string]reflect.Type) ...
function isEmptyValue (line 358) | func isEmptyValue(v reflect.Value) bool {
type field (line 379) | type field struct
type fieldSlice (line 387) | type fieldSlice
method Len (line 389) | func (fs fieldSlice) Len() int { return len(fs) }
method Swap (line 390) | func (fs fieldSlice) Swap(i, j int) { fs[i], fs[j] = fs[j], fs[i] }
method Less (line 391) | func (fs fieldSlice) Less(i, j int) bool { return fs[i].name < fs[j].n...
type encoderCacheT (line 393) | type encoderCacheT struct
method get (line 404) | func (c *encoderCacheT) get(t reflect.Type) encoderFunc {
method set (line 410) | func (c *encoderCacheT) set(t reflect.Type, e encoderFunc) {
function getTags (line 419) | func getTags(f reflect.StructField) (tags nomsTags) {
function validateField (line 455) | func validateField(f reflect.StructField, t reflect.Type) {
function typeFields (line 464) | func typeFields(t reflect.Type, seenStructs map[string]reflect.Type, com...
function listEncoder (line 525) | func listEncoder(t reflect.Type, seenStructs map[string]reflect.Type) en...
function setFromListEncoder (line 552) | func setFromListEncoder(t reflect.Type, seenStructs map[string]reflect.T...
function setEncoder (line 578) | func setEncoder(t reflect.Type, seenStructs map[string]reflect.Type) enc...
function mapEncoder (line 604) | func mapEncoder(t reflect.Type, seenStructs map[string]reflect.Type) enc...
function shouldEncodeAsSet (line 634) | func shouldEncodeAsSet(t reflect.Type, tags nomsTags) bool {
FILE: go/marshal/encode_test.go
function TestEncode (line 20) | func TestEncode(tt *testing.T) {
function assertEncodeErrorMessage (line 158) | func assertEncodeErrorMessage(t *testing.T, v interface{}, expectedMessa...
function TestInvalidTypes (line 167) | func TestInvalidTypes(t *testing.T) {
function TestEncodeEmbeddedStructSkip (line 173) | func TestEncodeEmbeddedStructSkip(t *testing.T) {
function TestEncodeEmbeddedStructWithName (line 194) | func TestEncodeEmbeddedStructWithName(t *testing.T) {
function TestEncodeEmbeddedStruct (line 218) | func TestEncodeEmbeddedStruct(t *testing.T) {
function TestEncodeEmbeddedStructOriginal (line 252) | func TestEncodeEmbeddedStructOriginal(t *testing.T) {
function TestEncodeNonExportedField (line 280) | func TestEncodeNonExportedField(t *testing.T) {
function TestEncodeTaggingSkip (line 287) | func TestEncodeTaggingSkip(t *testing.T) {
function TestEncodeNamedFields (line 305) | func TestEncodeNamedFields(t *testing.T) {
function TestEncodeInvalidNamedFields (line 326) | func TestEncodeInvalidNamedFields(t *testing.T) {
function TestEncodeOmitEmpty (line 333) | func TestEncodeOmitEmpty(t *testing.T) {
function ExampleMarshal (line 486) | func ExampleMarshal() {
function TestEncodeSlice (line 504) | func TestEncodeSlice(t *testing.T) {
function TestEncodeArray (line 515) | func TestEncodeArray(t *testing.T) {
function TestEncodeStructWithSlice (line 526) | func TestEncodeStructWithSlice(t *testing.T) {
function TestEncodeStructWithArrayOfNomsValue (line 542) | func TestEncodeStructWithArrayOfNomsValue(t *testing.T) {
function TestEncodeNomsTypePtr (line 558) | func TestEncodeNomsTypePtr(t *testing.T) {
function TestEncodeRecursive (line 586) | func TestEncodeRecursive(t *testing.T) {
function TestEncodeMap (line 632) | func TestEncodeMap(t *testing.T) {
function TestEncodeInterface (line 665) | func TestEncodeInterface(t *testing.T) {
function TestEncodeSet (line 687) | func TestEncodeSet(t *testing.T) {
function TestEncodeOpt (line 759) | func TestEncodeOpt(t *testing.T) {
function TestEncodeSetWithTags (line 799) | func TestEncodeSetWithTags(t *testing.T) {
function TestInvalidTag (line 833) | func TestInvalidTag(t *testing.T) {
function TestEncodeCanSkipUnexportedField (line 844) | func TestEncodeCanSkipUnexportedField(t *testing.T) {
function TestEncodeOriginal (line 862) | func TestEncodeOriginal(t *testing.T) {
function TestNomsTypes (line 923) | func TestNomsTypes(t *testing.T) {
type primitiveType (line 954) | type primitiveType
method MarshalNoms (line 956) | func (t primitiveType) MarshalNoms(vrw types.ValueReadWriter) (types.V...
function TestMarshalerPrimitiveType (line 960) | func TestMarshalerPrimitiveType(t *testing.T) {
type primitiveSliceType (line 971) | type primitiveSliceType
method MarshalNoms (line 973) | func (u primitiveSliceType) MarshalNoms(vrw types.ValueReadWriter) (ty...
function TestMarshalerPrimitiveSliceType (line 977) | func TestMarshalerPrimitiveSliceType(t *testing.T) {
type primitiveMapType (line 988) | type primitiveMapType
method MarshalNoms (line 990) | func (u primitiveMapType) MarshalNoms(vrw types.ValueReadWriter) (type...
function TestMarshalerPrimitiveMapType (line 998) | func TestMarshalerPrimitiveMapType(t *testing.T) {
type primitiveStructType (line 1012) | type primitiveStructType struct
method MarshalNoms (line 1016) | func (u primitiveStructType) MarshalNoms(vrw types.ValueReadWriter) (t...
function TestMarshalerPrimitiveStructType (line 1020) | func TestMarshalerPrimitiveStructType(t *testing.T) {
type builtinType (line 1031) | type builtinType
method MarshalNoms (line 1033) | func (u builtinType) MarshalNoms(vrw types.ValueReadWriter) (types.Val...
function TestMarshalerBuiltinType (line 1038) | func TestMarshalerBuiltinType(t *testing.T) {
type wrappedMarshalerType (line 1051) | type wrappedMarshalerType
method MarshalNoms (line 1053) | func (u wrappedMarshalerType) MarshalNoms(vrw types.ValueReadWriter) (...
function TestMarshalerWrapperMarshalerType (line 1057) | func TestMarshalerWrapperMarshalerType(t *testing.T) {
type TestComplexStructType (line 1068) | type TestComplexStructType struct
function TestMarshalerComplexStructType (line 1078) | func TestMarshalerComplexStructType(t *testing.T) {
type returnsMarshalerError (line 1115) | type returnsMarshalerError struct
method MarshalNoms (line 1119) | func (u returnsMarshalerError) MarshalNoms(vrw types.ValueReadWriter) ...
type returnsMarshalerNil (line 1123) | type returnsMarshalerNil struct
method MarshalNoms (line 1125) | func (u returnsMarshalerNil) MarshalNoms(vrw types.ValueReadWriter) (t...
type panicsMarshaler (line 1129) | type panicsMarshaler struct
method MarshalNoms (line 1131) | func (u panicsMarshaler) MarshalNoms(vrw types.ValueReadWriter) (types...
function TestMarshalerErrors (line 1135) | func TestMarshalerErrors(t *testing.T) {
type TestStructWithNameImpl (line 1153) | type TestStructWithNameImpl struct
method MarshalNomsStructName (line 1157) | func (ts TestStructWithNameImpl) MarshalNomsStructName() string {
function TestMarshalStructName (line 1160) | func TestMarshalStructName(t *testing.T) {
type TestStructWithNameImpl2 (line 1175) | type TestStructWithNameImpl2 struct
method MarshalNomsStructName (line 1179) | func (ts TestStructWithNameImpl2) MarshalNomsStructName() string {
function TestMarshalStructName2 (line 1182) | func TestMarshalStructName2(t *testing.T) {
FILE: go/marshal/encode_type.go
function MarshalType (line 25) | func MarshalType(v interface{}) (nt *types.Type, err error) {
function MarshalTypeOpt (line 30) | func MarshalTypeOpt(v interface{}, opt Opt) (nt *types.Type, err error) {
function MustMarshalType (line 49) | func MustMarshalType(v interface{}) (nt *types.Type) {
function MustMarshalTypeOpt (line 54) | func MustMarshalTypeOpt(v interface{}, opt Opt) (nt *types.Type) {
type TypeMarshaler (line 70) | type TypeMarshaler interface
function encodeType (line 80) | func encodeType(t reflect.Type, seenStructs map[string]reflect.Type, tag...
function structEncodeType (line 175) | func structEncodeType(t reflect.Type, seenStructs map[string]reflect.Typ...
FILE: go/marshal/encode_type_test.go
function TestMarshalTypeType (line 18) | func TestMarshalTypeType(tt *testing.T) {
function assertMarshalTypeErrorMessage (line 107) | func assertMarshalTypeErrorMessage(t *testing.T, v interface{}, expected...
function TestMarshalTypeInvalidTypes (line 113) | func TestMarshalTypeInvalidTypes(t *testing.T) {
function TestMarshalTypeEmbeddedStruct (line 117) | func TestMarshalTypeEmbeddedStruct(t *testing.T) {
function TestMarshalTypeEmbeddedStructSkip (line 137) | func TestMarshalTypeEmbeddedStructSkip(t *testing.T) {
function TestMarshalTypeEmbeddedStructNamed (line 156) | func TestMarshalTypeEmbeddedStructNamed(t *testing.T) {
function TestMarshalTypeEncodeNonExportedField (line 178) | func TestMarshalTypeEncodeNonExportedField(t *testing.T) {
function TestMarshalTypeEncodeTaggingSkip (line 185) | func TestMarshalTypeEncodeTaggingSkip(t *testing.T) {
function TestMarshalTypeNamedFields (line 200) | func TestMarshalTypeNamedFields(t *testing.T) {
function TestMarshalTypeInvalidNamedFields (line 218) | func TestMarshalTypeInvalidNamedFields(t *testing.T) {
function TestMarshalTypeOmitEmpty (line 226) | func TestMarshalTypeOmitEmpty(t *testing.T) {
function ExampleMarshalType (line 238) | func ExampleMarshalType() {
function TestMarshalTypeSlice (line 258) | func TestMarshalTypeSlice(t *testing.T) {
function TestMarshalTypeArray (line 267) | func TestMarshalTypeArray(t *testing.T) {
function TestMarshalTypeStructWithSlice (line 276) | func TestMarshalTypeStructWithSlice(t *testing.T) {
function TestMarshalTypeRecursive (line 290) | func TestMarshalTypeRecursive(t *testing.T) {
function TestMarshalTypeMap (line 314) | func TestMarshalTypeMap(t *testing.T) {
function TestMarshalTypeSet (line 336) | func TestMarshalTypeSet(t *testing.T) {
function TestEncodeTypeOpt (line 367) | func TestEncodeTypeOpt(t *testing.T) {
function TestMarshalTypeSetWithTags (line 404) | func TestMarshalTypeSetWithTags(t *testing.T) {
function TestMarshalTypeInvalidTag (line 423) | func TestMarshalTypeInvalidTag(t *testing.T) {
function TestMarshalTypeCanSkipUnexportedField (line 434) | func TestMarshalTypeCanSkipUnexportedField(t *testing.T) {
function TestMarshalTypeOriginal (line 449) | func TestMarshalTypeOriginal(t *testing.T) {
function TestMarshalTypeNomsTypes (line 465) | func TestMarshalTypeNomsTypes(t *testing.T) {
method MarshalNomsType (line 487) | func (t primitiveType) MarshalNomsType() (*types.Type, error) {
function TestTypeMarshalerPrimitiveType (line 491) | func TestTypeMarshalerPrimitiveType(t *testing.T) {
method MarshalNomsType (line 499) | func (u primitiveSliceType) MarshalNomsType() (*types.Type, error) {
function TestTypeMarshalerPrimitiveSliceType (line 503) | func TestTypeMarshalerPrimitiveSliceType(t *testing.T) {
method MarshalNomsType (line 511) | func (u primitiveMapType) MarshalNomsType() (*types.Type, error) {
function TestTypeMarshalerPrimitiveMapType (line 515) | func TestTypeMarshalerPrimitiveMapType(t *testing.T) {
function TestTypeMarshalerPrimitiveStructTypeNoMarshalNomsType (line 523) | func TestTypeMarshalerPrimitiveStructTypeNoMarshalNomsType(t *testing.T) {
method MarshalNomsType (line 532) | func (u builtinType) MarshalNomsType() (*types.Type, error) {
function TestTypeMarshalerBuiltinType (line 536) | func TestTypeMarshalerBuiltinType(t *testing.T) {
method MarshalNomsType (line 544) | func (u wrappedMarshalerType) MarshalNomsType() (*types.Type, error) {
function TestTypeMarshalerWrapperMarshalerType (line 548) | func TestTypeMarshalerWrapperMarshalerType(t *testing.T) {
method MarshalNomsType (line 556) | func (u returnsMarshalerError) MarshalNomsType() (*types.Type, error) {
method MarshalNomsType (line 560) | func (u returnsMarshalerNil) MarshalNomsType() (*types.Type, error) {
method MarshalNomsType (line 564) | func (u panicsMarshaler) MarshalNomsType() (*types.Type, error) {
function TestTypeMarshalerErrors (line 568) | func TestTypeMarshalerErrors(t *testing.T) {
function TestMarshalTypeStructName (line 583) | func TestMarshalTypeStructName(t *testing.T) {
function TestMarshalTypeStructName2 (line 591) | func TestMarshalTypeStructName2(t *testing.T) {
type OutPhoto (line 599) | type OutPhoto struct
type OutFace (line 604) | type OutFace struct
method MarshalNomsStructName (line 608) | func (f OutFace) MarshalNomsStructName() string {
function TestMarshalTypeOutface (line 612) | func TestMarshalTypeOutface(t *testing.T) {
FILE: go/merge/candidate.go
type candidate (line 18) | type candidate interface
type mapCandidate (line 25) | type mapCandidate struct
method diff (line 29) | func (mc mapCandidate) diff(p candidate, change chan<- types.ValueChan...
method get (line 33) | func (mc mapCandidate) get(k types.Value) types.Value {
method pathConcat (line 37) | func (mc mapCandidate) pathConcat(change types.ValueChanged, path type...
method getValue (line 47) | func (mc mapCandidate) getValue() types.Value {
type setCandidate (line 51) | type setCandidate struct
method diff (line 55) | func (sc setCandidate) diff(p candidate, change chan<- types.ValueChan...
method get (line 59) | func (sc setCandidate) get(k types.Value) types.Value {
method pathConcat (line 63) | func (sc setCandidate) pathConcat(change types.ValueChanged, path type...
method getValue (line 73) | func (sc setCandidate) getValue() types.Value {
type structCandidate (line 77) | type structCandidate struct
method diff (line 81) | func (sc structCandidate) diff(p candidate, change chan<- types.ValueC...
method get (line 85) | func (sc structCandidate) get(key types.Value) types.Value {
method pathConcat (line 93) | func (sc structCandidate) pathConcat(change types.ValueChanged, path t...
method getValue (line 102) | func (sc structCandidate) getValue() types.Value {
FILE: go/merge/three_way.go
type Policy (line 18) | type Policy
type ResolveFunc (line 29) | type ResolveFunc
function None (line 32) | func None(aChange, bChange types.DiffChangeType, a, b types.Value, path ...
function Ours (line 37) | func Ours(aChange, bChange types.DiffChangeType, a, b types.Value, path ...
function Theirs (line 42) | func Theirs(aChange, bChange types.DiffChangeType, a, b types.Value, pat...
type ErrMergeConflict (line 48) | type ErrMergeConflict struct
method Error (line 52) | func (e *ErrMergeConflict) Error() string {
function newMergeConflict (line 56) | func newMergeConflict(format string, args ...interface{}) *ErrMergeConfl...
function NewThreeWay (line 62) | func NewThreeWay(resolve ResolveFunc) Policy {
function ThreeWay (line 136) | func ThreeWay(a, b, parent types.Value, vrw types.ValueReadWriter, resol...
function unmergeable (line 158) | func unmergeable(a, b types.Value) bool {
type merger (line 166) | type merger struct
method threeWay (line 179) | func (m *merger) threeWay(a, b, parent types.Value, path types.Path) (...
method threeWayMapMerge (line 223) | func (m *merger) threeWayMapMerge(a, b, parent types.Map, path types.P...
method threeWaySetMerge (line 238) | func (m *merger) threeWaySetMerge(a, b, parent types.Set, path types.P...
method threeWayStructMerge (line 253) | func (m *merger) threeWayStructMerge(a, b, parent types.Struct, path t...
function updateProgress (line 172) | func updateProgress(progress chan<- struct{}) {
function listAssert (line 278) | func listAssert(vrw types.ValueReadWriter, a, b, parent types.Value) (aL...
function mapAssert (line 290) | func mapAssert(vrw types.ValueReadWriter, a, b, parent types.Value) (aMa...
function refAssert (line 302) | func refAssert(a, b, parent types.Value, vrw types.ValueReadWriter) (aVa...
function setAssert (line 323) | func setAssert(vrw types.ValueReadWriter, a, b, parent types.Value) (aSe...
function structAssert (line 335) | func structAssert(a, b, parent types.Value) (aStruct, bStruct, pStruct t...
FILE: go/merge/three_way_keyval_test.go
function TestThreeWayMapMerge (line 14) | func TestThreeWayMapMerge(t *testing.T) {
function TestThreeWayStructMerge (line 18) | func TestThreeWayStructMerge(t *testing.T) {
type kvs (line 22) | type kvs
method items (line 24) | func (kv kvs) items() []interface{} {
method remove (line 28) | func (kv kvs) remove(k interface{}) kvs {
method set (line 38) | func (kv kvs) set(k, v interface{}) kvs {
type ThreeWayKeyValMergeSuite (line 66) | type ThreeWayKeyValMergeSuite struct
method TestThreeWayMerge_DoNothing (line 104) | func (s *ThreeWayKeyValMergeSuite) TestThreeWayMerge_DoNothing() {
method TestThreeWayMerge_NoRecursion (line 108) | func (s *ThreeWayKeyValMergeSuite) TestThreeWayMerge_NoRecursion() {
method TestThreeWayMerge_RecursiveCreate (line 113) | func (s *ThreeWayKeyValMergeSuite) TestThreeWayMerge_RecursiveCreate() {
method TestThreeWayMerge_RecursiveCreateNil (line 118) | func (s *ThreeWayKeyValMergeSuite) TestThreeWayMerge_RecursiveCreateNi...
method TestThreeWayMerge_RecursiveMerge (line 123) | func (s *ThreeWayKeyValMergeSuite) TestThreeWayMerge_RecursiveMerge() {
method TestThreeWayMerge_RefMerge (line 128) | func (s *ThreeWayKeyValMergeSuite) TestThreeWayMerge_RefMerge() {
method TestThreeWayMerge_RecursiveMultiLevelMerge (line 140) | func (s *ThreeWayKeyValMergeSuite) TestThreeWayMerge_RecursiveMultiLev...
method TestThreeWayMerge_CustomMerge (line 150) | func (s *ThreeWayKeyValMergeSuite) TestThreeWayMerge_CustomMerge() {
method TestThreeWayMerge_MergeOurs (line 180) | func (s *ThreeWayKeyValMergeSuite) TestThreeWayMerge_MergeOurs() {
method TestThreeWayMerge_MergeTheirs (line 193) | func (s *ThreeWayKeyValMergeSuite) TestThreeWayMerge_MergeTheirs() {
method TestThreeWayMerge_NilConflict (line 206) | func (s *ThreeWayKeyValMergeSuite) TestThreeWayMerge_NilConflict() {
method TestThreeWayMerge_ImmediateConflict (line 211) | func (s *ThreeWayKeyValMergeSuite) TestThreeWayMerge_ImmediateConflict...
method TestThreeWayMerge_RefConflict (line 216) | func (s *ThreeWayKeyValMergeSuite) TestThreeWayMerge_RefConflict() {
method TestThreeWayMerge_NestedConflict (line 228) | func (s *ThreeWayKeyValMergeSuite) TestThreeWayMerge_NestedConflict() {
method TestThreeWayMerge_NestedConflictingOperation (line 234) | func (s *ThreeWayKeyValMergeSuite) TestThreeWayMerge_NestedConflicting...
type ThreeWayMapMergeSuite (line 70) | type ThreeWayMapMergeSuite struct
method SetupSuite (line 74) | func (s *ThreeWayMapMergeSuite) SetupSuite() {
type ThreeWayStructMergeSuite (line 85) | type ThreeWayStructMergeSuite struct
method SetupSuite (line 89) | func (s *ThreeWayStructMergeSuite) SetupSuite() {
FILE: go/merge/three_way_list.go
function threeWayListMerge (line 14) | func threeWayListMerge(a, b, parent types.List) (merged types.List, err ...
function overlap (line 90) | func overlap(s1, s2 types.Splice) bool {
function canMerge (line 99) | func canMerge(a, b types.List, aSplice, bSplice types.Splice) bool {
function merge (line 114) | func merge(s1, s2 types.Splice) types.Splice {
function apply (line 118) | func apply(source, target types.List, offset uint64, s types.Splice) typ...
function describeSplice (line 131) | func describeSplice(s types.Splice) string {
FILE: go/merge/three_way_list_test.go
function TestThreeWayListMerge (line 14) | func TestThreeWayListMerge(t *testing.T) {
type ThreeWayListMergeSuite (line 18) | type ThreeWayListMergeSuite struct
method SetupSuite (line 22) | func (s *ThreeWayListMergeSuite) SetupSuite() {
method TestThreeWayMerge_DoNothing (line 35) | func (s *ThreeWayListMergeSuite) TestThreeWayMerge_DoNothing() {
method TestThreeWayMerge_NoLengthChange (line 39) | func (s *ThreeWayListMergeSuite) TestThreeWayMerge_NoLengthChange() {
method TestThreeWayMerge_HandleEmpty (line 47) | func (s *ThreeWayListMergeSuite) TestThreeWayMerge_HandleEmpty() {
method TestThreeWayMerge_HandleNil (line 53) | func (s *ThreeWayListMergeSuite) TestThreeWayMerge_HandleNil() {
method TestThreeWayMerge_MakeLonger (line 57) | func (s *ThreeWayListMergeSuite) TestThreeWayMerge_MakeLonger() {
method TestThreeWayMerge_MakeShorter (line 66) | func (s *ThreeWayListMergeSuite) TestThreeWayMerge_MakeShorter() {
method TestThreeWayMerge_BothSidesRemove (line 74) | func (s *ThreeWayListMergeSuite) TestThreeWayMerge_BothSidesRemove() {
method TestThreeWayMerge_OverlapSameRemoveNoInsert (line 82) | func (s *ThreeWayListMergeSuite) TestThreeWayMerge_OverlapSameRemoveNo...
method TestThreeWayMerge_OverlapSameRemoveSameInsert (line 90) | func (s *ThreeWayListMergeSuite) TestThreeWayMerge_OverlapSameRemoveSa...
method TestThreeWayMerge_RemoveUpToOtherSideInsertionPoint (line 98) | func (s *ThreeWayListMergeSuite) TestThreeWayMerge_RemoveUpToOtherSide...
method TestThreeWayMerge_ConflictingAppends (line 106) | func (s *ThreeWayListMergeSuite) TestThreeWayMerge_ConflictingAppends() {
method TestThreeWayMerge_OverlappingRemoves (line 113) | func (s *ThreeWayListMergeSuite) TestThreeWayMerge_OverlappingRemoves() {
method TestThreeWayMerge_SameRemoveAddPrefix (line 120) | func (s *ThreeWayListMergeSuite) TestThreeWayMerge_SameRemoveAddPrefix...
method TestThreeWayMerge_RemoveSupersetAddPrefix (line 127) | func (s *ThreeWayListMergeSuite) TestThreeWayMerge_RemoveSupersetAddPr...
method TestThreeWayMerge_RemoveOtherSideInsertionPoint (line 134) | func (s *ThreeWayListMergeSuite) TestThreeWayMerge_RemoveOtherSideInse...
FILE: go/merge/three_way_ordered_sequence.go
type applyFunc (line 14) | type applyFunc
method threeWayOrderedSequenceMerge (line 16) | func (m *merger) threeWayOrderedSequenceMerge(a, b, parent candidate, ap...
method mergeChanges (line 73) | func (m *merger) mergeChanges(aChange, bChange types.ValueChanged, a, b,...
function stopAndDrain (line 107) | func stopAndDrain(stop chan<- struct{}, drain <-chan types.ValueChanged) {
function describeChange (line 113) | func describeChange(change types.ValueChanged) string {
FILE: go/merge/three_way_set_test.go
function TestThreeWaySetMerge (line 14) | func TestThreeWaySetMerge(t *testing.T) {
type items (line 18) | type items
method items (line 20) | func (kv items) items() []interface{} {
type ThreeWaySetMergeSuite (line 24) | type ThreeWaySetMergeSuite struct
method SetupSuite (line 28) | func (s *ThreeWaySetMergeSuite) SetupSuite() {
method TestThreeWayMerge_DoNothing (line 51) | func (s *ThreeWaySetMergeSuite) TestThreeWayMerge_DoNothing() {
method TestThreeWayMerge_Primitives (line 55) | func (s *ThreeWaySetMergeSuite) TestThreeWayMerge_Primitives() {
method TestThreeWayMerge_HandleEmpty (line 60) | func (s *ThreeWaySetMergeSuite) TestThreeWayMerge_HandleEmpty() {
method TestThreeWayMerge_HandleNil (line 65) | func (s *ThreeWaySetMergeSuite) TestThreeWayMerge_HandleNil() {
method TestThreeWayMerge_Refs (line 70) | func (s *ThreeWaySetMergeSuite) TestThreeWayMerge_Refs() {
method TestThreeWayMerge_ImmediateConflict (line 82) | func (s *ThreeWaySetMergeSuite) TestThreeWayMerge_ImmediateConflict() {
FILE: go/merge/three_way_test.go
type seq (line 16) | type seq interface
type ThreeWayMergeSuite (line 20) | type ThreeWayMergeSuite struct
method SetupTest (line 27) | func (s *ThreeWayMergeSuite) SetupTest() {
method TearDownTest (line 32) | func (s *ThreeWayMergeSuite) TearDownTest() {
method tryThreeWayMerge (line 36) | func (s *ThreeWayMergeSuite) tryThreeWayMerge(a, b, p, exp seq) {
method tryThreeWayConflict (line 44) | func (s *ThreeWayMergeSuite) tryThreeWayConflict(a, b, p types.Value, ...
function valsToTypesValues (line 53) | func valsToTypesValues(f func(seq) types.Value, items ...interface{}) []...
function valToTypesValue (line 62) | func valToTypesValue(f func(seq) types.Value, v interface{}) types.Value {
function TestThreeWayMerge_PrimitiveConflict (line 77) | func TestThreeWayMerge_PrimitiveConflict(t *testing.T) {
FILE: go/metrics/histogram.go
type Histogram (line 29) | type Histogram struct
method Sample (line 44) | func (h *Histogram) Sample(v uint64) {
method SampleTimeSince (line 61) | func (h *Histogram) SampleTimeSince(t time.Time) {
method SampleLen (line 71) | func (h *Histogram) SampleLen(l int) {
method bucketVal (line 75) | func (h Histogram) bucketVal(bucket int) uint64 {
method Sum (line 81) | func (h Histogram) Sum() uint64 {
method Add (line 87) | func (h *Histogram) Add(other Histogram) {
method Delta (line 99) | func (h Histogram) Delta(other Histogram) Histogram {
method Mean (line 113) | func (h Histogram) Mean() uint64 {
method Samples (line 123) | func (h Histogram) Samples() uint64 {
method String (line 131) | func (h Histogram) String() string {
method Report (line 156) | func (h Histogram) Report() string {
type ToStringFunc (line 35) | type ToStringFunc
function identToString (line 37) | func identToString(v uint64) string {
constant bucketCount (line 41) | bucketCount = 64
function NewTimeHistogram (line 139) | func NewTimeHistogram() Histogram {
function timeToString (line 143) | func timeToString(v uint64) string {
function NewByteHistogram (line 148) | func NewByteHistogram() Histogram {
constant colWidth (line 152) | colWidth = 100
FILE: go/metrics/histogram_test.go
function TestHistogramBucketValue (line 13) | func TestHistogramBucketValue(t *testing.T) {
function TestHistogramBasic (line 24) | func TestHistogramBasic(t *testing.T) {
function TestHistogramLarge (line 52) | func TestHistogramLarge(t *testing.T) {
function TestHistogramAdd (line 60) | func TestHistogramAdd(t *testing.T) {
function TestHistogramString (line 78) | func TestHistogramString(t *testing.T) {
function TestHistogramReport (line 98) | func TestHistogramReport(t *testing.T) {
FILE: go/nbs/aws_chunk_source.go
function newAWSChunkSource (line 14) | func newAWSChunkSource(ddb *ddbTableStore, s3 *s3ObjectReader, al awsLim...
type awsChunkSource (line 53) | type awsChunkSource struct
method hash (line 58) | func (acs *awsChunkSource) hash() addr {
type awsTableReaderAt (line 62) | type awsTableReaderAt struct
method hash (line 74) | func (atra *awsTableReaderAt) hash() addr {
method ReadAtWithStats (line 78) | func (atra *awsTableReaderAt) ReadAtWithStats(p []byte, off int64, sta...
method getTableReaderAt (line 83) | func (atra *awsTableReaderAt) getTableReaderAt(stats *Stats) tableRead...
FILE: go/nbs/aws_chunk_source_test.go
function TestAWSChunkSource (line 13) | func TestAWSChunkSource(t *testing.T) {
FILE: go/nbs/aws_table_persister.go
constant minS3PartSize (line 22) | minS3PartSize = 5 * 1 << 20
constant maxS3PartSize (line 23) | maxS3PartSize = 64 * 1 << 20
constant maxS3Parts (line 24) | maxS3Parts = 10000
constant maxDynamoChunks (line 26) | maxDynamoChunks = 64
constant maxDynamoItemSize (line 27) | maxDynamoItemSize = 400 * (1 << 10)
constant defaultS3PartSize (line 29) | defaultS3PartSize = minS3PartSize
type awsTablePersister (line 32) | type awsTablePersister struct
method Open (line 59) | func (s3p awsTablePersister) Open(name addr, chunkCount uint32, stats ...
method Persist (line 76) | func (s3p awsTablePersister) Persist(mt *memTable, haver chunkReader, ...
method newReaderFromIndexData (line 94) | func (s3p awsTablePersister) newReaderFromIndexData(idxData []byte, na...
method multipartUpload (line 104) | func (s3p awsTablePersister) multipartUpload(data []byte, key string) {
method startMultipartUpload (line 114) | func (s3p awsTablePersister) startMultipartUpload(key string) string {
method abortMultipartUpload (line 123) | func (s3p awsTablePersister) abortMultipartUpload(key, uploadID string) {
method completeMultipartUpload (line 132) | func (s3p awsTablePersister) completeMultipartUpload(key, uploadID str...
method uploadParts (line 142) | func (s3p awsTablePersister) uploadParts(data []byte, key, uploadID st...
method ConjoinAll (line 239) | func (s3p awsTablePersister) ConjoinAll(sources chunkSources, stats *S...
method loadIntoCache (line 256) | func (s3p awsTablePersister) loadIntoCache(name addr) {
method executeCompactionPlan (line 267) | func (s3p awsTablePersister) executeCompactionPlan(plan compactionPlan...
method assembleTable (line 277) | func (s3p awsTablePersister) assembleTable(plan compactionPlan, key, u...
method uploadPartCopy (line 460) | func (s3p awsTablePersister) uploadPartCopy(src string, srcStart, srcE...
method uploadPart (line 476) | func (s3p awsTablePersister) uploadPart(data []byte, key, uploadID str...
type awsLimits (line 42) | type awsLimits struct
method tableFitsInDynamo (line 48) | func (al awsLimits) tableFitsInDynamo(name addr, dataLen int, chunkCou...
method tableMayBeInDynamo (line 55) | func (al awsLimits) tableMayBeInDynamo(chunkCount uint32) bool {
type s3UploadedPart (line 71) | type s3UploadedPart struct
function getNumParts (line 217) | func getNumParts(dataLen, minPartSize uint64) uint64 {
type partsByPartNum (line 225) | type partsByPartNum
method Len (line 227) | func (s partsByPartNum) Len() int {
method Less (line 231) | func (s partsByPartNum) Less(i, j int) bool {
method Swap (line 235) | func (s partsByPartNum) Swap(i, j int) {
type copyPart (line 392) | type copyPart struct
type manualPart (line 397) | type manualPart struct
function dividePlan (line 403) | func dividePlan(plan compactionPlan, minPartSize, maxPartSize uint64) (c...
function splitOnMaxSize (line 442) | func splitOnMaxSize(dataLen, maxPartSize uint64) []int64 {
FILE: go/nbs/aws_table_persister_test.go
function TestAWSTablePersisterPersist (line 18) | func TestAWSTablePersisterPersist(t *testing.T) {
function makeFakeDTS (line 194) | func makeFakeDTS(ddb ddbsvc, tc *sizecache.SizeCache) *ddbTableStore {
type waitOnStoreTableCache (line 198) | type waitOnStoreTableCache struct
method checkout (line 204) | func (mtc *waitOnStoreTableCache) checkout(h addr) io.ReaderAt {
method checkin (line 210) | func (mtc *waitOnStoreTableCache) checkin(h addr) {}
method store (line 212) | func (mtc *waitOnStoreTableCache) store(h addr, data io.Reader, size u...
type failingFakeS3 (line 219) | type failingFakeS3 struct
method UploadPart (line 225) | func (m *failingFakeS3) UploadPart(input *s3.UploadPartInput) (*s3.Upl...
function TestAWSTablePersisterDividePlan (line 235) | func TestAWSTablePersisterDividePlan(t *testing.T) {
function TestAWSTablePersisterCalcPartSizes (line 268) | func TestAWSTablePersisterCalcPartSizes(t *testing.T) {
function TestAWSTablePersisterConjoinAll (line 289) | func TestAWSTablePersisterConjoinAll(t *testing.T) {
function bytesToChunkSource (line 464) | func bytesToChunkSource(bs ...[]byte) chunkSource {
FILE: go/nbs/block_store_test.go
constant testMemTableSize (line 24) | testMemTableSize = 1 << 8
function TestBlockStoreSuite (line 26) | func TestBlockStoreSuite(t *testing.T) {
type BlockStoreSuite (line 30) | type BlockStoreSuite struct
method SetupTest (line 37) | func (suite *BlockStoreSuite) SetupTest() {
method TearDownTest (line 47) | func (suite *BlockStoreSuite) TearDownTest() {
method TestChunkStoreMissingDir (line 52) | func (suite *BlockStoreSuite) TestChunkStoreMissingDir() {
method TestChunkStoreNotDir (line 57) | func (suite *BlockStoreSuite) TestChunkStoreNotDir() {
method TestChunkStorePut (line 63) | func (suite *BlockStoreSuite) TestChunkStorePut() {
method TestChunkStorePutMany (line 92) | func (suite *BlockStoreSuite) TestChunkStorePutMany() {
method TestChunkStoreStatsSummary (line 108) | func (suite *BlockStoreSuite) TestChunkStoreStatsSummary() {
method TestChunkStorePutMoreThanMemTable (line 121) | func (suite *BlockStoreSuite) TestChunkStorePutMoreThanMemTable() {
method TestChunkStoreGetMany (line 140) | func (suite *BlockStoreSuite) TestChunkStoreGetMany() {
method TestChunkStoreHasMany (line 170) | func (suite *BlockStoreSuite) TestChunkStoreHasMany() {
method TestChunkStoreExtractChunks (line 191) | func (suite *BlockStoreSuite) TestChunkStoreExtractChunks() {
method TestChunkStoreFlushOptimisticLockFail (line 210) | func (suite *BlockStoreSuite) TestChunkStoreFlushOptimisticLockFail() {
method TestChunkStoreRebaseOnNoOpFlush (line 235) | func (suite *BlockStoreSuite) TestChunkStoreRebaseOnNoOpFlush() {
method TestChunkStorePutWithRebase (line 253) | func (suite *BlockStoreSuite) TestChunkStorePutWithRebase() {
method TestChunkStoreGetNonExisting (line 420) | func (suite *BlockStoreSuite) TestChunkStoreGetNonExisting() {
function TestBlockStoreConjoinOnCommit (line 293) | func TestBlockStoreConjoinOnCommit(t *testing.T) {
type cannedConjoin (line 382) | type cannedConjoin struct
type fakeConjoiner (line 387) | type fakeConjoiner struct
method ConjoinRequired (line 391) | func (fc *fakeConjoiner) ConjoinRequired(ts tableSet) bool {
method Conjoin (line 398) | func (fc *fakeConjoiner) Conjoin(upstream manifestContents, mm manifes...
function assertInputInStore (line 414) | func assertInputInStore(input []byte, h hash.Hash, s chunks.ChunkStore, ...
FILE: go/nbs/cache.go
constant defaultCacheMemTableSize (line 17) | defaultCacheMemTableSize uint64 = 1 << 27
function NewCache (line 20) | func NewCache() *NomsBlockCache {
type NomsBlockCache (line 29) | type NomsBlockCache struct
method Insert (line 35) | func (nbc *NomsBlockCache) Insert(c chunks.Chunk) {
method Has (line 40) | func (nbc *NomsBlockCache) Has(hash hash.Hash) bool {
method HasMany (line 46) | func (nbc *NomsBlockCache) HasMany(hashes hash.HashSet) hash.HashSet {
method Get (line 52) | func (nbc *NomsBlockCache) Get(hash hash.Hash) chunks.Chunk {
method GetMany (line 59) | func (nbc *NomsBlockCache) GetMany(hashes hash.HashSet, foundChunks ch...
method ExtractChunks (line 65) | func (nbc *NomsBlockCache) ExtractChunks(chunkChan chan *chunks.Chunk) {
method Count (line 70) | func (nbc *NomsBlockCache) Count() uint32 {
method Destroy (line 75) | func (nbc *NomsBlockCache) Destroy() error {
FILE: go/nbs/conjoiner.go
type conjoiner (line 16) | type conjoiner interface
type inlineConjoiner (line 32) | type inlineConjoiner struct
method ConjoinRequired (line 36) | func (c inlineConjoiner) ConjoinRequired(ts tableSet) bool {
method Conjoin (line 40) | func (c inlineConjoiner) Conjoin(upstream manifestContents, mm manifes...
function conjoin (line 44) | func conjoin(upstream manifestContents, mm manifestUpdater, p tablePersi...
function conjoinTables (line 91) | func conjoinTables(p tablePersister, upstream []tableSpec, stats *Stats)...
function chooseConjoinees (line 118) | func chooseConjoinees(upstream chunkSources) (toConjoin, toKeep chunkSou...
function toSpecs (line 133) | func toSpecs(srcs chunkSources) []tableSpec {
FILE: go/nbs/conjoiner_test.go
type tableSpecsByAscendingCount (line 18) | type tableSpecsByAscendingCount
method Len (line 20) | func (ts tableSpecsByAscendingCount) Len() int { return len(ts) }
method Less (line 21) | func (ts tableSpecsByAscendingCount) Less(i, j int) bool {
method Swap (line 28) | func (ts tableSpecsByAscendingCount) Swap(i, j int) { ts[i], ts[j] = t...
function makeTestSrcs (line 30) | func makeTestSrcs(tableSizes []uint32, p tablePersister) (srcs chunkSour...
function TestConjoin (line 50) | func TestConjoin(t *testing.T) {
type updatePreemptManifest (line 169) | type updatePreemptManifest struct
method Update (line 174) | func (u updatePreemptManifest) Update(lastLock addr, newContents manif...
FILE: go/nbs/dynamo_fake_test.go
type fakeDDB (line 17) | type fakeDDB struct
method readerForTable (line 35) | func (m *fakeDDB) readerForTable(name addr) chunkReader {
method GetItem (line 44) | func (m *fakeDDB) GetItem(input *dynamodb.GetItemInput) (*dynamodb.Get...
method putRecord (line 68) | func (m *fakeDDB) putRecord(k string, l, r []byte, v string, s string) {
method putData (line 72) | func (m *fakeDDB) putData(k string, d []byte) {
method PutItem (line 76) | func (m *fakeDDB) PutItem(input *dynamodb.PutItemInput) (*dynamodb.Put...
type record (line 23) | type record struct
function makeFakeDDB (line 28) | func makeFakeDDB(t *testing.T) *fakeDDB {
function checkCondition (line 124) | func checkCondition(current record, expressionAttrVals map[string]*dynam...
FILE: go/nbs/dynamo_manifest.go
constant dbAttr (line 21) | dbAttr = "db"
constant lockAttr (line 22) | lockAttr = "lck"
constant rootAttr (line 23) | rootAttr = "root"
constant versAttr (line 24) | versAttr = "vers"
constant nbsVersAttr (line 25) | nbsVersAttr = "nbsVers"
constant tableSpecsAttr (line 26) | tableSpecsAttr = "specs"
type ddbsvc (line 34) | type ddbsvc interface
type dynamoManifest (line 40) | type dynamoManifest struct
method Name (line 51) | func (dm dynamoManifest) Name() string {
method ParseIfExists (line 55) | func (dm dynamoManifest) ParseIfExists(stats *Stats, readHook func()) ...
method Update (line 99) | func (dm dynamoManifest) Update(lastLock addr, newContents manifestCon...
function newDynamoManifest (line 45) | func newDynamoManifest(table, namespace string, ddb ddbsvc) manifest {
function validateManifest (line 85) | func validateManifest(item map[string]*dynamodb.AttributeValue) (valid, ...
function errIsConditionalCheckFailed (line 144) | func errIsConditionalCheckFailed(err error) bool {
FILE: go/nbs/dynamo_manifest_test.go
constant table (line 16) | table = "testTable"
constant db (line 17) | db = "testDB"
function makeDynamoManifestFake (line 20) | func makeDynamoManifestFake(t *testing.T) (mm manifest, ddb *fakeDDB) {
function TestDynamoManifestParseIfExists (line 26) | func TestDynamoManifestParseIfExists(t *testing.T) {
function makeContents (line 52) | func makeContents(lock, root string, specs []tableSpec) manifestContents {
function TestDynamoManifestUpdateWontClobberOldVersion (line 56) | func TestDynamoManifestUpdateWontClobberOldVersion(t *testing.T) {
function TestDynamoManifestUpdate (line 69) | func TestDynamoManifestUpdate(t *testing.T) {
function TestDynamoManifestCaching (line 109) | func TestDynamoManifestCaching(t *testing.T) {
function TestDynamoManifestUpdateEmpty (line 142) | func TestDynamoManifestUpdateEmpty(t *testing.T) {
FILE: go/nbs/dynamo_table_reader.go
constant dataAttr (line 20) | dataAttr = "data"
constant tablePrefix (line 21) | tablePrefix = "*"
type dynamoTableReaderAt (line 25) | type dynamoTableReaderAt struct
method ReadAtWithStats (line 38) | func (dtra *dynamoTableReaderAt) ReadAtWithStats(p []byte, off int64, ...
type tableNotInDynamoErr (line 30) | type tableNotInDynamoErr struct
method Error (line 34) | func (t tableNotInDynamoErr) Error() string {
type ddbTableStore (line 48) | type ddbTableStore struct
method ReadTable (line 55) | func (dts *ddbTableStore) ReadTable(name addr, stats *Stats) (data []b...
method readTable (line 82) | func (dts *ddbTableStore) readTable(name addr) (data []byte, err error) {
method Write (line 120) | func (dts *ddbTableStore) Write(name addr, data []byte) error {
function fmtTableName (line 116) | func fmtTableName(name addr) string {
FILE: go/nbs/dynamo_table_reader_test.go
function TestDynamoTableReaderAt (line 15) | func TestDynamoTableReaderAt(t *testing.T) {
type eventuallyConsistentDDB (line 113) | type eventuallyConsistentDDB struct
method GetItem (line 117) | func (ec *eventuallyConsistentDDB) GetItem(input *dynamodb.GetItemInpu...
FILE: go/nbs/factory.go
constant defaultAWSReadLimit (line 21) | defaultAWSReadLimit = 1024
constant awsMaxTables (line 22) | awsMaxTables = 128
constant defaultSmallTableCacheSize (line 24) | defaultSmallTableCacheSize = 1 << 28
type AWSStoreFactory (line 28) | type AWSStoreFactory struct
method CreateStore (line 71) | func (asf *AWSStoreFactory) CreateStore(ns string) chunks.ChunkStore {
method CreateStoreFromCache (line 76) | func (asf *AWSStoreFactory) CreateStoreFromCache(ns string) chunks.Chu...
method Shutter (line 86) | func (asf *AWSStoreFactory) Shutter() {
function NewAWSStoreFactory (line 41) | func NewAWSStoreFactory(sess *session.Session, table, bucket string, max...
type LocalStoreFactory (line 89) | type LocalStoreFactory struct
method CreateStore (line 128) | func (lsf *LocalStoreFactory) CreateStore(ns string) chunks.ChunkStore {
method CreateStoreFromCache (line 137) | func (lsf *LocalStoreFactory) CreateStoreFromCache(ns string) chunks.C...
method Shutter (line 151) | func (lsf *LocalStoreFactory) Shutter() {
function checkDir (line 99) | func checkDir(dir string) error {
function NewLocalStoreFactory (line 110) | func NewLocalStoreFactory(dir string, indexCacheSize uint64, maxOpenFile...
FILE: go/nbs/factory_test.go
function TestLocalStoreFactory (line 19) | func TestLocalStoreFactory(t *testing.T) {
FILE: go/nbs/fd_cache.go
function newFDCache (line 15) | func newFDCache(targetSize int) *fdCache {
type fdCache (line 24) | type fdCache struct
method RefFile (line 42) | func (fc *fdCache) RefFile(path string) (f *os.File, err error) {
method UnrefFile (line 84) | func (fc *fdCache) UnrefFile(path string) {
method Drop (line 114) | func (fc *fdCache) Drop() {
method reportEntries (line 124) | func (fc *fdCache) reportEntries() sort.StringSlice {
type fdCacheEntry (line 30) | type fdCacheEntry struct
FILE: go/nbs/fd_cache_test.go
function TestFDCache (line 19) | func TestFDCache(t *testing.T) {
FILE: go/nbs/file_manifest.go
constant manifestFileName (line 23) | manifestFileName = "manifest"
constant lockFileName (line 24) | lockFileName = "LOCK"
type fileManifest (line 32) | type fileManifest struct
method Name (line 36) | func (fm fileManifest) Name() string {
method ParseIfExists (line 46) | func (fm fileManifest) ParseIfExists(stats *Stats, readHook func()) (e...
method Update (line 100) | func (fm fileManifest) Update(lastLock addr, newContents manifestConte...
function openIfExists (line 73) | func openIfExists(path string) *os.File {
function parseManifest (line 82) | func parseManifest(r io.Reader) manifestContents {
function writeManifest (line 145) | func writeManifest(temp io.Writer, contents manifestContents) {
function checkClose (line 154) | func checkClose(c io.Closer) {
function flock (line 158) | func flock(lockFilePath string) io.Closer {
FILE: go/nbs/file_manifest_test.go
function makeFileManifestTempDir (line 21) | func makeFileManifestTempDir(t *testing.T) fileManifest {
function TestFileManifestLoadIfExists (line 27) | func TestFileManifestLoadIfExists(t *testing.T) {
function TestFileManifestLoadIfExistsHoldsLock (line 55) | func TestFileManifestLoadIfExistsHoldsLock(t *testing.T) {
function TestFileManifestUpdateWontClobberOldVersion (line 86) | func TestFileManifestUpdateWontClobberOldVersion(t *testing.T) {
function TestFileManifestUpdateEmpty (line 99) | func TestFileManifestUpdateEmpty(t *testing.T) {
function TestFileManifestUpdate (line 125) | func TestFileManifestUpdate(t *testing.T) {
function tryClobberManifest (line 174) | func tryClobberManifest(dir, contents string) ([]byte, error) {
function clobberManifest (line 179) | func clobberManifest(dir, contents string) error {
function runClobber (line 186) | func runClobber(dir, contents string) ([]byte, error) {
FILE: go/nbs/file_table_persister.go
constant tempTablePrefix (line 17) | tempTablePrefix = "nbs_table_"
function newFSTablePersister (line 19) | func newFSTablePersister(dir string, fc *fdCache, indexCache *indexCache...
type fsTablePersister (line 24) | type fsTablePersister struct
method Open (line 30) | func (ftp *fsTablePersister) Open(name addr, chunkCount uint32, stats ...
method Persist (line 34) | func (ftp *fsTablePersister) Persist(mt *memTable, haver chunkReader, ...
method persistTable (line 39) | func (ftp *fsTablePersister) persistTable(name addr, data []byte, chun...
method ConjoinAll (line 61) | func (ftp *fsTablePersister) ConjoinAll(sources chunkSources, stats *S...
FILE: go/nbs/file_table_persister_test.go
function TestFSTableCacheOnOpen (line 19) | func TestFSTableCacheOnOpen(t *testing.T) {
function makeTempDir (line 60) | func makeTempDir(t *testing.T) string {
function writeTableData (line 66) | func writeTableData(dir string, chunx ...[]byte) (name addr, err error) {
function removeTables (line 73) | func removeTables(dir string, names ...addr) error {
function contains (line 82) | func contains(s sort.StringSlice, e string) bool {
function TestFSTablePersisterPersist (line 91) | func TestFSTablePersisterPersist(t *testing.T) {
function persistTableData (line 109) | func persistTableData(p tablePersister, chunx ...[]byte) (src chunkSourc...
function TestFSTablePersisterPersistNoData (line 119) | func TestFSTablePersisterPersistNoData(t *testing.T) {
function TestFSTablePersisterCacheOnPersist (line 142) | func TestFSTablePersisterCacheOnPersist(t *testing.T) {
function TestFSTablePersisterConjoinAll (line 171) | func TestFSTablePersisterConjoinAll(t *testing.T) {
function TestFSTablePersisterConjoinAllDups (line 205) | func TestFSTablePersisterConjoinAllDups(t *testing.T) {
FILE: go/nbs/fs_table_cache.go
type tableCache (line 20) | type tableCache interface
type fsTableCache (line 26) | type fsTableCache struct
method init (line 42) | func (ftc *fsTableCache) init(concurrency int) {
method checkout (line 95) | func (ftc *fsTableCache) checkout(h addr) io.ReaderAt {
method checkin (line 106) | func (ftc *fsTableCache) checkin(h addr) {
method store (line 110) | func (ftc *fsTableCache) store(h addr, data io.Reader, size uint64) {
method expire (line 129) | func (ftc *fsTableCache) expire(h addr) {
function newFSTableCache (line 32) | func newFSTableCache(dir string, cacheSize uint64, maxOpenFds int) *fsTa...
FILE: go/nbs/fs_table_cache_test.go
function TestFSTableCache (line 19) | func TestFSTableCache(t *testing.T) {
function assertDataInReaderAt (line 106) | func assertDataInReaderAt(t *testing.T, data []byte, r io.ReaderAt) {
FILE: go/nbs/manifest.go
type manifest (line 17) | type manifest interface
type manifestUpdater (line 39) | type manifestUpdater interface
type manifestContents (line 58) | type manifestContents struct
method size (line 65) | func (mc manifestContents) size() (size uint64) {
function newManifestLocks (line 73) | func newManifestLocks() *manifestLocks {
type manifestLocks (line 77) | type manifestLocks struct
method lockForFetch (line 83) | func (ml *manifestLocks) lockForFetch(db string) {
method unlockForFetch (line 87) | func (ml *manifestLocks) unlockForFetch(db string) {
method lockForUpdate (line 91) | func (ml *manifestLocks) lockForUpdate(db string) {
method unlockForUpdate (line 95) | func (ml *manifestLocks) unlockForUpdate(db string) {
function lockByName (line 99) | func lockByName(db string, c *sync.Cond, locks map[string]struct{}) {
function unlockByName (line 112) | func unlockByName(db string, c *sync.Cond, locks map[string]struct{}) {
type manifestManager (line 123) | type manifestManager struct
method lockOutFetch (line 129) | func (mm manifestManager) lockOutFetch() {
method allowFetch (line 133) | func (mm manifestManager) allowFetch() {
method LockForUpdate (line 137) | func (mm manifestManager) LockForUpdate() {
method UnlockForUpdate (line 141) | func (mm manifestManager) UnlockForUpdate() {
method updateWillFail (line 145) | func (mm manifestManager) updateWillFail(lastLock addr) (cached manife...
method Fetch (line 154) | func (mm manifestManager) Fetch(stats *Stats) (exists bool, contents m...
method Update (line 176) | func (mm manifestManager) Update(lastLock addr, newContents manifestCo...
method Name (line 192) | func (mm manifestManager) Name() string {
type tableSpec (line 196) | type tableSpec struct
function parseSpecs (line 201) | func parseSpecs(tableInfo []string) []tableSpec {
function formatSpecs (line 212) | func formatSpecs(specs []tableSpec, tableInfo []string) {
function generateLockHash (line 226) | func generateLockHash(root hash.Hash, specs []tableSpec) (lock addr) {
FILE: go/nbs/manifest_cache.go
function newManifestCache (line 16) | func newManifestCache(maxSize uint64) *manifestCache {
type manifestCacheEntry (line 24) | type manifestCacheEntry struct
type manifestCache (line 30) | type manifestCache struct
method Get (line 41) | func (mc *manifestCache) Get(db string) (contents manifestContents, t ...
method entry (line 54) | func (mc *manifestCache) entry(key string) (manifestCacheEntry, bool) {
method Put (line 70) | func (mc *manifestCache) Put(db string, contents manifestContents, t t...
FILE: go/nbs/manifest_cache_test.go
function TestSizeCache (line 14) | func TestSizeCache(t *testing.T) {
FILE: go/nbs/mem_table.go
type memTable (line 15) | type memTable struct
method addChunk (line 27) | func (mt *memTable) addChunk(h addr, data []byte) bool {
method count (line 49) | func (mt *memTable) count() uint32 {
method uncompressedLen (line 53) | func (mt *memTable) uncompressedLen() uint64 {
method has (line 57) | func (mt *memTable) has(h addr) (has bool) {
method hasMany (line 62) | func (mt *memTable) hasMany(addrs []hasRecord) (remaining bool) {
method get (line 77) | func (mt *memTable) get(h addr, stats *Stats) []byte {
method getMany (line 81) | func (mt *memTable) getMany(reqs []getRecord, foundChunks chan *chunks...
method extract (line 94) | func (mt *memTable) extract(chunks chan<- extractRecord) {
method write (line 101) | func (mt *memTable) write(haver chunkReader, stats *Stats) (name addr,...
function newMemTable (line 23) | func newMemTable(memTableSize uint64) *memTable {
FILE: go/nbs/mem_table_test.go
function TestMemTableAddHasGetChunk (line 17) | func TestMemTableAddHasGetChunk(t *testing.T) {
function TestMemTableAddOverflowChunk (line 42) | func TestMemTableAddOverflowChunk(t *testing.T) {
function TestMemTableWrite (line 71) | func TestMemTableWrite(t *testing.T) {
type tableReaderAtAdapter (line 101) | type tableReaderAtAdapter struct
method ReadAtWithStats (line 109) | func (adapter tableReaderAtAdapter) ReadAtWithStats(p []byte, off int6...
function tableReaderAtFromBytes (line 105) | func tableReaderAtFromBytes(b []byte) tableReaderAt {
function TestMemTableSnappyWriteOutOfLine (line 113) | func TestMemTableSnappyWriteOutOfLine(t *testing.T) {
type outOfLineSnappy (line 131) | type outOfLineSnappy struct
method Encode (line 135) | func (o *outOfLineSnappy) Encode(dst, src []byte) []byte {
type chunkReaderGroup (line 147) | type chunkReaderGroup
method has (line 149) | func (crg chunkReaderGroup) has(h addr) bool {
method get (line 158) | func (crg chunkReaderGroup) get(h addr, stats *Stats) []byte {
method hasMany (line 167) | func (crg chunkReaderGroup) hasMany(addrs []hasRecord) (remaining bool) {
method getMany (line 176) | func (crg chunkReaderGroup) getMany(reqs []getRecord, foundChunks chan...
method count (line 185) | func (crg chunkReaderGroup) count() (count uint32) {
method uncompressedLen (line 192) | func (crg chunkReaderGroup) uncompressedLen() (data uint64) {
method extract (line 199) | func (crg chunkReaderGroup) extract(chunks chan<- extractRecord) {
FILE: go/nbs/mmap_table_reader.go
type mmapTableReader (line 20) | type mmapTableReader struct
method hash (line 83) | func (mmtr *mmapTableReader) hash() addr {
constant fileBlockSize (line 27) | fileBlockSize = 1 << 12
function init (line 35) | func init() {
function newMmapTableReader (line 41) | func newMmapTableReader(dir string, h addr, chunkCount uint32, indexCach...
type cacheReaderAt (line 87) | type cacheReaderAt struct
method ReadAtWithStats (line 92) | func (cra *cacheReaderAt) ReadAtWithStats(p []byte, off int64, stats *...
FILE: go/nbs/mmap_table_reader_test.go
function TestMmapTableReader (line 16) | func TestMmapTableReader(t *testing.T) {
FILE: go/nbs/persisting_chunk_source.go
function newPersistingChunkSource (line 17) | func newPersistingChunkSource(mt *memTable, haver chunkReader, p tablePe...
type persistingChunkSource (line 40) | type persistingChunkSource struct
method getReader (line 48) | func (ccs *persistingChunkSource) getReader() chunkReader {
method has (line 57) | func (ccs *persistingChunkSource) has(h addr) bool {
method hasMany (line 63) | func (ccs *persistingChunkSource) hasMany(addrs []hasRecord) bool {
method get (line 69) | func (ccs *persistingChunkSource) get(h addr, stats *Stats) []byte {
method getMany (line 75) | func (ccs *persistingChunkSource) getMany(reqs []getRecord, foundChunk...
method count (line 81) | func (ccs *persistingChunkSource) count() uint32 {
method uncompressedLen (line 87) | func (ccs *persistingChunkSource) uncompressedLen() uint64 {
method hash (line 93) | func (ccs *persistingChunkSource) hash() addr {
method index (line 99) | func (ccs *persistingChunkSource) index() tableIndex {
method reader (line 105) | func (ccs *persistingChunkSource) reader() io.Reader {
method calcReads (line 111) | func (ccs *persistingChunkSource) calcReads(reqs []getRecord, blockSiz...
method extract (line 117) | func (ccs *persistingChunkSource) extract(chunks chan<- extractRecord) {
type emptyChunkSource (line 123) | type emptyChunkSource struct
method has (line 125) | func (ecs emptyChunkSource) has(h addr) bool {
method hasMany (line 129) | func (ecs emptyChunkSource) hasMany(addrs []hasRecord) bool {
method get (line 133) | func (ecs emptyChunkSource) get(h addr, stats *Stats) []byte {
method getMany (line 137) | func (ecs emptyChunkSource) getMany(reqs []getRecord, foundChunks chan...
method count (line 141) | func (ecs emptyChunkSource) count() uint32 {
method uncompressedLen (line 145) | func (ecs emptyChunkSource) uncompressedLen() uint64 {
method hash (line 149) | func (ecs emptyChunkSource) hash() addr {
method index (line 153) | func (ecs emptyChunkSource) index() tableIndex {
method reader (line 157) | func (ecs emptyChunkSource) reader() io.Reader {
method calcReads (line 161) | func (ecs emptyChunkSource) calcReads(reqs []getRecord, blockSize uint...
method extract (line 165) | func (ecs emptyChunkSource) extract(chunks chan<- extractRecord) {}
FILE: go/nbs/persisting_chunk_source_test.go
function TestPersistingChunkStoreEmpty (line 13) | func TestPersistingChunkStoreEmpty(t *testing.T) {
type pausingFakeTablePersister (line 20) | type pausingFakeTablePersister struct
method Persist (line 25) | func (ftp pausingFakeTablePersister) Persist(mt *memTable, haver chunk...
function TestPersistingChunkStore (line 30) | func TestPersistingChunkStore(t *testing.T) {
FILE: go/nbs/root_tracker_test.go
function TestChunkStoreZeroValue (line 18) | func TestChunkStoreZeroValue(t *testing.T) {
function TestChunkStoreVersion (line 28) | func TestChunkStoreVersion(t *testing.T) {
function TestChunkStoreRebase (line 40) | func TestChunkStoreRebase(t *testing.T) {
function TestChunkStoreCommit (line 63) | func TestChunkStoreCommit(t *testing.T) {
function TestChunkStoreManifestAppearsAfterConstruction (line 88) | func TestChunkStoreManifestAppearsAfterConstruction(t *testing.T) {
function TestChunkStoreManifestFirstWriteByOtherProcess (line 104) | func TestChunkStoreManifestFirstWriteByOtherProcess(t *testing.T) {
function TestChunkStoreCommitOptimisticLockFail (line 121) | func TestChunkStoreCommitOptimisticLockFail(t *testing.T) {
function TestChunkStoreManifestPreemptiveOptimisticLockFail (line 135) | func TestChunkStoreManifestPreemptiveOptimisticLockFail(t *testing.T) {
function TestChunkStoreCommitLocksOutFetch (line 166) | func TestChunkStoreCommitLocksOutFetch(t *testing.T) {
function TestChunkStoreSerializeCommits (line 196) | func TestChunkStoreSerializeCommits(t *testing.T) {
function makeStoreWithFakes (line 242) | func makeStoreWithFakes(t *testing.T) (fm *fakeManifest, p tablePersiste...
function interloperWrite (line 251) | func interloperWrite(fm *fakeManifest, p tablePersister, rootChunk []byt...
function createMemTable (line 259) | func createMemTable(chunks [][]byte) *memTable {
function assertDataInStore (line 267) | func assertDataInStore(slices [][]byte, store chunks.ChunkStore, assert ...
type fakeManifest (line 274) | type fakeManifest struct
method Name (line 280) | func (fm *fakeManifest) Name() string { return fm.name }
method ParseIfExists (line 284) | func (fm *fakeManifest) ParseIfExists(stats *Stats, readHook func()) (...
method Update (line 298) | func (fm *fakeManifest) Update(lastLock addr, newContents manifestCont...
method set (line 309) | func (fm *fakeManifest) set(version string, lock addr, root hash.Hash,...
function newFakeTableSet (line 313) | func newFakeTableSet() tableSet {
function newFakeTablePersister (line 317) | func newFakeTablePersister() tablePersister {
type fakeTablePersister (line 321) | type fakeTablePersister struct
method Persist (line 326) | func (ftp fakeTablePersister) Persist(mt *memTable, haver chunkReader,...
method ConjoinAll (line 339) | func (ftp fakeTablePersister) ConjoinAll(sources chunkSources, stats *...
method Open (line 392) | func (ftp fakeTablePersister) Open(name addr, chunkCount uint32, stats...
function compactSourcesToBuffer (line 350) | func compactSourcesToBuffer(sources chunkSources) (name addr, data []byt...
type chunkSourceAdapter (line 398) | type chunkSourceAdapter struct
method hash (line 403) | func (csa chunkSourceAdapter) hash() addr {
method index (line 407) | func (csa chunkSourceAdapter) index() tableIndex {
FILE: go/nbs/s3_fake_test.go
type mockAWSError (line 25) | type mockAWSError
method Error (line 27) | func (m mockAWSError) Error() string { return string(m) }
method Code (line 28) | func (m mockAWSError) Code() string { return string(m) }
method Message (line 29) | func (m mockAWSError) Message() string { return string(m) }
method OrigErr (line 30) | func (m mockAWSError) OrigErr() error { return nil }
function makeFakeS3 (line 32) | func makeFakeS3(t *testing.T) *fakeS3 {
type fakeS3 (line 41) | type fakeS3 struct
method readerForTable (line 57) | func (m *fakeS3) readerForTable(name addr) chunkReader {
method AbortMultipartUpload (line 66) | func (m *fakeS3) AbortMultipartUpload(input *s3.AbortMultipartUploadIn...
method CreateMultipartUpload (line 81) | func (m *fakeS3) CreateMultipartUpload(input *s3.CreateMultipartUpload...
method UploadPart (line 99) | func (m *fakeS3) UploadPart(input *s3.UploadPartInput) (*s3.UploadPart...
method UploadPartCopy (line 122) | func (m *fakeS3) UploadPartCopy(input *s3.UploadPartCopyInput) (*s3.Up...
method CompleteMultipartUpload (line 156) | func (m *fakeS3) CompleteMultipartUpload(input *s3.CompleteMultipartUp...
method GetObject (line 176) | func (m *fakeS3) GetObject(input *s3.GetObjectInput) (*s3.GetObjectOut...
method PutObject (line 218) | func (m *fakeS3) PutObject(input *s3.PutObjectInput) (*s3.PutObjectOut...
type fakeS3Multipart (line 52) | type fakeS3Multipart struct
function parseRange (line 198) | func parseRange(hdr string, total int) (start, end int) {
FILE: go/nbs/s3_table_reader.go
constant s3RangePrefix (line 23) | s3RangePrefix = "bytes"
constant s3BlockSize (line 24) | s3BlockSize = (1 << 10) * 512
type s3TableReaderAt (line 27) | type s3TableReaderAt struct
method ReadAtWithStats (line 42) | func (s3tra *s3TableReaderAt) ReadAtWithStats(p []byte, off int64, sta...
type s3svc (line 32) | type s3svc interface
type s3ObjectReader (line 47) | type s3ObjectReader struct
method ReadAt (line 54) | func (s3or *s3ObjectReader) ReadAt(name addr, p []byte, off int64, sta...
method ReadFromEnd (line 81) | func (s3or *s3ObjectReader) ReadFromEnd(name addr, p []byte, stats *St...
method readRange (line 90) | func (s3or *s3ObjectReader) readRange(name addr, p []byte, rangeHeader...
function s3RangeHeader (line 76) | func s3RangeHeader(off, length int64) string {
function isConnReset (line 134) | func isConnReset(err error) bool {
FILE: go/nbs/s3_table_reader_test.go
function TestS3TableReaderAt (line 20) | func TestS3TableReaderAt(t *testing.T) {
type flakyS3 (line 70) | type flakyS3 struct
method GetObject (line 79) | func (fs3 *flakyS3) GetObject(input *s3.GetObjectInput) (output *s3.Ge...
function makeFlakyS3 (line 75) | func makeFlakyS3(svc s3svc) *flakyS3 {
type resettingReader (line 88) | type resettingReader struct
method Read (line 90) | func (rr resettingReader) Read(p []byte) (n int, err error) {
FILE: go/nbs/stats.go
type Stats (line 13) | type Stats struct
method Add (line 84) | func (s *Stats) Add(other Stats) {
method Delta (line 127) | func (s Stats) Delta(other Stats) Stats {
method String (line 172) | func (s Stats) String() string {
function NewStats (line 56) | func NewStats() *Stats {
FILE: go/nbs/stats_test.go
function TestStats (line 17) | func TestStats(t *testing.T) {
FILE: go/nbs/store.go
constant StorageVersion (line 26) | StorageVersion = "4"
constant defaultMemTableSize (line 28) | defaultMemTableSize uint64 = (1 << 20) * 128
constant defaultMaxTables (line 29) | defaultMaxTables = 192
constant defaultIndexCacheSize (line 31) | defaultIndexCacheSize = (1 << 20) * 8
constant defaultManifestCacheSize (line 32) | defaultManifestCacheSize = 1 << 23
constant preflushChunkCount (line 33) | preflushChunkCount = 8
function makeGlobalCaches (line 43) | func makeGlobalCaches() {
type NomsBlockStore (line 52) | type NomsBlockStore struct
method Put (line 135) | func (nbs *NomsBlockStore) Put(c chunks.Chunk) {
method addChunk (line 145) | func (nbs *NomsBlockStore) addChunk(h addr, data []byte) bool {
method Get (line 159) | func (nbs *NomsBlockStore) Get(h hash.Hash) chunks.Chunk {
method GetMany (line 185) | func (nbs *NomsBlockStore) GetMany(hashes hash.HashSet, foundChunks ch...
method CalcReads (line 233) | func (nbs *NomsBlockStore) CalcReads(hashes hash.HashSet, blockSize ui...
method extractChunks (line 248) | func (nbs *NomsBlockStore) extractChunks(chunkChan chan<- *chunks.Chun...
method Count (line 266) | func (nbs *NomsBlockStore) Count() uint32 {
method Has (line 278) | func (nbs *NomsBlockStore) Has(h hash.Hash) bool {
method HasMany (line 296) | func (nbs *NomsBlockStore) HasMany(hashes hash.HashSet) hash.HashSet {
method Rebase (line 349) | func (nbs *NomsBlockStore) Rebase() {
method Root (line 358) | func (nbs *NomsBlockStore) Root() hash.Hash {
method Commit (line 364) | func (nbs *NomsBlockStore) Commit(current, last hash.Hash) bool {
method updateManifest (line 412) | func (nbs *NomsBlockStore) updateManifest(current, last hash.Hash) err...
method Version (line 463) | func (nbs *NomsBlockStore) Version() string {
method Close (line 467) | func (nbs *NomsBlockStore) Close() (err error) {
method Stats (line 471) | func (nbs *NomsBlockStore) Stats() interface{} {
method StatsSummary (line 475) | func (nbs *NomsBlockStore) StatsSummary() string {
function NewAWSStore (line 68) | func NewAWSStore(table, ns, bucket string, s3 s3svc, ddb ddbsvc, memTabl...
function NewLocalStore (line 84) | func NewLocalStore(dir string, memTableSize uint64) *NomsBlockStore {
function newNomsBlockStore (line 93) | func newNomsBlockStore(mm manifestManager, p tablePersister, c conjoiner...
function newNomsBlockStoreWithContents (line 118) | func newNomsBlockStoreWithContents(mm manifestManager, mc manifestConten...
function toGetRecords (line 217) | func toGetRecords(hashes hash.HashSet) []getRecord {
function toHasRecords (line 332) | func toHasRecords(hashes hash.HashSet) []hasRecord {
FILE: go/nbs/table.go
constant addrSize (line 106) | addrSize uint64 = 20
constant addrPrefixSize (line 107) | addrPrefixSize uint64 = 8
constant addrSuffixSize (line 108) | addrSuffixSize = addrSize - addrPrefixSize
constant uint64Size (line 109) | uint64Size uint64 = 8
constant uint32Size (line 110) | uint32Size uint64 = 4
constant ordinalSize (line 111) | ordinalSize uint64 = uint32Size
constant lengthSize (line 112) | lengthSize uint64 = uint32Size
constant magicNumber (line 113) | magicNumber = "\xff\xb5\xd8\xc2\x24\x63\xee\x50"
constant magicNumberSize (line 114) | magicNumberSize uint64 = uint64(len(magicNumber))
constant footerSize (line 115) | footerSize = uint32Size + uint64Size + magicNumberSize
constant prefixTupleSize (line 116) | prefixTupleSize = addrPrefixSize + ordinalSize
constant checksumSize (line 117) | checksumSize uint64 = uint32Size
constant maxChunkLengthSize (line 118) | maxChunkLengthSize uint64 = binary.MaxVarintLen64
constant maxChunkSize (line 119) | maxChunkSize uint64 = 0xffffffff
function crc (line 124) | func crc(b []byte) uint32 {
function computeAddrDefault (line 128) | func computeAddrDefault(data []byte) addr {
type addr (line 137) | type addr
method String (line 141) | func (a addr) String() string {
method Prefix (line 145) | func (a addr) Prefix() uint64 {
method Checksum (line 149) | func (a addr) Checksum() uint32 {
function ParseAddr (line 153) | func ParseAddr(b []byte) (h addr) {
function ValidateAddr (line 158) | func ValidateAddr(s string) bool {
type addrSlice (line 163) | type addrSlice
method Len (line 165) | func (hs addrSlice) Len() int { return len(hs) }
method Less (line 166) | func (hs addrSlice) Less(i, j int) bool { return bytes.Compare(hs[i][:...
method Swap (line 167) | func (hs addrSlice) Swap(i, j int) { hs[i], hs[j] = hs[j], hs[i] }
type hasRecord (line 169) | type hasRecord struct
type hasRecordByPrefix (line 176) | type hasRecordByPrefix
method Len (line 178) | func (hs hasRecordByPrefix) Len() int { return len(hs) }
method Less (line 179) | func (hs hasRecordByPrefix) Less(i, j int) bool { return hs[i].prefix ...
method Swap (line 180) | func (hs hasRecordByPrefix) Swap(i, j int) { hs[i], hs[j] = hs[j]...
type hasRecordByOrder (line 182) | type hasRecordByOrder
method Len (line 184) | func (hs hasRecordByOrder) Len() int { return len(hs) }
method Less (line 185) | func (hs hasRecordByOrder) Less(i, j int) bool { return hs[i].order < ...
method Swap (line 186) | func (hs hasRecordByOrder) Swap(i, j int) { hs[i], hs[j] = hs[j],...
type getRecord (line 188) | type getRecord struct
type getRecordByPrefix (line 194) | type getRecordByPrefix
method Len (line 196) | func (hs getRecordByPrefix) Len() int { return len(hs) }
method Less (line 197) | func (hs getRecordByPrefix) Less(i, j int) bool { return hs[i].prefix ...
method Swap (line 198) | func (hs getRecordByPrefix) Swap(i, j int) { hs[i], hs[j] = hs[j]...
type extractRecord (line 200) | type extractRecord struct
type chunkReader (line 206) | type chunkReader interface
type chunkReadPlanner (line 216) | type chunkReadPlanner interface
type chunkSource (line 227) | type chunkSource interface
type chunkSources (line 237) | type chunkSources
FILE: go/nbs/table_persister.go
type tablePersister (line 23) | type tablePersister interface
type indexCache (line 39) | type indexCache struct
method lockEntry (line 52) | func (sic *indexCache) lockEntry(name addr) {
method unlockEntry (line 65) | func (sic *indexCache) unlockEntry(name addr) {
method get (line 76) | func (sic *indexCache) get(name addr) (tableIndex, bool) {
method put (line 83) | func (sic *indexCache) put(name addr, idx tableIndex) {
function newIndexCache (line 46) | func newIndexCache(size uint64) *indexCache {
type chunkSourcesByAscendingCount (line 88) | type chunkSourcesByAscendingCount
method Len (line 90) | func (csbc chunkSourcesByAscendingCount) Len() int { return len(csbc) }
method Less (line 91) | func (csbc chunkSourcesByAscendingCount) Less(i, j int) bool {
method Swap (line 99) | func (csbc chunkSourcesByAscendingCount) Swap(i, j int) { csbc[i], csb...
type chunkSourcesByDescendingDataSize (line 101) | type chunkSourcesByDescendingDataSize
method Len (line 103) | func (csbds chunkSourcesByDescendingDataSize) Len() int { return len(c...
method Less (line 104) | func (csbds chunkSourcesByDescendingDataSize) Less(i, j int) bool {
method Swap (line 112) | func (csbds chunkSourcesByDescendingDataSize) Swap(i, j int) { csbds[i...
type sourceWithSize (line 114) | type sourceWithSize struct
type compactionPlan (line 119) | type compactionPlan struct
method lengths (line 126) | func (cp compactionPlan) lengths() []byte {
method suffixes (line 131) | func (cp compactionPlan) suffixes() []byte {
function planConjoin (line 136) | func planConjoin(sources chunkSources, stats *Stats) (plan compactionPla...
function nameFromSuffixes (line 195) | func nameFromSuffixes(suffixes []byte) (name addr) {
function calcChunkDataLen (line 205) | func calcChunkDataLen(index tableIndex) uint64 {
FILE: go/nbs/table_persister_test.go
function TestPlanCompaction (line 13) | func TestPlanCompaction(t *testing.T) {
FILE: go/nbs/table_reader.go
type tableIndex (line 20) | type tableIndex struct
method prefixIdxToOrdinal (line 102) | func (ti tableIndex) prefixIdxToOrdinal(idx uint32) uint32 {
method prefixIdx (line 108) | func (ti tableIndex) prefixIdx(prefix uint64) (idx uint32) {
method ordinalSuffixMatches (line 126) | func (ti tableIndex) ordinalSuffixMatches(ordinal uint32, h addr) bool {
method lookupOrdinal (line 132) | func (ti tableIndex) lookupOrdinal(h addr) uint32 {
type tableReaderAt (line 28) | type tableReaderAt interface
type tableReader (line 34) | type tableReader struct
method hasMany (line 151) | func (tr tableReader) hasMany(addrs []hasRecord) (remaining bool) {
method count (line 192) | func (tr tableReader) count() uint32 {
method uncompressedLen (line 196) | func (tr tableReader) uncompressedLen() uint64 {
method index (line 200) | func (tr tableReader) index() tableIndex {
method has (line 205) | func (tr tableReader) has(h addr) bool {
method get (line 212) | func (tr tableReader) get(h addr, stats *Stats) (data []byte) {
method readAtOffsets (line 243) | func (tr tableReader) readAtOffsets(
method getMany (line 276) | func (tr tableReader) getMany(
method getManyAtOffsets (line 289) | func (tr tableReader) getManyAtOffsets(
method findOffsets (line 343) | func (tr tableReader) findOffsets(reqs []getRecord) (ors offsetRecSlic...
method parseChunk (line 401) | func (tr tableReader) parseChunk(buff []byte) []byte {
method calcReads (line 413) | func (tr tableReader) calcReads(reqs []getRecord, blockSize uint64) (r...
method extract (line 451) | func (tr tableReader) extract(chunks chan<- extractRecord) {
method reader (line 476) | func (tr tableReader) reader() io.Reader {
function parseTableIndex (line 41) | func parseTableIndex(buff []byte) tableIndex {
function computeOffsets (line 77) | func computeOffsets(count uint32, buff []byte) (lengths []uint32, offset...
function computePrefixes (line 90) | func computePrefixes(count uint32, buff []byte) (prefixes []uint64, ordi...
function newTableReader (line 146) | func newTableReader(index tableIndex, r tableReaderAt, blockSize uint64)...
type offsetRec (line 231) | type offsetRec struct
type offsetRecSlice (line 237) | type offsetRecSlice
method Len (line 239) | func (hs offsetRecSlice) Len() int { return len(hs) }
method Less (line 240) | func (hs offsetRecSlice) Less(i, j int) bool { return hs[i].offset < h...
method Swap (line 241) | func (hs offsetRecSlice) Swap(i, j int) { hs[i], hs[j] = hs[j], h...
function canReadAhead (line 383) | func canReadAhead(fRec offsetRec, fLength uint32, readStart, readEnd, bl...
type readerAdapter (line 480) | type readerAdapter struct
method Read (line 485) | func (ra *readerAdapter) Read(p []byte) (n int, err error) {
FILE: go/nbs/table_set.go
constant concurrentCompactions (line 14) | concurrentCompactions = 5
function newTableSet (line 16) | func newTableSet(persister tablePersister) tableSet {
type tableSet (line 21) | type tableSet struct
method has (line 27) | func (ts tableSet) has(h addr) bool {
method hasMany (line 39) | func (ts tableSet) hasMany(addrs []hasRecord) (remaining bool) {
method get (line 51) | func (ts tableSet) get(h addr, stats *Stats) []byte {
method getMany (line 66) | func (ts tableSet) getMany(reqs []getRecord, foundChunks chan *chunks....
method calcReads (line 88) | func (ts tableSet) calcReads(reqs []getRecord, blockSize uint64) (read...
method count (line 110) | func (ts tableSet) count() uint32 {
method uncompressedLen (line 120) | func (ts tableSet) uncompressedLen() uint64 {
method physicalLen (line 130) | func (ts tableSet) physicalLen() uint64 {
method Size (line 143) | func (ts tableSet) Size() int {
method Novel (line 149) | func (ts tableSet) Novel() int {
method Upstream (line 154) | func (ts tableSet) Upstream() int {
method Prepend (line 160) | func (ts tableSet) Prepend(mt *memTable, stats *Stats) tableSet {
method extract (line 173) | func (ts tableSet) extract(chunks chan<- extractRecord) {
method Flatten (line 185) | func (ts tableSet) Flatten() (flattened tableSet) {
method Rebase (line 202) | func (ts tableSet) Rebase(specs []tableSpec, stats *Stats) tableSet {
method ToSpecs (line 242) | func (ts tableSet) ToSpecs() []tableSpec {
FILE: go/nbs/table_set_test.go
function TestTableSetPrependEmpty (line 15) | func TestTableSetPrependEmpty(t *testing.T) {
function TestTableSetPrepend (line 20) | func TestTableSetPrepend(t *testing.T) {
function TestTableSetToSpecsExcludesEmptyTable (line 41) | func TestTableSetToSpecsExcludesEmptyTable(t *testing.T) {
function TestTableSetFlattenExcludesEmptyTable (line 61) | func TestTableSetFlattenExcludesEmptyTable(t *testing.T) {
function TestTableSetExtract (line 81) | func TestTableSetExtract(t *testing.T) {
function TestTableSetRebase (line 109) | func TestTableSetRebase(t *testing.T) {
function TestTableSetPhysicalLen (line 136) | func TestTableSetPhysicalLen(t *testing.T) {
FILE: go/nbs/table_test.go
function buildTable (line 21) | func buildTable(chunks [][]byte) ([]byte, addr) {
function TestSimple (line 40) | func TestSimple(t *testing.T) {
function assertChunksInReader (line 71) | func assertChunksInReader(chunks [][]byte, r chunkReader, assert *assert...
function assertChunksNotInReader (line 77) | func assertChunksNotInReader(chunks [][]byte, r chunkReader, assert *ass...
function TestHasMany (line 83) | func TestHasMany(t *testing.T) {
function TestHasManySequentialPrefix (line 109) | func TestHasManySequentialPrefix(t *testing.T) {
function TestGetMany (line 153) | func TestGetMany(t *testing.T) {
function TestCalcReads (line 188) | func TestCalcReads(t *testing.T) {
function TestExtract (line 219) | func TestExtract(t *testing.T) {
function Test65k (line 244) | func Test65k(t *testing.T) {
function computeAddrCommonPrefix (line 278) | func computeAddrCommonPrefix(data []byte) addr {
function doTestNGetMany (line 290) | func doTestNGetMany(t *testing.T, count int) {
function Test65kGetMany (line 328) | func Test65kGetMany(t *testing.T) {
function Test2kGetManyCommonPrefix (line 332) | func Test2kGetManyCommonPrefix(t *testing.T) {
function TestEmpty (line 341) | func TestEmpty(t *testing.T) {
FILE: go/nbs/table_writer.go
type tableWriter (line 19) | type tableWriter struct
method addChunk (line 72) | func (tw *tableWriter) addChunk(h addr, data []byte) bool {
method finish (line 108) | func (tw *tableWriter) finish() (uncompressedLength uint64, blockAddr ...
method writeIndex (line 131) | func (tw *tableWriter) writeIndex() {
method writeFooter (line 165) | func (tw *tableWriter) writeFooter() {
type snappyEncoder (line 30) | type snappyEncoder interface
type realSnappyEncoder (line 34) | type realSnappyEncoder struct
method Encode (line 36) | func (r realSnappyEncoder) Encode(dst, src []byte) []byte {
function maxTableSize (line 40) | func maxTableSize(numChunks, totalData uint64) uint64 {
function indexSize (line 48) | func indexSize(numChunks uint32) uint64 {
function lengthsOffset (line 52) | func lengthsOffset(numChunks uint32) uint64 {
function suffixesOffset (line 56) | func suffixesOffset(numChunks uint32) uint64 {
function newTableWriter (line 61) | func newTableWriter(buff []byte, snapper snappyEncoder) *tableWriter {
type prefixIndexRec (line 119) | type prefixIndexRec struct
type prefixIndexSlice (line 125) | type prefixIndexSlice
method Len (line 127) | func (hs prefixIndexSlice) Len() int { return len(hs) }
method Less (line 128) | func (hs prefixIndexSlice) Less(i, j int) bool { return hs[i].prefix <...
method Swap (line 129) | func (hs prefixIndexSlice) Swap(i, j int) { hs[i], hs[j] = hs[j],...
function writeFooter (line 169) | func writeFooter(dst []byte, chunkCount uint32, uncData uint64) (consume...
FILE: go/nbs/test/manifest_clobber.go
function main (line 15) | func main() {
FILE: go/ngql/query.go
constant atKey (line 19) | atKey = "at"
constant countKey (line 20) | countKey = "count"
constant elementsKey (line 21) | elementsKey = "elements"
constant entriesKey (line 22) | entriesKey = "entries"
constant keyKey (line 23) | keyKey = "key"
constant keysKey (line 24) | keysKey = "keys"
constant rootKey (line 25) | rootKey = "root"
constant rootQueryKey (line 26) | rootQueryKey = "Root"
constant scalarValue (line 27) | scalarValue = "scalarValue"
constant sizeKey (line 28) | sizeKey = "size"
constant targetHashKey (line 29) | targetHashKey = "targetHash"
constant targetValueKey (line 30) | targetValueKey = "targetValue"
constant throughKey (line 31) | throughKey = "through"
constant valueKey (line 32) | valueKey = "value"
constant valuesKey (line 33) | valuesKey = "values"
constant vrwKey (line 34) | vrwKey = "vrw"
function NewRootQueryObject (line 39) | func NewRootQueryObject(rootValue types.Value, tm *TypeMap) *graphql.Obj...
method NewRootQueryObject (line 46) | func (tc *TypeConverter) NewRootQueryObject(rootValue types.Value) *grap...
function NewContext (line 64) | func NewContext(vrw types.ValueReader) context.Context {
function Query (line 70) | func Query(rootValue types.Value, query string, vrw types.ValueReadWrite...
function queryWithSchemaConfig (line 76) | func queryWithSchemaConfig(rootValue types.Value, query string, schemaCo...
function Error (line 92) | func Error(err error, w io.Writer) {
FILE: go/ngql/query_test.go
type QueryGraphQLSuite (line 23) | type QueryGraphQLSuite struct
method SetupTest (line 37) | func (suite *QueryGraphQLSuite) SetupTest() {
method assertQueryResult (line 41) | func (suite *QueryGraphQLSuite) assertQueryResult(v types.Value, q, ex...
method TestScalars (line 47) | func (suite *QueryGraphQLSuite) TestScalars() {
method TestStructBasic (line 63) | func (suite *QueryGraphQLSuite) TestStructBasic() {
method TestEmptyStruct (line 76) | func (suite *QueryGraphQLSuite) TestEmptyStruct() {
method TestEmbeddedStruct (line 82) | func (suite *QueryGraphQLSuite) TestEmbeddedStruct() {
method TestListBasic (line 96) | func (suite *QueryGraphQLSuite) TestListBasic() {
method TestListOfStruct (line 126) | func (suite *QueryGraphQLSuite) TestListOfStruct() {
method TestListOfStructWithOptionalFields (line 147) | func (suite *QueryGraphQLSuite) TestListOfStructWithOptionalFields() {
method TestSetBasic (line 170) | func (suite *QueryGraphQLSuite) TestSetBasic() {
method TestSetOfStruct (line 206) | func (suite *QueryGraphQLSuite) TestSetOfStruct() {
method TestMapBasic (line 227) | func (suite *QueryGraphQLSuite) TestMapBasic() {
method TestMapOfStruct (line 246) | func (suite *QueryGraphQLSuite) TestMapOfStruct() {
method TestRef (line 267) | func (suite *QueryGraphQLSuite) TestRef() {
method TestListOfUnionOfStructs (line 288) | func (suite *QueryGraphQLSuite) TestListOfUnionOfStructs() {
method TestListOfUnionOfStructsConflictingFieldTypes (line 310) | func (suite *QueryGraphQLSuite) TestListOfUnionOfStructsConflictingFie...
method TestListOfUnionOfScalars (line 331) | func (suite *QueryGraphQLSuite) TestListOfUnionOfScalars() {
method TestCyclicStructs (line 341) | func (suite *QueryGraphQLSuite) TestCyclicStructs() {
method TestCyclicStructsWithUnion (line 362) | func (suite *QueryGraphQLSuite) TestCyclicStructsWithUnion() {
method TestNestedCollection (line 434) | func (suite *QueryGraphQLSuite) TestNestedCollection() {
method TestLoFi (line 452) | func (suite *QueryGraphQLSuite) TestLoFi() {
method TestError (line 461) | func (suite *QueryGraphQLSuite) TestError() {
method TestMapArgs (line 468) | func (suite *QueryGraphQLSuite) TestMapArgs() {
method TestMapKeysArg (line 568) | func (suite *QueryGraphQLSuite) TestMapKeysArg() {
method TestSetArgs (line 602) | func (suite *QueryGraphQLSuite) TestSetArgs() {
method TestMapValues (line 704) | func (suite *QueryGraphQLSuite) TestMapValues() {
method TestMapKeys (line 782) | func (suite *QueryGraphQLSuite) TestMapKeys() {
method TestMapNullable (line 860) | func (suite *QueryGraphQLSuite) TestMapNullable() {
method TestStructWithOptionalField (line 881) | func (suite *QueryGraphQLSuite) TestStructWithOptionalField() {
method TestMutationScalarArgs (line 917) | func (suite *QueryGraphQLSuite) TestMutationScalarArgs() {
method TestMutationWeirdosArgs (line 938) | func (suite *QueryGraphQLSuite) TestMutationWeirdosArgs() {
method assertMutationTypes (line 953) | func (suite *QueryGraphQLSuite) assertMutationTypes(query, expected st...
method TestMutationCollectionArgs (line 976) | func (suite *QueryGraphQLSuite) TestMutationCollectionArgs() {
method TestMapWithComplexKeys (line 1022) | func (suite *QueryGraphQLSuite) TestMapWithComplexKeys() {
method TestSetWithComplexKeys (line 1077) | func (suite *QueryGraphQLSuite) TestSetWithComplexKeys() {
method TestInputToNomsValue (line 1116) | func (suite *QueryGraphQLSuite) TestInputToNomsValue() {
method TestErrorsInInputType (line 1196) | func (suite *QueryGraphQLSuite) TestErrorsInInputType() {
method TestVariables (line 1222) | func (suite *QueryGraphQLSuite) TestVariables() {
method TestNameFunc (line 1384) | func (suite *QueryGraphQLSuite) TestNameFunc() {
function TestQueryGraphQL (line 28) | func TestQueryGraphQL(t *testing.T) {
function newTestValueStore (line 32) | func newTestValueStore() *types.ValueStore {
function TestGetListElementsWithSet (line 1488) | func TestGetListElementsWithSet(t *testing.T) {
function TestNoErrorOnNonCyclicTypeRefsInputType (line 1506) | func TestNoErrorOnNonCyclicTypeRefsInputType(t *testing.T) {
function TestErrorOnCyclicTypeRefsInputType (line 1524) | func TestErrorOnCyclicTypeRefsInputType(t *testing.T) {
FILE: go/ngql/types.go
type TypeConverter (line 21) | type TypeConverter struct
method getTypeName (line 37) | func (tc *TypeConverter) getTypeName(nomsType *types.Type) string {
method getInputTypeName (line 41) | func (tc *TypeConverter) getInputTypeName(nomsType *types.Type) string {
method NomsTypeToGraphQLType (line 47) | func (tc *TypeConverter) NomsTypeToGraphQLType(nomsType *types.Type) g...
method NomsTypeToGraphQLInputType (line 54) | func (tc *TypeConverter) NomsTypeToGraphQLInputType(nomsType *types.Ty...
method scalarToValue (line 108) | func (tc *TypeConverter) scalarToValue(nomsType *types.Type, scalarTyp...
method nomsTypeToGraphQLType (line 137) | func (tc *TypeConverter) nomsTypeToGraphQLType(nomsType *types.Type, b...
method nomsTypeToGraphQLInputType (line 205) | func (tc *TypeConverter) nomsTypeToGraphQLInputType(nomsType *types.Ty...
method unionToGQLUnion (line 268) | func (tc *TypeConverter) unionToGQLUnion(nomsType *types.Type) *graphq...
method structToGQLObject (line 306) | func (tc *TypeConverter) structToGQLObject(nomsType *types.Type) *grap...
method listAndSetToGraphQLInputObject (line 342) | func (tc *TypeConverter) listAndSetToGraphQLInputObject(nomsType *type...
method mapToGraphQLInputObject (line 351) | func (tc *TypeConverter) mapToGraphQLInputObject(nomsType *types.Type)...
method structToGQLInputObject (line 368) | func (tc *TypeConverter) structToGQLInputObject(nomsType *types.Type) ...
method mapEntryToGraphQLObject (line 653) | func (tc *TypeConverter) mapEntryToGraphQLObject(keyType, valueType gr...
method mapEntryToGraphQLInputObject (line 677) | func (tc *TypeConverter) mapEntryToGraphQLInputObject(keyType, valueTy...
method listAndSetToGraphQLObject (line 795) | func (tc *TypeConverter) listAndSetToGraphQLObject(nomsType *types.Typ...
method mapToGraphQLObject (line 849) | func (tc *TypeConverter) mapToGraphQLObject(nomsType *types.Type) *gra...
method refToGraphQLObject (line 930) | func (tc *TypeConverter) refToGraphQLObject(nomsType *types.Type) *gra...
function NewTypeConverter (line 27) | func NewTypeConverter() *TypeConverter {
type NameFunc (line 35) | type NameFunc
type TypeMap (line 60) | type TypeMap
type typeMapKey (line 62) | type typeMapKey struct
function NewTypeMap (line 69) | func NewTypeMap() *TypeMap {
type graphQLTypeMode (line 82) | type graphQLTypeMode
constant inputMode (line 85) | inputMode graphQLTypeMode = iota
constant outputMode (line 86) | outputMode
type getFieldFn (line 94) | type getFieldFn
type getSubvaluesFn (line 99) | type getSubvaluesFn
function isScalar (line 121) | func isScalar(nomsType *types.Type) bool {
function NomsTypeToGraphQLType (line 132) | func NomsTypeToGraphQLType(nomsType *types.Type, boxedIfScalar bool, tm ...
function NomsTypeToGraphQLInputType (line 200) | func NomsTypeToGraphQLInputType(nomsType *types.Type, tm *TypeMap) (grap...
function isEmptyNomsUnion (line 263) | func isEmptyNomsUnion(nomsType *types.Type) bool {
function getListElements (line 407) | func getListElements(vrw types.ValueReadWriter, v types.Value, args map[...
function getSetElements (line 458) | func getSetElements(vrw types.ValueReadWriter, v types.Value, args map[s...
function getCollectionArgs (line 505) | func getCollectionArgs(vrw types.ValueReadWriter, col types.Collection, ...
type mapAppender (line 552) | type mapAppender
type mapiter (line 554) | type mapiter interface
function getMapElements (line 560) | func getMapElements(vrw types.ValueReadWriter, v types.Value, args map[s...
function getCountArg (line 611) | func getCountArg(count uint64, args map[string]interface{}) (c uint64, s...
function getThroughArg (line 629) | func getThroughArg(vrw types.ValueReadWriter, nomsKeyType *types.Type, a...
type iteratorFactory (line 636) | type iteratorFactory struct
type mapEntry (line 642) | type mapEntry struct
function DefaultNameFunc (line 694) | func DefaultNameFunc(nomsType *types.Type, isInputType bool) string {
function GetTypeName (line 702) | func GetTypeName(nomsType *types.Type) string {
function GetInputTypeName (line 708) | func GetInputTypeName(nomsType *types.Type) string {
function getTypeName (line 712) | func getTypeName(nomsType *types.Type, suffix string) string {
function argsWithSize (line 783) | func argsWithSize() graphql.Fields {
function mapAppendKey (line 912) | func mapAppendKey(slice []interface{}, k, v types.Value) []interface{} {
function mapAppendValue (line 916) | func mapAppendValue(slice []interface{}, k, v types.Value) []interface{} {
function mapAppendEntry (line 920) | func mapAppendEntry(slice []interface{}, k, v types.Value) []interface{} {
function MaybeGetScalar (line 958) | func MaybeGetScalar(v types.Value) interface{} {
function InputToNomsValue (line 976) | func InputToNomsValue(vrw types.ValueReadWriter, arg interface{}, nomsTy...
type mapIteratorForKeys (line 1024) | type mapIteratorForKeys struct
method Valid (line 1030) | func (it *mapIteratorForKeys) Valid() bool {
method Entry (line 1034) | func (it *mapIteratorForKeys) Entry() (k, v types.Value) {
method Next (line 1043) | func (it *mapIteratorForKeys) Next() bool {
type setFirstIterator (line 1048) | type setFirstIterator struct
method Next (line 1052) | func (it *setFirstIterator) Next() types.Value {
method SkipTo (line 1056) | func (it *setFirstIterator) SkipTo(v types.Value) types.Value {
type mapFirstIterator (line 1060) | type mapFirstIterator struct
method Valid (line 1064) | func (it *mapFirstIterator) Valid() bool {
method Entry (line 1068) | func (it *mapFirstIterator) Entry() (types.Value, types.Value) {
method Next (line 1077) | func (it *mapFirstIterator) Next() bool {
FILE: go/nomdl/lexer.go
type lexer (line 12) | type lexer struct
method next (line 17) | func (lex *lexer) next() rune {
method peek (line 27) | func (lex *lexer) peek() rune {
method pos (line 36) | func (lex *lexer) pos() scanner.Position {
method tokenText (line 43) | func (lex *lexer) tokenText() string {
method eat (line 50) | func (lex *lexer) eat(expected rune) rune {
method eatIf (line 56) | func (lex *lexer) eatIf(expected rune) bool {
method check (line 65) | func (lex *lexer) check(expected, actual rune) {
method tokenMismatch (line 71) | func (lex *lexer) tokenMismatch(expected, actual rune) {
method unexpectedToken (line 75) | func (lex *lexer) unexpectedToken(actual rune) {
function raiseSyntaxError (line 79) | func raiseSyntaxError(msg string, pos scanner.Position) {
type syntaxError (line 86) | type syntaxError struct
method Error (line 91) | func (e syntaxError) Error() string {
function catchSyntaxError (line 95) | func catchSyntaxError(f func()) (errRes error) {
FILE: go/nomdl/parser.go
type Parser (line 20) | type Parser struct
method ensureAtEnd (line 78) | func (p *Parser) ensureAtEnd() {
method parseType (line 130) | func (p *Parser) parseType() *types.Type {
method parseTypeWithToken (line 135) | func (p *Parser) parseTypeWithToken(tok rune, tokenText string) *types...
method parseSingleType (line 155) | func (p *Parser) parseSingleType() *types.Type {
method parseSingleTypeWithToken (line 160) | func (p *Parser) parseSingleTypeWithToken(tok rune, tokenText string) ...
method parseStructType (line 195) | func (p *Parser) parseStructType() *types.Type {
method parseSingleElemType (line 229) | func (p *Parser) parseSingleElemType(allowEmptyUnion bool) *types.Type {
method parseCycleType (line 239) | func (p *Parser) parseCycleType() *types.Type {
method parseMapType (line 247) | func (p *Parser) parseMapType() *types.Type {
method parseValue (line 312) | func (p *Parser) parseValue() types.Value {
method parseFloat (line 362) | func (p *Parser) parseFloat() types.Number {
method parseList (line 368) | func (p *Parser) parseList() types.List {
method parseSet (line 386) | func (p *Parser) parseSet() types.Set {
method parseMap (line 405) | func (p *Parser) parseMap() types.Map {
method blobString (line 427) | func (p *Parser) blobString(s string) []byte {
method parseBlob (line 447) | func (p *Parser) parseBlob() types.Blob {
method parseStruct (line 467) | func (p *Parser) parseStruct() types.Struct {
type ParserOptions (line 26) | type ParserOptions struct
function New (line 32) | func New(vrw types.ValueReadWriter, r io.Reader, options ParserOptions) ...
function ParseType (line 43) | func ParseType(code string) (typ *types.Type, err error) {
function MustParseType (line 54) | func MustParseType(code string) *types.Type {
function Parse (line 61) | func Parse(vrw types.ValueReadWriter, code string) (v types.Value, err e...
function MustParse (line 72) | func MustParse(vrw types.ValueReadWriter, code string) types.Value {
FILE: go/nomdl/parser_test.go
function newTestValueStore (line 17) | func newTestValueStore() *types.ValueStore {
function assertParseType (line 22) | func assertParseType(t *testing.T, code string, expected *types.Type) {
function assertParse (line 30) | func assertParse(t *testing.T, vrw types.ValueReadWriter, code string, e...
function assertParseError (line 40) | func assertParseError(t *testing.T, code, msg string) {
function TestSimpleTypes (line 55) | func TestSimpleTypes(t *testing.T) {
function TestWhitespace (line 64) | func TestWhitespace(t *testing.T) {
function TestComments (line 71) | func TestComments(t *testing.T) {
function TestCompoundTypes (line 78) | func TestCompoundTypes(t *testing.T) {
function TestStructTypes (line 122) | func TestStructTypes(t *testing.T) {
function TestUnionTypes (line 198) | func TestUnionTypes(t *testing.T) {
function TestValuePrimitives (line 227) | func TestValuePrimitives(t *testing.T) {
function TestValueList (line 271) | func TestValueList(t *testing.T) {
function TestEmptyValuesInEditors (line 292) | func TestEmptyValuesInEditors(t *testing.T) {
function TestValueSet (line 308) | func TestValueSet(t *testing.T) {
function TestValueMap (line 330) | func TestValueMap(t *testing.T) {
function TestValueType (line 354) | func TestValueType(t *testing.T) {
function TestValueStruct (line 361) | func TestValueStruct(t *testing.T) {
function TestValueBlob (line 390) | func TestValueBlob(t *testing.T) {
function TestRoundTrips (line 425) | func TestRoundTrips(t *testing.T) {
FILE: go/perf/hash-perf-rig/main.go
function main (line 23) | func main() {
FILE: go/perf/suite/suite.go
type PerfSuite (line 113) | type PerfSuite struct
method Suite (line 343) | func (suite *PerfSuite) Suite() *PerfSuite {
method NewAssert (line 348) | func (suite *PerfSuite) NewAssert() *assert.Assertions {
method TempFile (line 354) | func (suite *PerfSuite) TempFile() *os.File {
method TempDir (line 364) | func (suite *PerfSuite) TempDir() string {
method tempPrefix (line 371) | func (suite *PerfSuite) tempPrefix() string {
method Pause (line 377) | func (suite *PerfSuite) Pause(fn func()) {
method OpenGlob (line 388) | func (suite *PerfSuite) OpenGlob(pattern ...string) []io.Reader {
method CloseGlob (line 405) | func (suite *PerfSuite) CloseGlob(files []io.Reader) {
method getEnvironment (line 422) | func (suite *PerfSuite) getEnvironment(vrw types.ValueReadWriter) type...
method getGitHead (line 459) | func (suite *PerfSuite) getGitHead(dir string) string {
method StartRemoteDatabase (line 492) | func (suite *PerfSuite) StartRemoteDatabase() (host string, stopFn fun...
type SetupRepSuite (line 139) | type SetupRepSuite interface
type TearDownRepSuite (line 144) | type TearDownRepSuite interface
type perfSuiteT (line 148) | type perfSuiteT interface
type environment (line 152) | type environment struct
type timeInfo (line 160) | type timeInfo struct
type testRep (line 164) | type testRep
type nopWriter (line 166) | type nopWriter struct
method Write (line 168) | func (r nopWriter) Write(p []byte) (int, error) {
function Run (line 173) | func Run(datasetID string, t *testing.T, suiteT perfSuiteT) {
function callSafe (line 412) | func callSafe(name string, fun reflect.Value, args ...interface{}) error {
FILE: go/perf/suite/suite_test.go
type testSuite (line 18) | type testSuite struct
method TestNonEmptyPaths (line 27) | func (s *testSuite) TestNonEmptyPaths() {
method TestDatabase (line 34) | func (s *testSuite) TestDatabase() {
method TestTempFile (line 41) | func (s *testSuite) TestTempFile() {
method TestGlob (line 46) | func (s *testSuite) TestGlob() {
method TestPause (line 74) | func (s *testSuite) TestPause() {
method TestFoo (line 80) | func (s *testSuite) TestFoo() {
method TestBar (line 85) | func (s *testSuite) TestBar() {
method Test01Abc (line 90) | func (s *testSuite) Test01Abc() {
method Test02Def (line 95) | func (s *testSuite) Test02Def() {
method testNothing (line 100) | func (s *testSuite) testNothing() {
method Testimate (line 105) | func (s *testSuite) Testimate() {
method SetupTest (line 110) | func (s *testSuite) SetupTest() {
method TearDownTest (line 114) | func (s *testSuite) TearDownTest() {
method SetupRep (line 118) | func (s *testSuite) SetupRep() {
method TearDownRep (line 122) | func (s *testSuite) TearDownRep() {
method SetupSuite (line 126) | func (s *testSuite) SetupSuite() {
method TearDownSuite (line 130) | func (s *testSuite) TearDownSuite() {
method waitForSmidge (line 134) | func (s *testSuite) waitForSmidge() {
function TestSuite (line 139) | func TestSuite(t *testing.T) {
function TestSuiteWithMem (line 143) | func TestSuiteWithMem(t *testing.T) {
function runTestSuite (line 148) | func runTestSuite(t *testing.T, mem bool) {
function TestPrefixFlag (line 252) | func TestPrefixFlag(t *testing.T) {
function TestRunFlag (line 283) | func TestRunFlag(t *testing.T) {
FILE: go/sloppy/sloppy.go
constant maxOffsetPOT (line 12) | maxOffsetPOT = uint16(12)
constant maxTableSize (line 13) | maxTableSize = 1 << 14
constant maxLength (line 14) | maxLength = 1<<12 - 1
constant tableMask (line 15) | tableMask = maxTableSize - 1
constant shift (line 16) | shift = uint32(20)
type Sloppy (line 63) | type Sloppy struct
method Update (line 90) | func (sl *Sloppy) Update(src []byte) {
method Reset (line 143) | func (sl *Sloppy) Reset() {
method dontCopy (line 175) | func (sl *Sloppy) dontCopy(src []byte, from, to int) bool {
method maybeCopy (line 185) | func (sl *Sloppy) maybeCopy(src []byte) bool {
function New (line 75) | func New(f func(b byte) bool) *Sloppy {
function copyLongEnough (line 154) | func copyLongEnough(off, len uint16) bool {
type encoder (line 198) | type encoder interface
type binaryEncoder (line 203) | type binaryEncoder struct
method emitLiteral (line 207) | func (be binaryEncoder) emitLiteral(b byte) bool {
method emitCopy (line 211) | func (be binaryEncoder) emitCopy(offset, length uint16) bool {
function fbhash (line 233) | func fbhash(u uint32) uint32 {
function load32 (line 237) | func load32(b []byte, i int) uint32 {
FILE: go/sloppy/sloppy_test.go
type testSloppyEncoder (line 16) | type testSloppyEncoder struct
method emitLiteral (line 21) | func (tt *testSloppyEncoder) emitLiteral(b byte) bool {
method emitCopy (line 26) | func (tt *testSloppyEncoder) emitCopy(offset, length uint16) bool {
method emitString (line 32) | func (tt *testSloppyEncoder) emitString() {
function TestCopyLongEnough (line 39) | func TestCopyLongEnough(t *testing.T) {
function TestSloppySimple (line 66) | func TestSloppySimple(t *testing.T) {
function TestSloppyContinuation (line 113) | func TestSloppyContinuation(t *testing.T) {
function TestBinaryEncoder (line 159) | func TestBinaryEncoder(t *testing.T) {
function TestSnappyVsSloppy (line 182) | func TestSnappyVsSloppy(t *testing.T) {
FILE: go/spec/absolute_path.go
type AbsolutePath (line 26) | type AbsolutePath struct
method Resolve (line 86) | func (p AbsolutePath) Resolve(db datas.Database) (val types.Value) {
method IsEmpty (line 105) | func (p AbsolutePath) IsEmpty() bool {
method String (line 109) | func (p AbsolutePath) String() (str string) {
function NewAbsolutePath (line 40) | func NewAbsolutePath(str string) (AbsolutePath, error) {
function ReadAbsolutePaths (line 128) | func ReadAbsolutePaths(db datas.Database, paths ...string) ([]types.Valu...
FILE: go/spec/absolute_path_test.go
function TestAbsolutePathToAndFromString (line 19) | func TestAbsolutePathToAndFromString(t *testing.T) {
function TestAbsolutePaths (line 33) | func TestAbsolutePaths(t *testing.T) {
function TestReadAbsolutePaths (line 84) | func TestReadAbsolutePaths(t *testing.T) {
function TestAbsolutePathParseErrors (line 112) | func TestAbsolutePathParseErrors(t *testing.T) {
FILE: go/spec/commit_meta.go
constant CommitMetaDateFormat (line 17) | CommitMetaDateFormat = time.RFC3339
function CreateCommitMetaStruct (line 31) | func CreateCommitMetaStruct(db datas.Database, date, message string, key...
FILE: go/spec/commit_meta_test.go
function isEmptyStruct (line 17) | func isEmptyStruct(s types.Struct) bool {
function TestCreateCommitMetaStructBasic (line 21) | func TestCreateCommitMetaStructBasic(t *testing.T) {
function TestCreateCommitMetaStructFromFlags (line 30) | func TestCreateCommitMetaStructFromFlags(t *testing.T) {
function TestCreateCommitMetaStructFromArgs (line 47) | func TestCreateCommitMetaStructFromArgs(t *testing.T) {
function TestCreateCommitMetaStructFromFlagsAndArgs (line 64) | func TestCreateCommitMetaStructFromFlagsAndArgs(t *testing.T) {
function TestCreateCommitMetaStructBadDate (line 87) | func TestCreateCommitMetaStructBadDate(t *testing.T) {
function TestCreateCommitMetaStructBadMetaStrings (line 110) | func TestCreateCommitMetaStructBadMetaStrings(t *testing.T) {
function setCommitMetaFlags (line 151) | func setCommitMetaFlags(date, message, kvStrings string) {
function resetCommitMetaFlags (line 157) | func resetCommitMetaFlags() {
FILE: go/spec/spec.go
constant Separator (line 28) | Separator = "::"
type ProtocolImpl (line 36) | type ProtocolImpl interface
type SpecOptions (line 44) | type SpecOptions struct
type Spec (line 53) | type Spec struct
method String (line 158) | func (sp Spec) String() string {
method GetDatabase (line 173) | func (sp Spec) GetDatabase() datas.Database {
method GetDataset (line 184) | func (sp Spec) GetDataset() (ds datas.Dataset) {
method GetValue (line 193) | func (sp Spec) GetValue() (val types.Value) {
method Href (line 204) | func (sp Spec) Href() string {
method Pin (line 217) | func (sp Spec) Pin() (Spec, bool) {
method Close (line 243) | func (sp Spec) Close() error {
method createDatabase (line 253) | func (sp Spec) createDatabase() datas.Database {
method NewChunkStore (line 271) | func (sp Spec) NewChunkStore() chunks.ChunkStore {
function newSpec (line 71) | func newSpec(dbSpec string, opts SpecOptions) (Spec, error) {
function ForDatabase (line 86) | func ForDatabase(spec string) (Spec, error) {
function ForDatabaseOpts (line 91) | func ForDatabaseOpts(spec string, opts SpecOptions) (Spec, error) {
function ForDataset (line 96) | func ForDataset(spec string) (Spec, error) {
function ForDatasetOpts (line 101) | func ForDatasetOpts(spec string, opts SpecOptions) (Spec, error) {
function ForPath (line 130) | func ForPath(spec string) (Spec, error) {
function ForPathOpts (line 135) | func ForPathOpts(spec string, opts SpecOptions) (Spec, error) {
function parseDatabaseSpec (line 297) | func parseDatabaseSpec(spec string) (protocol, name string, err error) {
function splitDatabaseSpec (line 355) | func splitDatabaseSpec(spec string) (string, string, error) {
FILE: go/spec/spec_test.go
function TestMemDatabaseSpec (line 24) | func TestMemDatabaseSpec(t *testing.T) {
function TestMemDatasetSpec (line 41) | func TestMemDatasetSpec(t *testing.T) {
function TestMemHashPathSpec (line 63) | func TestMemHashPathSpec(t *testing.T) {
function TestMemDatasetPathSpec (line 83) | func TestMemDatasetPathSpec(t *testing.T) {
function TestNBSDatabaseSpec (line 104) | func TestNBSDatabaseSpec(t *testing.T) {
function TestCloseSpecWithoutOpen (line 160) | func TestCloseSpecWithoutOpen(t *testing.T) {
function TestHref (line 166) | func TestHref(t *testing.T) {
function TestForDatabase (line 194) | func TestForDatabase(t *testing.T) {
function TestForDataset (line 265) | func TestForDataset(t *testing.T) {
function TestForPath (line 346) | func TestForPath(t *testing.T) {
function TestPinPathSpec (line 402) | func TestPinPathSpec(t *testing.T) {
function TestPinDatasetSpec (line 428) | func TestPinDatasetSpec(t *testing.T) {
function TestAlreadyPinnedPathSpec (line 458) | func TestAlreadyPinnedPathSpec(t *testing.T) {
function TestMultipleSpecsSameNBS (line 468) | func TestMultipleSpecsSameNBS(t *testing.T) {
function TestAcccessingInvalidSpec (line 489) | func TestAcccessingInvalidSpec(t *testing.T) {
type testProtocol (line 517) | type testProtocol struct
method NewChunkStore (line 521) | func (t *testProtocol) NewChunkStore(sp Spec) (chunks.ChunkStore, erro...
method NewDatabase (line 525) | func (t *testProtocol) NewDatabase(sp Spec) (datas.Database, error) {
function TestExternalProtocol (line 532) | func TestExternalProtocol(t *testing.T) {
function TestMkDirAll (line 558) | func TestMkDirAll(t *testing.T) {
function TestNetworkError (line 568) | func TestNetworkError(t *testing.T) {
FILE: go/spec/util.go
function CreateDatabaseSpecString (line 12) | func CreateDatabaseSpecString(protocol, db string) string {
function CreateValueSpecString (line 16) | func CreateValueSpecString(protocol, db, path string) string {
function CreateHashSpecString (line 22) | func CreateHashSpecString(protocol, db string, h hash.Hash) string {
FILE: go/types/blob.go
type Blob (line 18) | type Blob struct
method Edit (line 30) | func (b Blob) Edit() *BlobEditor {
method ReadAt (line 35) | func (b Blob) ReadAt(p []byte, off int64) (n int, err error) {
method Reader (line 81) | func (b Blob) Reader() *BlobReader {
method Copy (line 85) | func (b Blob) Copy(w io.Writer) (n int64) {
method CopyReadAhead (line 92) | func (b Blob) CopyReadAhead(w io.Writer, chunkSize uint64, concurrency...
method Concat (line 135) | func (b Blob) Concat(other Blob) Blob {
method newChunker (line 142) | func (b Blob) newChunker(cur *sequenceCursor, vrw ValueReadWriter) *se...
method asSequence (line 146) | func (b Blob) asSequence() sequence {
method Value (line 151) | func (b Blob) Value() Value {
method WalkValues (line 155) | func (b Blob) WalkValues(cb ValueCallback) {
function newBlob (line 22) | func newBlob(seq sequence) Blob {
function NewEmptyBlob (line 26) | func NewEmptyBlob(vrw ValueReadWriter) Blob {
type BlobReader (line 158) | type BlobReader struct
method Read (line 163) | func (cbr *BlobReader) Read(p []byte) (n int, err error) {
method Seek (line 169) | func (cbr *BlobReader) Seek(offset int64, whence int) (int64, error) {
function makeBlobLeafChunkFn (line 191) | func makeBlobLeafChunkFn(vrw ValueReadWriter) makeChunkFn {
function chunkBlobLeaf (line 204) | func chunkBlobLeaf(vrw ValueReadWriter, buff []byte) (Collection, ordere...
function NewBlob (line 211) | func NewBlob(vrw ValueReadWriter, rs ...io.Reader) Blob {
function readBlobsP (line 215) | func readBlobsP(vrw ValueReadWriter, rs ...io.Reader) Blob {
function readBlob (line 245) | func readBlob(r io.Reader, vrw ValueReadWriter) Blob {
FILE: go/types/blob_editor.go
type BlobEditor (line 17) | type BlobEditor struct
method Kind (line 27) | func (be *BlobEditor) Kind() NomsKind {
method Value (line 31) | func (be *BlobEditor) Value() Value {
method Blob (line 35) | func (be *BlobEditor) Blob() Blob {
method Len (line 134) | func (be *BlobEditor) Len() uint64 {
method Splice (line 143) | func (be *BlobEditor) Splice(idx uint64, deleteCount uint64, insert []...
method Seek (line 193) | func (be *BlobEditor) Seek(offset int64, whence int) (int64, error) {
method Read (line 219) | func (be *BlobEditor) Read(p []byte) (n int, err error) {
method Write (line 286) | func (be *BlobEditor) Write(p []byte) (n int, err error) {
function NewBlobEditor (line 23) | func NewBlobEditor(b Blob) *BlobEditor {
function collapseBlobEdit (line 81) | func collapseBlobEdit(newEdit, edit *blobEdit) bool {
function adjustBlobIdx (line 297) | func adjustBlobIdx(idx uint64, e *blobEdit) uint64 {
type blobEdit (line 301) | type blobEdit struct
FILE: go/types/blob_editor_test.go
function TestBlobReadWriteFuzzer (line 21) | func TestBlobReadWriteFuzzer(t *testing.T) {
FILE: go/types/blob_leaf_sequence.go
type blobLeafSequence (line 9) | type blobLeafSequence struct
method writeTo (line 29) | func (bl blobLeafSequence) writeTo(w nomsWriter) {
method data (line 35) | func (bl blobLeafSequence) data() []byte {
method getCompareFn (line 40) | func (bl blobLeafSequence) getCompareFn(other sequence) compareFn {
method getItem (line 49) | func (bl blobLeafSequence) getItem(idx int) sequenceItem {
method typeOf (line 54) | func (bl blobLeafSequence) typeOf() *Type {
function newBlobLeafSequence (line 13) | func newBlobLeafSequence(vrw ValueReadWriter, data []byte) sequence {
FILE: go/types/blob_test.go
function randomBuff (line 19) | func randomBuff(powOfTwo uint) []byte {
type blobTestSuite (line 27) | type blobTestSuite struct
method TestRandomRead (line 86) | func (suite *blobTestSuite) TestRandomRead() {
function newBlobTestSuite (line 32) | func newBlobTestSuite(size uint, expectChunkCount int, expectPrependChun...
function TestBlobSuite4K (line 69) | func TestBlobSuite4K(t *testing.T) {
function TestBlobSuite64K (line 73) | func TestBlobSuite64K(t *testing.T) {
function TestBlobSuite256K (line 77) | func TestBlobSuite256K(t *testing.T) {
function TestBlobSuite1M (line 81) | func TestBlobSuite1M(t *testing.T) {
type testReader (line 140) | type testReader struct
method Read (line 145) | func (r *testReader) Read(p []byte) (n int, err error) {
function TestBlobFromReaderThatReturnsDataAndError (line 164) | func TestBlobFromReaderThatReturnsDataAndError(t *testing.T) {
function TestBlobSplice (line 180) | func TestBlobSplice(t *testing.T) {
function TestBlobConcat (line 208) | func TestBlobConcat(t *testing.T) {
function TestBlobNewParallel (line 249) | func TestBlobNewParallel(t *testing.T) {
function TestStreamingParallelBlob (line 283) | func TestStreamingParallelBlob(t *testing.T) {
FILE: go/types/bool.go
type Bool (line 12) | type Bool
method Value (line 15) | func (b Bool) Value() Value {
method Equals (line 19) | func (b Bool) Equals(other Value) bool {
method Less (line 23) | func (b Bool) Less(other Value) bool {
method Hash (line 30) | func (b Bool) Hash() hash.Hash {
method WalkValues (line 34) | func (b Bool) WalkValues(cb ValueCallback) {
method WalkRefs (line 37) | func (b Bool) WalkRefs(cb RefCallback) {
method typeOf (line 40) | func (b Bool) typeOf() *Type {
method Kind (line 44) | func (b Bool) Kind() NomsKind {
method valueReadWriter (line 48) | func (b Bool) valueReadWriter() ValueReadWriter {
method writeTo (line 52) | func (b Bool) writeTo(w nomsWriter) {
method valueBytes (line 57) | func (b Bool) valueBytes() []byte {
FILE: go/types/codec.go
constant initialBufferSize (line 15) | initialBufferSize = 2048
type valueBytes (line 17) | type valueBytes interface
function EncodeValue (line 21) | func EncodeValue(v Value) chunks.Chunk {
function DecodeFromBytes (line 34) | func DecodeFromBytes(data []byte, vrw ValueReadWriter) Value {
function decodeFromBytesWithValidation (line 41) | func decodeFromBytesWithValidation(data []byte, vrw ValueReadWriter) Val...
function DecodeValue (line 50) | func DecodeValue(c chunks.Chunk, vrw ValueReadWriter) Value {
type nomsWriter (line 55) | type nomsWriter interface
type binaryNomsReader (line 67) | type binaryNomsReader struct
method pos (line 72) | func (b *binaryNomsReader) pos() uint32 {
method readUint8 (line 76) | func (b *binaryNomsReader) readUint8() uint8 {
method peekUint8 (line 82) | func (b *binaryNomsReader) peekUint8() uint8 {
method skipUint8 (line 86) | func (b *binaryNomsReader) skipUint8() {
method peekKind (line 90) | func (b *binaryNomsReader) peekKind() NomsKind {
method readKind (line 94) | func (b *binaryNomsReader) readKind() NomsKind {
method skipKind (line 98) | func (b *binaryNomsReader) skipKind() {
method readCount (line 102) | func (b *binaryNomsReader) readCount() uint64 {
method skipCount (line 108) | func (b *binaryNomsReader) skipCount() {
method readNumber (line 113) | func (b *binaryNomsReader) readNumber() Number {
method skipNumber (line 122) | func (b *binaryNomsReader) skipNumber() {
method readBool (line 129) | func (b *binaryNomsReader) readBool() bool {
method skipBool (line 133) | func (b *binaryNomsReader) skipBool() {
method readString (line 137) | func (b *binaryNomsReader) readString() string {
method skipString (line 145) | func (b *binaryNomsReader) skipString() {
method readHash (line 150) | func (b *binaryNomsReader) readHash() hash.Hash {
method skipHash (line 157) | func (b *binaryNomsReader) skipHash() {
method byteSlice (line 161) | func (b *binaryNomsReader) byteSlice(start, end uint32) []byte {
type binaryNomsWriter (line 165) | type binaryNomsWriter struct
method data (line 174) | func (b *binaryNomsWriter) data() []byte {
method reset (line 178) | func (b *binaryNomsWriter) reset() {
method ensureCapacity (line 182) | func (b *binaryNomsWriter) ensureCapacity(n uint32) {
method writeBytes (line 198) | func (b *binaryNomsWriter) writeBytes(v []byte) {
method writeUint8 (line 205) | func (b *binaryNomsWriter) writeUint8(v uint8) {
method writeCount (line 211) | func (b *binaryNomsWriter) writeCount(v uint64) {
method writeNumber (line 217) | func (b *binaryNomsWriter) writeNumber(v Number) {
method writeBool (line 226) | func (b *binaryNomsWriter) writeBool(v bool) {
method writeString (line 234) | func (b *binaryNomsWriter) writeString(v string) {
method writeHash (line 243) | func (b *binaryNomsWriter) writeHash(h hash.Hash) {
method writeRaw (line 249) | func (b *binaryNomsWriter) writeRaw(buff []byte) {
function newBinaryNomsWriter (line 170) | func newBinaryNomsWriter() binaryNomsWriter {
FILE: go/types/codec_test.go
function TestCodecWriteNumber (line 13) | func TestCodecWriteNumber(t *testing.T) {
function TestCodecReadNumber (line 36) | func TestCodecReadNumber(t *testing.T) {
FILE: go/types/collection.go
type Collection (line 7) | type Collection interface
FILE: go/types/collection_test.go
type collectionTestSuite (line 9) | type collectionTestSuite struct
method TestType (line 25) | func (suite *collectionTestSuite) TestType() {
method TestLen (line 29) | func (suite *collectionTestSuite) TestLen() {
method TestEquals (line 34) | func (suite *collectionTestSuite) TestEquals() {
method TestChunkCountAndType (line 40) | func (suite *collectionTestSuite) TestChunkCountAndType() {
method TestRoundTripAndValidate (line 48) | func (suite *collectionTestSuite) TestRoundTripAndValidate() {
method TestPrependChunkDiff (line 52) | func (suite *collectionTestSuite) TestPrependChunkDiff() {
method TestAppendChunkDiff (line 57) | func (suite *collectionTestSuite) TestAppendChunkDiff() {
type validateFn (line 22) | type validateFn
type deltaFn (line 23) | type deltaFn
function deriveCollectionHeight (line 62) | func deriveCollectionHeight(c Collection) uint64 {
function getRefHeightOfCollection (line 66) | func getRefHeightOfCollection(c Collection) uint64 {
FILE: go/types/common_supertype.go
function ContainCommonSupertype (line 21) | func ContainCommonSupertype(a, b *Type) bool {
function containCommonSupertypeImpl (line 26) | func containCommonSupertypeImpl(a, b *Type, aVisited, bVisited []*Type) ...
function unionsIntersect (line 51) | func unionsIntersect(a, b *Type, aVisited, bVisited []*Type) bool {
function typeList (line 64) | func typeList(t *Type) typeSlice {
function containersIntersect (line 71) | func containersIntersect(kind NomsKind, a, b *Type, aVisited, bVisited [...
function mapsIntersect (line 76) | func mapsIntersect(a, b *Type, aVisited, bVisited []*Type) bool {
function structsIntersect (line 100) | func structsIntersect(a, b *Type, aVisited, bVisited []*Type) bool {
FILE: go/types/common_supertype_test.go
function TestContainCommonSupertype (line 9) | func TestContainCommonSupertype(t *testing.T) {
FILE: go/types/compare_test.go
function TestCompareTotalOrdering (line 17) | func TestCompareTotalOrdering(t *testing.T) {
function TestCompareEmpties (line 51) | func TestCompareEmpties(t *testing.T) {
function TestCompareDifferentPrimitiveTypes (line 59) | func TestCompareDifferentPrimitiveTypes(t *testing.T) {
function TestComparePrimitives (line 87) | func TestComparePrimitives(t *testing.T) {
function TestCompareEncodedKeys (line 115) | func TestCompareEncodedKeys(t *testing.T) {
function encode (line 132) | func encode(v Value) []byte {
function compareInts (line 138) | func compareInts(i, j int) (res int) {
FILE: go/types/edit_distance.go
constant DEFAULT_MAX_SPLICE_MATRIX_SIZE (line 16) | DEFAULT_MAX_SPLICE_MATRIX_SIZE = 2e7
constant SPLICE_UNASSIGNED (line 17) | SPLICE_UNASSIGNED = math.MaxUint64
constant UNCHANGED (line 19) | UNCHANGED = 0
constant UPDATED (line 20) | UPDATED = 1
constant INSERTED (line 21) | INSERTED = 2
constant REMOVED (line 22) | REMOVED = 3
type Splice (line 27) | type Splice struct
method String (line 36) | func (s Splice) String() string {
type EditDistanceEqualsFn (line 34) | type EditDistanceEqualsFn
function uint64Min (line 40) | func uint64Min(a, b uint64) uint64 {
function uint64Min3 (line 47) | func uint64Min3(a, b, c uint64) uint64 {
function reverse (line 60) | func reverse(numbers []uint64) []uint64 {
function addSplice (line 68) | func addSplice(splices []Splice, s Splice) []Splice {
function calcSplices (line 76) | func calcSplices(previousLength uint64, currentLength uint64, maxSpliceM...
function calcEditDistances (line 165) | func calcEditDistances(eqFn EditDistanceEqualsFn, previousStart uint64, ...
function operationsFromEditDistances (line 203) | func operationsFromEditDistances(distances [][]uint64) []uint64 {
function sharedPrefix (line 248) | func sharedPrefix(eqFn EditDistanceEqualsFn, searchLength uint64) uint64 {
function sharedSuffix (line 258) | func sharedSuffix(eqFn EditDistanceEqualsFn, previousLength uint64, curr...
FILE: go/types/edit_distance_test.go
function assertDiff (line 13) | func assertDiff(assert *assert.Assertions, last []uint64, current []uint...
function TestEditDistanceAppend (line 19) | func TestEditDistanceAppend(t *testing.T) {
function TestEditDistancePrepend (line 29) | func TestEditDistancePrepend(t *testing.T) {
function TestEditDistanceChopFromEnd (line 39) | func TestEditDistanceChopFromEnd(t *testing.T) {
function TestEditDistanceChopFromStart (line 49) | func TestEditDistanceChopFromStart(t *testing.T) {
function TestEditDistanceChopFromMiddle (line 59) | func TestEditDistanceChopFromMiddle(t *testing.T) {
function TestEditDistanceA (line 69) | func TestEditDistanceA(t *testing.T) {
function TestEditDistanceRemoveABunch (line 82) | func TestEditDistanceRemoveABunch(t *testing.T) {
function TestEditDistanceAddABunch (line 97) | func TestEditDistanceAddABunch(t *testing.T) {
function TestEditDistanceUpdateABunch (line 113) | func TestEditDistanceUpdateABunch(t *testing.T) {
function TestEditDistanceLeftOverlap (line 127) | func TestEditDistanceLeftOverlap(t *testing.T) {
function TestEditDistanceRightOverlap (line 139) | func TestEditDistanceRightOverlap(t *testing.T) {
function TestEditDistanceWithin (line 151) | func TestEditDistanceWithin(t *testing.T) {
function TestEditDistanceWithout (line 163) | func TestEditDistanceWithout(t *testing.T) {
function TestEditDistanceMix1 (line 175) | func TestEditDistanceMix1(t *testing.T) {
function TestEditDistanceReverse (line 191) | func TestEditDistanceReverse(t *testing.T) {
FILE: go/types/encode_human_readable.go
type HRSCommenter (line 33) | type HRSCommenter interface
function RegisterHRSCommenter (line 50) | func RegisterHRSCommenter(typename, unique string, commenter HRSCommente...
function UnregisterHRSCommenter (line 63) | func UnregisterHRSCommenter(typename, unique string) {
function GetHRSCommenters (line 75) | func GetHRSCommenters(typename string) []HRSCommenter {
type hrsWriter (line 87) | type hrsWriter struct
method maybeWriteIndentation (line 95) | func (w *hrsWriter) maybeWriteIndentation() {
method write (line 104) | func (w *hrsWriter) write(s string) {
method indent (line 114) | func (w *hrsWriter) indent() {
method outdent (line 118) | func (w *hrsWriter) outdent() {
method newLine (line 122) | func (w *hrsWriter) newLine() {
method Write (line 172) | func (w *hrsWriter) Write(v Value) {
method writeStruct (line 299) | func (w *hrsWriter) writeStruct(v Struct) {
method writeSize (line 303) | func (w *hrsWriter) writeSize(v Value) {
method writeType (line 316) | func (w *hrsWriter) writeType(t *Type, seenStructs map[*Type]struct{}) {
method writeStructType (line 364) | func (w *hrsWriter) writeStructType(t *Type, seenStructs map[*Type]str...
type hexWriter (line 129) | type hexWriter struct
method Write (line 136) | func (w *hexWriter) Write(p []byte) (n int, err error) {
type hrsStructWriter (line 254) | type hrsStructWriter struct
method name (line 259) | func (w hrsStructWriter) name(n string) {
method count (line 277) | func (w hrsStructWriter) count(c uint64) {
method fieldName (line 283) | func (w hrsStructWriter) fieldName(n string) {
method fieldValue (line 288) | func (w hrsStructWriter) fieldValue(v Value) {
method end (line 294) | func (w hrsStructWriter) end() {
function encodedValueFormatMaxLines (line 396) | func encodedValueFormatMaxLines(v Value, floatFormat byte, maxLines uint...
function encodedValueFormat (line 407) | func encodedValueFormat(v Value, floatFormat byte) string {
function EncodedIndexValue (line 415) | func EncodedIndexValue(v Value) string {
function EncodedValue (line 420) | func EncodedValue(v Value) string {
function EncodedValueMaxLines (line 426) | func EncodedValueMaxLines(v Value, maxLines uint32) string {
function WriteEncodedValue (line 431) | func WriteEncodedValue(w io.Writer, v Value) error {
function WriteEncodedValueMaxLines (line 439) | func WriteEncodedValueMaxLines(w io.Writer, v Value, maxLines uint32) er...
FILE: go/types/encode_human_readable_test.go
function assertWriteHRSEqual (line 17) | func assertWriteHRSEqual(t *testing.T, expected string, v Value) {
function TestWriteHumanReadablePrimitiveValues (line 25) | func TestWriteHumanReadablePrimitiveValues(t *testing.T) {
function TestWriteHumanReadableRef (line 54) | func TestWriteHumanReadableRef(t *testing.T) {
function TestWriteHumanReadableCollections (line 62) | func TestWriteHumanReadableCollections(t *testing.T) {
function TestWriteHumanReadableNested (line 88) | func TestWriteHumanReadableNested(t *testing.T) {
function TestWriteHumanReadableStruct (line 116) | func TestWriteHumanReadableStruct(t *testing.T) {
function TestWriteHumanReadableListOfStruct (line 124) | func TestWriteHumanReadableListOfStruct(t *testing.T) {
function TestWriteHumanReadableBlob (line 150) | func TestWriteHumanReadableBlob(t *testing.T) {
function TestWriteHumanReadableListOfBlob (line 187) | func TestWriteHumanReadableListOfBlob(t *testing.T) {
function TestWriteHumanReadableType (line 201) | func TestWriteHumanReadableType(t *testing.T) {
function TestRecursiveStruct (line 218) | func TestRecursiveStruct(t *testing.T) {
function TestUnresolvedRecursiveStruct (line 258) | func TestUnresolvedRecursiveStruct(t *testing.T) {
type errorWriter (line 274) | type errorWriter struct
method Write (line 278) | func (w *errorWriter) Write(p []byte) (int, error) {
function TestWriteHumanReadableWriterError (line 282) | func TestWriteHumanReadableWriterError(t *testing.T) {
function TestEmptyCollections (line 289) | func TestEmptyCollections(t *testing.T) {
function TestEncodedValueMaxLines (line 306) | func TestEncodedValueMaxLines(t *testing.T) {
function TestWriteHumanReadableStructOptionalFields (line 319) | func TestWriteHumanReadableStructOptionalFields(t *testing.T) {
type TestCommenter (line 326) | type TestCommenter struct
method Comment (line 331) | func (c TestCommenter) Comment(v Value) string {
function TestRegisterCommenter (line 338) | func TestRegisterCommenter(t *testing.T) {
FILE: go/types/encoding_test.go
function toBinaryNomsReaderData (line 19) | func toBinaryNomsReaderData(data []interface{}) []byte {
function assertEncoding (line 47) | func assertEncoding(t *testing.T, expect []interface{}, v Value) {
function TestRoundTrips (line 59) | func TestRoundTrips(t *testing.T) {
function TestNonFiniteNumbers (line 119) | func TestNonFiniteNumbers(tt *testing.T) {
function TestWritePrimitives (line 133) | func TestWritePrimitives(t *testing.T) {
function TestWriteSimpleBlob (line 177) | func TestWriteSimpleBlob(t *testing.T) {
function TestWriteList (line 188) | func TestWriteList(t *testing.T) {
function TestWriteListOfList (line 199) | func TestWriteListOfList(t *testing.T) {
function TestWriteSet (line 213) | func TestWriteSet(t *testing.T) {
function TestWriteSetOfSet (line 225) | func TestWriteSetOfSet(t *testing.T) {
function TestWriteMap (line 238) | func TestWriteMap(t *testing.T) {
function TestWriteMapOfMap (line 250) | func TestWriteMapOfMap(t *testing.T) {
function TestWriteCompoundBlob (line 263) | func TestWriteCompoundBlob(t *testing.T) {
function TestWriteEmptyStruct (line 284) | func TestWriteEmptyStruct(t *testing.T) {
function TestWriteStruct (line 293) | func TestWriteStruct(t *testing.T) {
function TestWriteStructTooMuchData (line 303) | func TestWriteStructTooMuchData(t *testing.T) {
function TestWriteStructWithList (line 315) | func TestWriteStructWithList(t *testing.T) {
function TestWriteStructWithStruct (line 337) | func TestWriteStructWithStruct(t *testing.T) {
function TestWriteStructWithBlob (line 355) | func TestWriteStructWithBlob(t *testing.T) {
function TestWriteCompoundList (line 367) | func TestWriteCompoundList(t *testing.T) {
function TestWriteCompoundSet (line 385) | func TestWriteCompoundSet(t *testing.T) {
function TestWriteCompoundSetOfBlobs (line 404) | func TestWriteCompoundSetOfBlobs(t *testing.T) {
function TestWriteListOfUnion (line 434) | func TestWriteListOfUnion(t *testing.T) {
function TestWriteListOfStruct (line 452) | func TestWriteListOfStruct(t *testing.T) {
function TestWriteListOfUnionWithType (line 464) | func TestWriteListOfUnionWithType(t *testing.T) {
function TestWriteRef (line 486) | func TestWriteRef(t *testing.T) {
function TestWriteListOfTypes (line 497) | func TestWriteListOfTypes(t *testing.T) {
function nomsTestWriteRecursiveStruct (line 509) | func nomsTestWriteRecursiveStruct(t *testing.T) {
function TestWriteUnionList (line 527) | func TestWriteUnionList(t *testing.T) {
function TestWriteEmptyUnionList (line 539) | func TestWriteEmptyUnionList(t *testing.T) {
type bogusType (line 550) | type bogusType
method Value (line 552) | func (bg bogusType) Value() Value { return bg }
method Equals (line 553) | func (bg bogusType) Equals(other Value) bool { return false }
method Less (line 554) | func (bg bogusType) Less(other Value) bool { return false }
method Hash (line 555) | func (bg bogusType) Hash() hash.Hash { return hash.Hash{} }
method WalkValues (line 556) | func (bg bogusType) WalkValues(cb ValueCallback) {}
method WalkRefs (line 557) | func (bg bogusType) WalkRefs(cb RefCallback) {}
method Kind (line 558) | func (bg bogusType) Kind() NomsKind {
method typeOf (line 561) | func (bg bogusType) typeOf() *Type {
method writeTo (line 564) | func (bg bogusType) writeTo(nomsWriter) {
function TestBogusValueWithUnresolvedCycle (line 568) | func TestBogusValueWithUnresolvedCycle(t *testing.T) {
FILE: go/types/equals_test.go
function TestValueEquals (line 14) | func TestValueEquals(t *testing.T) {
FILE: go/types/get_hash.go
function getHash (line 11) | func getHash(v Value) hash.Hash {
function getHashNoOverride (line 18) | func getHashNoOverride(v Value) hash.Hash {
function EnsureHash (line 22) | func EnsureHash(h *hash.Hash, v Value) hash.Hash {
FILE: go/types/graph_builder.go
type GraphBuilder (line 49) | type GraphBuilder struct
method MapSet (line 71) | func (b *GraphBuilder) MapSet(keys []Value, k Value, v Value) {
method SetInsert (line 81) | func (b *GraphBuilder) SetInsert(keys []Value, v Value) {
method ListAppend (line 93) | func (b *GraphBuilder) ListAppend(keys []Value, v Value) {
method Build (line 111) | func (b *GraphBuilder) Build() Value {
method pushNewKeyOnStack (line 198) | func (b *GraphBuilder) pushNewKeyOnStack(key Value, kind NomsKind) {
method popKeyFromStack (line 216) | func (b *GraphBuilder) popKeyFromStack() {
method appendItemToCurrentTopOfStack (line 225) | func (b *GraphBuilder) appendItemToCurrentTopOfStack(kind NomsKind, it...
function NewGraphBuilder (line 58) | func NewGraphBuilder(vrw ValueReadWriter, rootKind NomsKind) *GraphBuild...
function newGraphBuilder (line 62) | func newGraphBuilder(vrw ValueReadWriter, opcStore opCacheStore, rootKin...
type graphOpContainer (line 100) | type graphOpContainer struct
type graphStackElem (line 231) | type graphStackElem struct
method done (line 274) | func (e *graphStackElem) done() Collection {
method String (line 309) | func (e *graphStackElem) String() string {
type graphStack (line 237) | type graphStack struct
method push (line 241) | func (s *graphStack) push(e *graphStackElem) {
method pop (line 245) | func (s *graphStack) pop() *graphStackElem {
method top (line 252) | func (s *graphStack) top() *graphStackElem {
method len (line 257) | func (s *graphStack) len() int {
method lastIdx (line 261) | func (s *graphStack) lastIdx() int {
method String (line 265) | func (s graphStack) String() string {
function commonPrefixCount (line 289) | func commonPrefixCount(stack graphStack, keys ValueSlice) int {
FILE: go/types/graph_builder_test.go
function TestGraphBuilderFindIndex (line 15) | func TestGraphBuilderFindIndex(t *testing.T) {
type testGraphOp (line 43) | type testGraphOp struct
function SafeEquals (line 49) | func SafeEquals(v1, v2 Value) bool {
function TestGraphBuilderEncodeDecodeAsKey (line 59) | func TestGraphBuilderEncodeDecodeAsKey(t *testing.T) {
function TestGraphBuilderEncodeDecodeAsValue (line 95) | func TestGraphBuilderEncodeDecodeAsValue(t *testing.T) {
function TestGraphBuilderMapSetGraphOp (line 125) | func TestGraphBuilderMapSetGraphOp(t *testing.T) {
function createTestMap (line 160) | func createTestMap(vrw ValueReadWriter, levels, avgSize int, valGen func...
function valGen (line 213) | func valGen() Value {
function dupSlice (line 229) | func dupSlice(s ValueSlice) ValueSlice {
function shuffle (line 235) | func shuffle(a []testGraphOp) {
function TestGraphBuilderNestedMapSet (line 245) | func TestGraphBuilderNestedMapSet(t *testing.T) {
function ExampleGraphBuilder_Build (line 306) | func ExampleGraphBuilder_Build() {
FILE: go/types/incremental_test.go
function getTestVals (line 15) | func getTestVals(vrw ValueReadWriter) []Value {
function isEncodedOutOfLine (line 28) | func isEncodedOutOfLine(v Value) int {
function TestIncrementalLoadList (line 36) | func TestIncrementalLoadList(t *testing.T) {
function SkipTestIncrementalLoadSet (line 66) | func SkipTestIncrementalLoadSet(t *testing.T) {
function SkipTestIncrementalLoadMap (line 87) | func SkipTestIncrementalLoadMap(t *testing.T) {
function SkipTestIncrementalAddRef (line 109) | func SkipTestIncrementalAddRef(t *testing.T) {
FILE: go/types/indexed_sequence_diff.go
function sendSpliceChange (line 7) | func sendSpliceChange(changes chan<- Splice, closeChan <-chan struct{}, ...
function indexedSequenceDiff (line 16) | func indexedSequenceDiff(last sequence, lastOffset uint64, current seque...
FILE: go/types/indexed_sequences.go
function newListMetaSequence (line 12) | func newListMetaSequence(level uint64, tuples []metaTuple, vrw ValueRead...
function newBlobMetaSequence (line 16) | func newBlobMetaSequence(level uint64, tuples []metaTuple, vrw ValueRead...
function advanceCursorToOffset (line 28) | func advanceCursorToOffset(cur *sequenceCursor, idx uint64) uint64 {
function newIndexedMetaSequenceChunkFn (line 59) | func newIndexedMetaSequenceChunkFn(kind NomsKind, vrw ValueReadWriter) m...
function orderedKeyFromSum (line 81) | func orderedKeyFromSum(msd []metaTuple) orderedKey {
function LoadLeafNodes (line 92) | func LoadLeafNodes(cols []Collection, startIdx, endIdx uint64) ([]Collec...
FILE: go/types/leaf_sequence.go
type leafSequence (line 13) | type leafSequence struct
method values (line 67) | func (seq leafSequence) values() []Value {
method valuesSlice (line 71) | func (seq leafSequence) valuesSlice(from, to uint64) []Value {
method getCompareFnHelper (line 84) | func (seq leafSequence) getCompareFnHelper(other leafSequence) compare...
method getCompareFn (line 95) | func (seq leafSequence) getCompareFn(other sequence) compareFn {
method typeOf (line 99) | func (seq leafSequence) typeOf() *Type {
method numLeaves (line 122) | func (seq leafSequence) numLeaves() uint64 {
method getChildSequence (line 126) | func (seq leafSequence) getChildSequence(idx int) sequence {
method treeLevel (line 130) | func (seq leafSequence) treeLevel() uint64 {
method isLeaf (line 134) | func (seq leafSequence) isLeaf() bool {
method cumulativeNumberOfLeaves (line 138) | func (seq leafSequence) cumulativeNumberOfLeaves(idx int) uint64 {
method getCompositeChildSequence (line 142) | func (seq leafSequence) getCompositeChildSequence(start uint64, length...
method getItem (line 146) | func (seq leafSequence) getItem(idx int) sequenceItem {
function newLeafSequence (line 17) | func newLeafSequence(vrw ValueReadWriter, buff []byte, offsets []uint32,...
function newLeafSequenceFromValues (line 21) | func newLeafSequenceFromValues(kind NomsKind, vrw ValueReadWriter, vs .....
function readLeafSequence (line 41) | func readLeafSequence(dec *valueDecoder) leafSequence {
function skipLeafSequence (line 48) | func skipLeafSequence(dec *valueDecoder) ([]uint32, uint64) {
function getValuesPerIdx (line 151) | func getValuesPerIdx(kind NomsKind) int {
FILE: go/types/less.go
type kindAndHash (line 11) | type kindAndHash interface
function valueLess (line 16) | func valueLess(v1, v2 kindAndHash) bool {
FILE: go/types/list.go
type List (line 22) | type List struct
method Edit (line 56) | func (l List) Edit() *ListEditor {
method asSequence (line 62) | func (l List) asSequence() sequence {
method Value (line 67) | func (l List) Value() Value {
method WalkValues (line 71) | func (l List) WalkValues(cb ValueCallback) {
method Get (line 79) | func (l List) Get(idx uint64) Value {
method Concat (line 88) | func (l List) Concat(other List) List {
method Iter (line 97) | func (l
Condensed preview — 339 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (2,782K chars).
[
{
"path": ".dockerignore",
"chars": 62,
"preview": ".git\ndoc\ncodecov.yml\nCONTRIBUTING.md\nLICENSE\nREADME.md\nsamples"
},
{
"path": ".gitignore",
"chars": 76,
"preview": "noms.iml\n*.pyc\n*.swp\n.vscode\n.idea\n.noms\n.nomsconfig\n.DS_Store\nnode_modules\n"
},
{
"path": ".travis.yml",
"chars": 1513,
"preview": "language: go\ngo:\n- 1.x\nos:\n- linux\nenv:\n- GO111MODULE=on\nbefore_install:\n# gox simplifies building for multiple architec"
},
{
"path": "CONTRIBUTING.md",
"chars": 4548,
"preview": "Contributing to Noms\n====================\n\n## Install Go\n\nFirst setup Go on your machine per https://golang.org/doc/inst"
},
{
"path": "Dockerfile",
"chars": 434,
"preview": "FROM golang:latest AS build\n\nENV NOMS_SRC=$GOPATH/src/github.com/attic-labs/noms\nENV CGO_ENABLED=1\nENV GOOS=linux\nENV DO"
},
{
"path": "HACKING.md",
"chars": 440,
"preview": "# Prerequisites\n\n* [Go 1.13 or later](https://golang.org/dl/)\n* Mac or Linux (Noms isn't currently supported on Windows)"
},
{
"path": "LICENSE",
"chars": 11357,
"preview": " Apache License\n Version 2.0, January 2004\n "
},
{
"path": "README.md",
"chars": 5846,
"preview": "# Warning - This project is not active\n\nNoms is not being maintained. You shouldn't use it, except maybe for fun or rese"
},
{
"path": "cmd/util/kingpin_command.go",
"chars": 346,
"preview": "// Copyright 2017 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "codecov.yml",
"chars": 1278,
"preview": "codecov:\n branch: master\n bot: \"mikegray\"\n ci:\n - \"jenkins3.noms.io\"\n\ncoverage:\n precision: 2 # how many "
},
{
"path": "doc/cli-tour.md",
"chars": 4608,
"preview": "[Home](../README.md) »\n\n[Technical Overview](intro.md) | [Use Cases](../README.md#use-cases) | *"
},
{
"path": "doc/decent/about.md",
"chars": 4117,
"preview": "[Home](../../README.md) » [Use Cases](../../README.md#use-cases) » **Decentralized** »\n\n**About** | [Quickst"
},
{
"path": "doc/decent/architectures.md",
"chars": 3653,
"preview": "[Home](../../README.md) » [Use Cases](../../README.md#use-cases) » **Decentralized** »\n\n[About](about.md) | "
},
{
"path": "doc/decent/demo-ipfs-chat.md",
"chars": 1751,
"preview": "[Home](../../README.md) » [Use Cases](../../README.md#use-cases) » **Decentralized** »\n\n[About](about.md) | "
},
{
"path": "doc/decent/demo-p2p-chat.md",
"chars": 1719,
"preview": "[Home](../../README.md) » [Use Cases](../../README.md#use-cases) » **Decentralized** »\n\n[About](about.md) | "
},
{
"path": "doc/decent/quickstart.md",
"chars": 6587,
"preview": "[Home](../../README.md) » [Use Cases](../../README.md#use-cases) » **Decentralized** »\n\n[About](about.md) | "
},
{
"path": "doc/faq.md",
"chars": 4680,
"preview": "[Home](../README.md) »\n\n[Technical Overview](intro.md) | [Use Cases](../README.md#use-cases) | ["
},
{
"path": "doc/go-tour.md",
"chars": 7285,
"preview": "[Home](../README.md) »\n\n[Technical Overview](intro.md) | [Use Cases](../README.md#use-cases) | ["
},
{
"path": "doc/intro.md",
"chars": 17515,
"preview": "[Home](../README.md) »\n\n**Technical Overview** | [Use Cases](../README.md#use-cases) | [Command-"
},
{
"path": "doc/spelling.md",
"chars": 6391,
"preview": "[Home](../README.md) »\n\n[Technical Overview](intro.md) | [Use Cases](../README.md#use-cases) | ["
},
{
"path": "go/chunks/chunk.go",
"chars": 2231,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/chunks/chunk_serializer.go",
"chars": 2039,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/chunks/chunk_serializer_test.go",
"chars": 1011,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/chunks/chunk_store.go",
"chars": 3058,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/chunks/chunk_store_common_test.go",
"chars": 2847,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/chunks/chunk_test.go",
"chars": 1259,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/chunks/memory_store.go",
"chars": 5038,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/chunks/memory_store_test.go",
"chars": 550,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/chunks/put_cache.go",
"chars": 1089,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/chunks/remote_requests.go",
"chars": 3182,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/chunks/remote_requests_test.go",
"chars": 3707,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/chunks/test_utils.go",
"chars": 2023,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/config/config.go",
"chars": 3151,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/config/config_test.go",
"chars": 5177,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/config/resolver.go",
"chars": 3807,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/config/resolver_test.go",
"chars": 3772,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/constants/http.go",
"chars": 408,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/constants/version.go",
"chars": 337,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/d/check_error.go",
"chars": 513,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/d/try.go",
"chars": 4303,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/d/try_test.go",
"chars": 3765,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/datas/commit.go",
"chars": 3641,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/datas/commit_options.go",
"chars": 890,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/datas/commit_test.go",
"chars": 6785,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/datas/database.go",
"chars": 5703,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/datas/database_common.go",
"chars": 9247,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/datas/database_server.go",
"chars": 4862,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/datas/database_test.go",
"chars": 17758,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/datas/dataset.go",
"chars": 2989,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/datas/dataset_test.go",
"chars": 4956,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/datas/http_chunk_store.go",
"chars": 14999,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/datas/http_chunk_store_test.go",
"chars": 13284,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/datas/pull.go",
"chars": 3714,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/datas/pull_test.go",
"chars": 9936,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/datas/pulling.md",
"chars": 4284,
"preview": "# Dataset pulling algorithm\nThe approach is to explore the chunk graph of both sink and source in order of decreasing re"
},
{
"path": "go/datas/remote_database_handlers.go",
"chars": 19760,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/datas/remote_database_handlers_test.go",
"chars": 12582,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/datas/serialize_hashes.go",
"chars": 1260,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/datas/serialize_hashes_test.go",
"chars": 1038,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/diff/apply_patch.go",
"chars": 9909,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/diff/apply_patch_test.go",
"chars": 12385,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/diff/diff.go",
"chars": 8825,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/diff/diff_test.go",
"chars": 10984,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/diff/patch.go",
"chars": 2685,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/diff/patch_test.go",
"chars": 3483,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/diff/print_diff.go",
"chars": 4148,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/diff/summary.go",
"chars": 5293,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/hash/base32.go",
"chars": 425,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/hash/base32_test.go",
"chars": 1798,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/hash/hash.go",
"chars": 4381,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/hash/hash_slice.go",
"chars": 707,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/hash/hash_slice_test.go",
"chars": 641,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/hash/hash_test.go",
"chars": 2980,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/marshal/decode.go",
"chars": 15777,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/marshal/decode_test.go",
"chars": 32256,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/marshal/encode.go",
"chars": 18517,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/marshal/encode_test.go",
"chars": 27262,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/marshal/encode_type.go",
"chars": 5944,
"preview": "// Copyright 2017 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/marshal/encode_type_test.go",
"chars": 14481,
"preview": "// Copyright 2017 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/merge/candidate.go",
"chars": 3006,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/merge/three_way.go",
"chars": 13065,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/merge/three_way_keyval_test.go",
"chars": 8008,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/merge/three_way_list.go",
"chars": 4992,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/merge/three_way_list_test.go",
"chars": 4727,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/merge/three_way_ordered_sequence.go",
"chars": 5480,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/merge/three_way_set_test.go",
"chars": 2458,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/merge/three_way_test.go",
"chars": 2343,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/metrics/histogram.go",
"chars": 4840,
"preview": "// Copyright 2017 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/metrics/histogram_test.go",
"chars": 3948,
"preview": "// Copyright 2017 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/NBS-on-AWS.md",
"chars": 3806,
"preview": "# Backing a Noms Block Store with AWS\n\nHow to use S3 and DynamoDB as the persistent storage layer for a Noms Block Store"
},
{
"path": "go/nbs/README.md",
"chars": 2607,
"preview": "# Noms Block Store\n\nA horizontally-scalable storage backend for Noms.\n\n## Overview\n\nNBS is a storage layer optimized for"
},
{
"path": "go/nbs/aws_chunk_source.go",
"chars": 2567,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/aws_chunk_source_test.go",
"chars": 2010,
"preview": "// Copyright 2017 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/aws_table_persister.go",
"chars": 15697,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/aws_table_persister_test.go",
"chars": 15163,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/block_store_test.go",
"chars": 12892,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/cache.go",
"chars": 2339,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/conjoiner.go",
"chars": 4833,
"preview": "// Copyright 2017 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/conjoiner_test.go",
"chars": 5669,
"preview": "// Copyright 2017 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/dynamo_fake_test.go",
"chars": 4361,
"preview": "// Copyright 2017 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/dynamo_manifest.go",
"chars": 4552,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/dynamo_manifest_test.go",
"chars": 5373,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/dynamo_table_reader.go",
"chars": 3650,
"preview": "// Copyright 2017 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/dynamo_table_reader_test.go",
"chars": 3366,
"preview": "// Copyright 2017 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/factory.go",
"chars": 4511,
"preview": "// Copyright 2017 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/factory_test.go",
"chars": 1364,
"preview": "// Copyright 2017 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/fd_cache.go",
"chars": 3774,
"preview": "// Copyright 2017 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/fd_cache_test.go",
"chars": 2535,
"preview": "// Copyright 2017 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/file_manifest.go",
"chars": 4861,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/file_manifest_test.go",
"chars": 7525,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/file_table_persister.go",
"chars": 2515,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/file_table_persister_test.go",
"chars": 6256,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/fs_table_cache.go",
"chars": 3046,
"preview": "// Copyright 2017 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/fs_table_cache_test.go",
"chars": 2902,
"preview": "// Copyright 2017 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/manifest.go",
"chars": 6989,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/manifest_cache.go",
"chars": 2797,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/manifest_cache_test.go",
"chars": 2631,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/mem_table.go",
"chars": 2957,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/mem_table_test.go",
"chars": 4794,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/mmap_table_reader.go",
"chars": 2296,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/mmap_table_reader_test.go",
"chars": 782,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/persisting_chunk_source.go",
"chars": 3534,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/persisting_chunk_source_test.go",
"chars": 1556,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/root_tracker_test.go",
"chars": 12515,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/s3_fake_test.go",
"chars": 7223,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/s3_table_reader.go",
"chars": 4593,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/s3_table_reader_test.go",
"chars": 2405,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/stats.go",
"chars": 7460,
"preview": "// Copyright 2017 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/stats_test.go",
"chars": 3204,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/store.go",
"chars": 11917,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/table.go",
"chars": 8321,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/table_persister.go",
"chars": 6713,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/table_persister_test.go",
"chars": 1380,
"preview": "// Copyright 2017 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/table_reader.go",
"chars": 14438,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/table_set.go",
"chars": 6518,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/table_set_test.go",
"chars": 4149,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/table_test.go",
"chars": 8988,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/table_writer.go",
"chars": 5650,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nbs/test/manifest_clobber.go",
"chars": 820,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/ngql/README.md",
"chars": 3205,
"preview": "# Noms GraphQL\n\nAn experimental bridge between Noms and [GraphQL](http://graphql.org/)\n\nThis is to be used with https://"
},
{
"path": "go/ngql/query.go",
"chars": 2955,
"preview": "// Copyright 2017 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/ngql/query_test.go",
"chars": 62878,
"preview": "// Copyright 2017 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/ngql/types.go",
"chars": 30762,
"preview": "// Copyright 2017 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nomdl/lexer.go",
"chars": 2017,
"preview": "// Copyright 2017 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nomdl/parser.go",
"chars": 9540,
"preview": "// Copyright 2017 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/nomdl/parser_test.go",
"chars": 19540,
"preview": "// Copyright 2017 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/perf/hash-perf-rig/README.md",
"chars": 847,
"preview": "This is a performance test rig for the two main types of hashing we do in NOMS - buzhash and sha1. There's also support "
},
{
"path": "go/perf/hash-perf-rig/main.go",
"chars": 1441,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/perf/suite/suite.go",
"chars": 15817,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/perf/suite/suite_test.go",
"chars": 8773,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/sloppy/sloppy.go",
"chars": 7505,
"preview": "// Copyright 2017 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/sloppy/sloppy_test.go",
"chars": 63807,
"preview": "// Copyright 2017 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/spec/absolute_path.go",
"chars": 3652,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/spec/absolute_path_test.go",
"chars": 3521,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/spec/commit_meta.go",
"chars": 3206,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/spec/commit_meta_test.go",
"chars": 5581,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/spec/spec.go",
"chars": 9387,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/spec/spec_test.go",
"chars": 16553,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/spec/util.go",
"chars": 724,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/types/blob.go",
"chars": 6705,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/types/blob_editor.go",
"chars": 6624,
"preview": "// Copyright 2017 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/types/blob_editor_test.go",
"chars": 1823,
"preview": "// Copyright 2017 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/types/blob_leaf_sequence.go",
"chars": 1652,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/types/blob_test.go",
"chars": 7572,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/types/bool.go",
"chars": 1068,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/types/codec.go",
"chars": 5420,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/types/codec_test.go",
"chars": 1851,
"preview": "// Copyright 2017 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/types/collection.go",
"chars": 257,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/types/collection_test.go",
"chars": 1927,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/types/common_supertype.go",
"chars": 4192,
"preview": "package types\n\nimport \"github.com/attic-labs/noms/go/d\"\n\n// ContainCommonSupertype returns true if it's possible to synt"
},
{
"path": "go/types/common_supertype_test.go",
"chars": 8453,
"preview": "package types\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestContainCommonSupertype(t *testing."
},
{
"path": "go/types/compare_test.go",
"chars": 3584,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/types/edit_distance.go",
"chars": 6143,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/types/edit_distance_test.go",
"chars": 4182,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/types/encode_human_readable.go",
"chars": 9957,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/types/encode_human_readable_test.go",
"chars": 10627,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/types/encoding_test.go",
"chars": 14830,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/types/equals_test.go",
"chars": 2410,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/types/get_hash.go",
"chars": 565,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/types/graph_builder.go",
"chars": 9717,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/types/graph_builder_test.go",
"chars": 8757,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/types/incremental_test.go",
"chars": 3186,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/types/indexed_sequence_diff.go",
"chars": 3419,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/types/indexed_sequences.go",
"chars": 3826,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/types/leaf_sequence.go",
"chars": 3876,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/types/less.go",
"chars": 453,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/types/list.go",
"chars": 7828,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/types/list_editor.go",
"chars": 6010,
"preview": "// Copyright 2017 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/types/list_editor_test.go",
"chars": 6354,
"preview": "// Copyright 2017 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/types/list_iterator.go",
"chars": 708,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/types/list_iterator_test.go",
"chars": 718,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/types/list_leaf_sequence.go",
"chars": 526,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/types/list_test.go",
"chars": 32041,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/types/make_type.go",
"chars": 3391,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/types/map.go",
"chars": 7690,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/types/map_editor.go",
"chars": 4894,
"preview": "// Copyright 2017 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/types/map_iterator.go",
"chars": 1333,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/types/map_iterator_test.go",
"chars": 2362,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/types/map_leaf_sequence.go",
"chars": 3981,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
},
{
"path": "go/types/map_test.go",
"chars": 41778,
"preview": "// Copyright 2016 Attic Labs, Inc. All rights reserved.\n// Licensed under the Apache License, version 2.0:\n// http://www"
}
]
// ... and 139 more files (download for full content)
About this extraction
This page contains the full source code of the attic-labs/noms GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 339 files (2.4 MB), approximately 656.1k tokens, and a symbol index with 3933 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.