Repository: ipld/go-ipld-prime Branch: master Commit: b6ff5a2d5c06 Files: 377 Total size: 2.0 MB Directory structure: gitextract_6epy99x4/ ├── .gitattributes ├── .github/ │ ├── actions/ │ │ └── go-test-setup/ │ │ └── action.yml │ ├── dependabot.yml │ └── workflows/ │ ├── generated-pr.yml │ ├── go-check.yml │ ├── go-test-prime.yml │ ├── go-test.yml │ ├── release-check.yml │ ├── releaser.yml │ ├── stale.yml │ └── tagpush.yml ├── .gitmodules ├── CHANGELOG.md ├── HACKME.md ├── HACKME_builderBehaviors.md ├── HACKME_mergeStrategy.md ├── HACKME_releases.md ├── LICENSE ├── README.md ├── README_migrationGuide.md ├── adl/ │ ├── interface.go │ └── rot13adl/ │ ├── example_test.go │ ├── rot13logic.go │ ├── rot13node.go │ ├── rot13node_test.go │ ├── rot13prototypes.go │ ├── rot13reification.go │ └── rot13substrate.go ├── adl.go ├── codec/ │ ├── README.md │ ├── api.go │ ├── cbor/ │ │ ├── multicodec.go │ │ └── roundtrip_test.go │ ├── dagcbor/ │ │ ├── common.go │ │ ├── doc.go │ │ ├── marshal.go │ │ ├── marshal_test.go │ │ ├── multicodec.go │ │ ├── nongreedy_test.go │ │ ├── roundtripCidlink_test.go │ │ ├── roundtrip_test.go │ │ ├── unmarshal.go │ │ └── unmarshal_test.go │ ├── dagjson/ │ │ ├── marshal.go │ │ ├── marshal_test.go │ │ ├── multicodec.go │ │ ├── nongreedy_test.go │ │ ├── options_test.go │ │ ├── roundtripBytes_test.go │ │ ├── roundtripCidlink_test.go │ │ ├── roundtrip_test.go │ │ └── unmarshal.go │ ├── decode_test.go │ ├── json/ │ │ ├── marshal_test.go │ │ └── multicodec.go │ └── raw/ │ ├── codec.go │ └── codec_test.go ├── codec.go ├── codecHelpers.go ├── codecHelpers_test.go ├── datamodel/ │ ├── copy.go │ ├── copy_test.go │ ├── doc.go │ ├── equal.go │ ├── equal_test.go │ ├── errors.go │ ├── kind.go │ ├── kind_test.go │ ├── link.go │ ├── node.go │ ├── nodeBuilder.go │ ├── path.go │ ├── pathSegment.go │ ├── path_test.go │ └── unit.go ├── datamodel.go ├── doc.go ├── examples_test.go ├── fluent/ │ ├── bench_test.go │ ├── doc.go │ ├── fluentBuilder.go │ ├── fluentBuilder_test.go │ ├── fluentRecover.go │ ├── fluentRecover_test.go │ ├── qp/ │ │ ├── example_test.go │ │ └── qp.go │ ├── reflect.go │ ├── reflect_test.go │ ├── toInterfaceValue.go │ └── toInterfaceValue_test.go ├── go.mod ├── go.sum ├── linking/ │ ├── cid/ │ │ ├── HACKME.md │ │ ├── cidLink.go │ │ ├── linksystem.go │ │ └── memorystorage.go │ ├── errors.go │ ├── functions.go │ ├── functions_test.go │ ├── linkingExamples_test.go │ ├── preload/ │ │ └── preload.go │ ├── setup.go │ └── types.go ├── linking.go ├── multicodec/ │ ├── defaultRegistry.go │ └── registry.go ├── must/ │ └── must.go ├── node/ │ ├── basic/ │ │ └── deprecated.go │ ├── basicnode/ │ │ ├── HACKME.md │ │ ├── any.go │ │ ├── any_test.go │ │ ├── bench_test.go │ │ ├── bool.go │ │ ├── bytes.go │ │ ├── bytes_stream.go │ │ ├── bytes_test.go │ │ ├── float.go │ │ ├── int.go │ │ ├── int_test.go │ │ ├── link.go │ │ ├── list.go │ │ ├── list_test.go │ │ ├── map.go │ │ ├── map_test.go │ │ ├── prototypes.go │ │ ├── string.go │ │ └── string_test.go │ ├── bindnode/ │ │ ├── api.go │ │ ├── api_test.go │ │ ├── custom_test.go │ │ ├── example_test.go │ │ ├── fuzz_test.go │ │ ├── generate.go │ │ ├── infer.go │ │ ├── infer_test.go │ │ ├── node.go │ │ ├── registry/ │ │ │ ├── registry.go │ │ │ └── registry_test.go │ │ ├── repr.go │ │ ├── schema_test.go │ │ └── testdata/ │ │ └── fuzz/ │ │ └── FuzzBindnodeViaDagCBOR/ │ │ ├── 164dd28629e6a5637f02c6eaad32eee5bf8ea41ce6d1dae95334a7db7dd6188f │ │ ├── 199afa754020c4587bc87633033b4e56ecdb5ecc2bb0dbac46b799d6c18c5201 │ │ ├── 2c17d42168478a0837ebba66f6aa98bdf4bb81b5196062d0e6b23c8b0325be6a │ │ ├── 425499b0c3693d87a0b11db10d21c540283dfb88610d114eb5b23485d5c2f342 │ │ ├── 636f8e5cdaf52572b826e615a9bdf90f15f8acf2880b467b3949f5356c2ddded │ │ ├── 669d57fbbe55b8ceb7e78e3ccd3b90e76c4b740d25284a66eac8aac0dbb2475d │ │ ├── 7f9a6898dead41ba2f5fd4f07f2ddb44d54e7648d66e54982d09996bc720cf56 │ │ ├── 821c248529299bb6b68e443e4b00cc11c6605f766f25e619d85e6d6b40a33dac │ │ ├── bf7c410983f3e696a03e743df1bc6f606137871f7fb557af74836b7aa04e56ad │ │ ├── dc95e9aad454ed9109e93b824f92d4bb00c9c778af29e6fcb38a6083c78d7dbb │ │ ├── f63d4652cdac3208fc0a0d0b755615320443b60ef80c0ecdef99c9d895ae2124 │ │ └── fe73a19655fff8b93193c3e90f9cd7b24b10ae0467175cf2264ff3ec135215a6 │ ├── doc.go │ ├── gendemo/ │ │ ├── doc.go │ │ ├── gen.go │ │ ├── gendemo_test.go │ │ ├── ipldsch_minima.go │ │ ├── ipldsch_satisfaction.go │ │ └── ipldsch_types.go │ ├── mixins/ │ │ ├── HACKME.md │ │ ├── bool.go │ │ ├── bytes.go │ │ ├── delim.go │ │ ├── delim_test.go │ │ ├── float.go │ │ ├── int.go │ │ ├── link.go │ │ ├── list.go │ │ ├── map.go │ │ ├── string.go │ │ └── tmplMixin.txt │ └── tests/ │ ├── HACKME.md │ ├── byteSpecs.go │ ├── checkers.go │ ├── checkers_test.go │ ├── corpus/ │ │ ├── corpus.go │ │ ├── corpus_test.go │ │ └── util.go │ ├── listSpecs.go │ ├── mapBenchmarks.go │ ├── mapBenchmarks_test.go │ ├── mapFixtures.go │ ├── mapSpecs.go │ ├── marshalBenchmarks.go │ ├── schema.go │ ├── schemaLinks.go │ ├── schemaLists.go │ ├── schemaMaps.go │ ├── schemaScalars.go │ ├── schemaStruct.go │ ├── schemaStructReprListpairs.go │ ├── schemaStructReprStringjoin.go │ ├── schemaStructReprTuple.go │ ├── schemaStructsContainingMaybe.go │ ├── schemaUnions.go │ ├── schemaUnionsKinded.go │ ├── schemaUnionsStringprefix.go │ ├── stringSpecs.go │ ├── testEngine.go │ ├── testcase.go │ ├── testutil.go │ ├── traversalBenchmarks.go │ ├── unmarshalBenchmarks.go │ └── util.go ├── operations.go ├── printer/ │ ├── doc.go │ ├── printer.go │ └── printer_test.go ├── schema/ │ ├── dmt/ │ │ ├── compile.go │ │ ├── doc.go │ │ ├── gen.go │ │ ├── operations.go │ │ ├── roundtrip_test.go │ │ ├── schema.go │ │ └── types.go │ ├── dsl/ │ │ ├── parse.go │ │ └── parse_test.go │ ├── errors.go │ ├── gen/ │ │ └── go/ │ │ ├── HACKME.md │ │ ├── HACKME_abbrevs.md │ │ ├── HACKME_dry.md │ │ ├── HACKME_maybe.md │ │ ├── HACKME_memorylayout.md │ │ ├── HACKME_scalars.md │ │ ├── HACKME_templates.md │ │ ├── HACKME_testing.md │ │ ├── HACKME_tradeoffs.md │ │ ├── HACKME_wip.md │ │ ├── README.md │ │ ├── README_behaviors.md │ │ ├── README_wishes.md │ │ ├── _test/ │ │ │ └── .gitignore │ │ ├── adjunctCfg.go │ │ ├── externUtil.go │ │ ├── genBool.go │ │ ├── genBoolReprBool.go │ │ ├── genBytes.go │ │ ├── genBytesReprBytes.go │ │ ├── genFloat.go │ │ ├── genFloatReprFloat.go │ │ ├── genInt.go │ │ ├── genIntReprInt.go │ │ ├── genLink.go │ │ ├── genLinkReprLink.go │ │ ├── genList.go │ │ ├── genListReprList.go │ │ ├── genMap.go │ │ ├── genMapReprMap.go │ │ ├── genString.go │ │ ├── genStringReprString.go │ │ ├── genStruct.go │ │ ├── genStructReprMap.go │ │ ├── genStructReprStringjoin.go │ │ ├── genStructReprTuple.go │ │ ├── genUnion.go │ │ ├── genUnionReprKeyed.go │ │ ├── genUnionReprKinded.go │ │ ├── genUnionReprStringprefix.go │ │ ├── generate.go │ │ ├── generators.go │ │ ├── genpartsCommon.go │ │ ├── genpartsList.go │ │ ├── genpartsMap.go │ │ ├── genpartsMinima.go │ │ ├── genpartsStrictoid.go │ │ ├── mixins/ │ │ │ ├── boolGenMixin.go │ │ │ ├── bytesGenMixin.go │ │ │ ├── floatGenMixin.go │ │ │ ├── intGenMixin.go │ │ │ ├── kindTraits.go │ │ │ ├── linkGenMixin.go │ │ │ ├── listGenMixin.go │ │ │ ├── mapGenMixin.go │ │ │ ├── stringGenMixin.go │ │ │ └── templateUtil.go │ │ ├── templateUtil.go │ │ ├── testEngine_disabled_test.go │ │ ├── testEngine_nocgo_test.go │ │ ├── testEngine_plugin_test.go │ │ ├── testEngine_test.go │ │ ├── testLinks_test.go │ │ ├── testLists_test.go │ │ ├── testMaps_test.go │ │ ├── testScalars_test.go │ │ ├── testStructReprStringjoin_test.go │ │ ├── testStructReprTuple_test.go │ │ ├── testStruct_test.go │ │ ├── testStructsContainingMaybe_test.go │ │ ├── testUnionsKinded_test.go │ │ ├── testUnionsStringprefix_test.go │ │ └── testUnions_test.go │ ├── kind.go │ ├── maybe.go │ ├── tmpBuilders.go │ ├── type.go │ ├── typeMethods.go │ ├── typedNode.go │ ├── typesystem.go │ └── validate.go ├── schema.go ├── storage/ │ ├── README_adapters.md │ ├── api.go │ ├── benchmarks/ │ │ ├── README.md │ │ ├── go.mod │ │ ├── go.sum │ │ └── storageBenchmarks_test.go │ ├── bsadapter/ │ │ ├── README.md │ │ ├── bsadapter.go │ │ ├── go.mod │ │ └── go.sum │ ├── bsrvadapter/ │ │ ├── README.md │ │ ├── bsrvadapter.go │ │ ├── go.mod │ │ └── go.sum │ ├── doc.go │ ├── dsadapter/ │ │ ├── README.md │ │ ├── dsadapter.go │ │ ├── go.mod │ │ └── go.sum │ ├── fsstore/ │ │ └── fsstore.go │ ├── funcs.go │ ├── memstore/ │ │ └── memstore.go │ ├── sharding/ │ │ ├── sharding.go │ │ ├── sharding.test │ │ ├── sharding_bench_test.go │ │ └── sharding_test.go │ └── tests/ │ ├── benchmarks.go │ └── generators.go ├── testutil/ │ ├── garbage/ │ │ ├── garbage.go │ │ └── garbage_test.go │ ├── indent.go │ ├── indent_test.go │ ├── multibytenode.go │ ├── multibytenode_test.go │ └── simplebytes.go ├── traversal/ │ ├── common.go │ ├── doc.go │ ├── example_select_links_test.go │ ├── fns.go │ ├── focus.go │ ├── focus_test.go │ ├── patch/ │ │ ├── eval.go │ │ ├── parse.go │ │ ├── patch.ipldsch │ │ └── patch_test.go │ ├── select_links.go │ ├── select_links_test.go │ ├── selector/ │ │ ├── builder/ │ │ │ ├── builder.go │ │ │ └── builder_test.go │ │ ├── condition.go │ │ ├── condition_test.go │ │ ├── exploreAll.go │ │ ├── exploreAll_test.go │ │ ├── exploreFields.go │ │ ├── exploreFields_test.go │ │ ├── exploreIndex.go │ │ ├── exploreIndex_test.go │ │ ├── exploreInterpretAs.go │ │ ├── exploreRange.go │ │ ├── exploreRange_test.go │ │ ├── exploreRecursive.go │ │ ├── exploreRecursiveEdge.go │ │ ├── exploreRecursive_test.go │ │ ├── exploreUnion.go │ │ ├── exploreUnion_test.go │ │ ├── fieldKeys.go │ │ ├── matcher.go │ │ ├── matcher_test.go │ │ ├── matcher_util.go │ │ ├── parse/ │ │ │ ├── selector_parse.go │ │ │ └── selector_parse_test.go │ │ ├── selector.go │ │ └── spec_test.go │ ├── walk.go │ ├── walk_test.go │ └── walk_with_stop_test.go └── version.json ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitattributes ================================================ **/testdata/fuzz/** binary ================================================ FILE: .github/actions/go-test-setup/action.yml ================================================ name: go-ipld-prime custom action setup description: Adds additional options to `go test` to skip behavior tests in the main test executions runs: using: "composite" steps: - name: Disable codegen behavior tests shell: bash run: | echo "GOFLAGS=$GOFLAGS -tags=skipgenbehavtests" >> $GITHUB_ENV ================================================ FILE: .github/dependabot.yml ================================================ version: 2 updates: - package-ecosystem: "gomod" directory: "/" schedule: interval: "weekly" - package-ecosystem: "gomod" directory: "/storage/dsadapter/" schedule: interval: "weekly" - package-ecosystem: "gomod" directory: "/storage/benchmarks/" schedule: interval: "weekly" - package-ecosystem: "gomod" directory: "/storage/bsadapter/" schedule: interval: "weekly" - package-ecosystem: "gomod" directory: "/storage/bsrvadapter/" schedule: interval: "weekly" ================================================ FILE: .github/workflows/generated-pr.yml ================================================ name: Close Generated PRs on: schedule: - cron: '0 0 * * *' workflow_dispatch: permissions: issues: write pull-requests: write jobs: stale: uses: ipdxco/unified-github-workflows/.github/workflows/reusable-generated-pr.yml@v1 ================================================ FILE: .github/workflows/go-check.yml ================================================ name: Go Checks on: pull_request: push: branches: ["master"] workflow_dispatch: permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} cancel-in-progress: true jobs: go-check: uses: ipdxco/unified-github-workflows/.github/workflows/go-check.yml@v1.0 ================================================ FILE: .github/workflows/go-test-prime.yml ================================================ on: [push, pull_request] name: Go Test (go-ipld-prime custom) # Similar to go-test.yml but runs only on Linux and doesn't run tests with # coverpkg so we can properly execute the codegen behavior tests which are # skipped in go-test.yml execution. jobs: unit: strategy: fail-fast: false matrix: os: [ "ubuntu" ] go: [ "1.24.x", "1.25.x" ] runs-on: ${{ format('{0}-latest', matrix.os) }} name: ${{ matrix.os }} (go ${{ matrix.go }}) steps: - uses: actions/checkout@v2 with: submodules: recursive - uses: actions/setup-go@v2 with: go-version: ${{ matrix.go }} - name: Go information run: | go version go env - name: Run tests uses: protocol/multiple-go-modules@v1.2 with: run: | go test -v ./... ================================================ FILE: .github/workflows/go-test.yml ================================================ name: Go Test on: pull_request: push: branches: ["master"] workflow_dispatch: permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} cancel-in-progress: true jobs: go-test: uses: ipdxco/unified-github-workflows/.github/workflows/go-test.yml@v1.0 secrets: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} ================================================ FILE: .github/workflows/release-check.yml ================================================ name: Release Checker on: pull_request_target: paths: [ 'version.json' ] types: [ opened, synchronize, reopened, labeled, unlabeled ] workflow_dispatch: permissions: contents: write pull-requests: write concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: release-check: uses: ipdxco/unified-github-workflows/.github/workflows/release-check.yml@v1.0 ================================================ FILE: .github/workflows/releaser.yml ================================================ name: Releaser on: push: paths: [ 'version.json' ] workflow_dispatch: permissions: contents: write concurrency: group: ${{ github.workflow }}-${{ github.sha }} cancel-in-progress: true jobs: releaser: uses: ipdxco/unified-github-workflows/.github/workflows/releaser.yml@v1.0 ================================================ FILE: .github/workflows/stale.yml ================================================ name: Close Stale Issues on: schedule: - cron: '0 0 * * *' workflow_dispatch: permissions: issues: write pull-requests: write jobs: stale: uses: ipdxco/unified-github-workflows/.github/workflows/reusable-stale-issue.yml@v1 ================================================ FILE: .github/workflows/tagpush.yml ================================================ name: Tag Push Checker on: push: tags: - v* permissions: contents: read issues: write concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: releaser: uses: ipdxco/unified-github-workflows/.github/workflows/tagpush.yml@v1.0 ================================================ FILE: .gitmodules ================================================ [submodule ".ipld"] path = .ipld url = https://github.com/ipld/ipld/ ================================================ FILE: CHANGELOG.md ================================================ CHANGELOG ========= Here is collected some brief notes on major changes over time, sorted by tag in which they are first available. Of course for the "detailed changelog", you can always check the commit log! But hopefully this summary _helps_. Note about version numbering: All release tags are in the "v0.${x}" range. _We do not expect to make a v1 release._ Nonetheless, this should not be taken as a statement that the library isn't _usable_ already. Much of this code is used in other libraries and products, and we do take some care about making changes. (If you're ever wondering about stability of a feature, ask -- or contribute more tests ;)) - [Planned/Upcoming Changes](#planned-upcoming-changes) - [Released Changes Log](#released-changes) Planned/Upcoming Changes ------------------------ Here are some outlines of changes we intend to make that affect the public API: - **IPLD Amend**: is likely to land soon; it implements a more efficient underlying architecture to support IPLD Patch and related features. IPLD Amend adds an interface to allow incremental changes to `Node`s in an efficient way. Whereas IPLD Patch is a protocol for expressing changes. We're still working on figuring out exactly where it fits in the stack and making sure it won't be disruptive but early benchmarks are very promising for both Patch and traversal-based transforms. See https://github.com/ipld/go-ipld-prime/pull/445 for more. - **Layered `Node` implementation optimizations**: When layering different implementations of `Node` builders or consumers, having to defer through basicnode types can lead to large inefficiencies of memory and speed. We are looking at ways to improve this situation, including ways to *assemble* layered assemblers. See https://github.com/ipld/go-ipld-prime/issues/443 for discussion and some initial plans. - **Selectors**: There have been some recurring wishes to do something about the Selector package layout. There's no intended or prioritized date for this. See https://github.com/ipld/go-ipld-prime/issues/236 for more. - **Absent / "Not found" values**: There may be some upcoming changes to exactly how "not found" values are handled in order to clarify and standardize the subject. There's no finalized date for this. See https://github.com/ipld/go-ipld-prime/issues/360 for more. Released Changes ---------------- ### v0.21.0 _2023 August 10_ go-ipld-prime's release policy says that: > even numbers should be easy upgrades; odd numbers may change things This release is an odd number, and it does change some minor things. #### 🛠 Breaking * **Build**: The minimum version of Go has been bumped from 1.18 to **1.19**. * **Dependencies**: The go-cid dependency was upgraded from v0.3.2 to **v0.4.1**. This is a relatively minor change but the introduction of `ErrInvalidCid` wrapping may be breaking for some users. * **Selectors**: Remove hard error when a traversal encounters a slice matcher with a node that is not a string or bytes, by @rvagg [#529](https://github.com/ipld/go-ipld-prime/pull/529) #### 🔦 Features * **Traversal**: `Preloader` functionality, by @hannahhoward and @rvagg [#452](https://github.com/ipld/go-ipld-prime/pull/452) * See the [traversal](https://pkg.go.dev/github.com/ipld/go-ipld-prime/traversal) package documentation for more information on how a `Preloader` can be used to introduce parallelism into a traversal. The [Lassie](https://github.com/filecoin-project/lassie) project is currently using this functionality to speed up Bitswap block fetching; future releases of go-ipld-prime may include additional functionality being prototyped in Lassie to manage parallelism and caching. * **Schemas**: Support `listpairs` struct representation in DSL parsing, by @rvagg [#514](https://github.com/ipld/go-ipld-prime/pull/514) * **Schemas**: Support `inline` union representation in DSL parsing, by @rvagg [#527](https://github.com/ipld/go-ipld-prime/pull/527) * **Bindnode**: Support `listpairs` struct representation, by @rvagg [#514](https://github.com/ipld/go-ipld-prime/pull/514) * **Selectors**: Support negative values for slice matcher's `From` and `To`, by @rvagg [#530](https://github.com/ipld/go-ipld-prime/pull/530) * The slice matcher is currently being used to support byte-range fetching for the [IPFS Trustless Gateway](https://specs.ipfs.tech/http-gateways/trustless-gateway/) specification. Work is ongoing and can be seen in both the [Lassie](https://github.com/filecoin-project/lassie) and [Frisbii](https://github.com/filecoin-project/lassie) projects. * _Please note that this feature is currently considered **experimental** and shoule be used with care and with the expectation that it may change in the near future. Expect a release or two before this feature is considered stable._ #### 🩹 Fixes * **Traversal**: `StartAtPath` work properly for matching walks, by @rvagg [#500](https://github.com/ipld/go-ipld-prime/pull/500) * **Traversal**: `WalkTransforming()` work properly, by @EdSchouten [#516](https://github.com/ipld/go-ipld-prime/pull/516) * **Basicnode**: `basic.NewInt()` returns a pointer like other constructors, by @hacdias [#525](https://github.com/ipld/go-ipld-prime/pull/525) * **Selectors**: Cache offsets for sequential reads in slice matcher, by @rvagg [#529](https://github.com/ipld/go-ipld-prime/pull/529) #### 🌟 Thanks! Thanks to @hacdias and @EdSchouten for their first contributions to go-ipld-prime! ### v0.20.0 go-ipld-prime's release policy says that: > even numbers should be easy upgrades; odd numbers may change things As such, v0.20.0 is a relatively minor release with a grab-bag of small improvements and fixes. _2023 February 11_ Schema errors can now [`errors.Is`](https://pkg.go.dev/errors#Is): * \[[`61c9ab10d4`](https://github.com/ipld/go-ipld-prime/commit/61c9ab10d4)] - **feat**: support errors.Is for schema errors (Ian Davis) [#476](https://github.com/ipld/go-ipld-prime/pull/476) Schema DMT (schema/dmt) is now more usable from the outside and has a new `ConcatenateSchemas` function that can be used to combine two schemas into one: * \[[`db9d8a7512`](https://github.com/ipld/go-ipld-prime/commit/db9d8a7512)] - Export schema/dmt.TypeSystem. (Eric Myhre) [#483](https://github.com/ipld/go-ipld-prime/pull/483) * \[[`39818c169a`](https://github.com/ipld/go-ipld-prime/commit/39818c169a)] - Add a SchemaConcatenate operation. (Eric Myhre) [#483](https://github.com/ipld/go-ipld-prime/pull/483) * \[[`c68ba53c67`](https://github.com/ipld/go-ipld-prime/commit/c68ba53c67)] - More accurate name for structure that contains easy access to prototypes. (Eric Myhre) [#483](https://github.com/ipld/go-ipld-prime/pull/483) * \[[`2ecabf1217`](https://github.com/ipld/go-ipld-prime/commit/2ecabf1217)] - Add several pieces of docs to schema/dmt. (Eric Myhre) * \[[`33475f0448`](https://github.com/ipld/go-ipld-prime/commit/33475f0448)] - Fix mispatched package declaration. (Eric Myhre) The DAG-CBOR codec now has an `DontParseBeyondEnd` option (default `false`) that allows it to parse undelimited streamed objects. This matches the same functionality already in DAG-JSON and should only be used for specialised cases: * \[[`7b00b1490f`](https://github.com/ipld/go-ipld-prime/commit/7b00b1490f)] - feat(dagcbor): mode to allow parsing undelimited streamed objects (Rod Vagg) [#490](https://github.com/ipld/go-ipld-prime/pull/490) `datamodel.Copy` got some direct test coverage and will now complain if you try to copy a `nil` node: * \[[`f4bb2daa27`](https://github.com/ipld/go-ipld-prime/commit/f4bb2daa27)] - fix(datamodel): add tests to Copy, make it complain on nil (Rod Vagg) [#491](https://github.com/ipld/go-ipld-prime/pull/491) The LinkSystem data loading check will compare links (CIDs) to ensure it loaded what you wanted; this now properly supports the case where your link is a pointer: * \[[`1fc56b8e7a`](https://github.com/ipld/go-ipld-prime/commit/1fc56b8e7a)] - Fix hash mismatch error on matching link pointer (Masih H. Derkani) [#480](https://github.com/ipld/go-ipld-prime/pull/480) ### v0.19.0 _2022 October 13_ go-ipld-prime's release policy says that: > even numbers should be easy upgrades; odd numbers may change things The major change in this release is a bump to Go 1.18. #### 🛠 Breaking Changes Update go.mod to Go 1.18. #### 🔦 Highlights * **Codecs**: [Correct JSON codec Bytes handling](https://github.com/ipld/go-ipld-prime/pull/472). This change does not impact DAG-JSON, which is the generally recommended codec for JSON output as the JSON codec cannot properly handle Bytes or Links. * **Dependencies**: * Update to go-multihash@v0.2.1: https://github.com/multiformats/go-multihash/releases/tag/v0.2.1 * Update to go-multicodec@v0.6.0: https://github.com/multiformats/go-multicodec/releases/tag/v0.6.0 * Update to go-cid@v0.3.2: https://github.com/ipfs/go-cid/compare/v0.2.0...v0.3.2 ### v0.18.0 _2022 August 01_ go-ipld-prime's release policy says that: > even numbers should be easy upgrades; odd numbers may change things So, as an even number, this v0.18.0 release should be a smooth ride for upgraders from v0.17.0. We have 3 major feature additions, all focused on [Bindnode](https://pkg.go.dev/github.com/ipld/go-ipld-prime/node/bindnode). #### 🔦 Highlights * **Bindnode**: [Custom Go type converters](https://github.com/ipld/go-ipld-prime/pull/414) - Bindnode performs bidirectional mapping of Go types to the IPLD Data Model, and in doing so, it assumes a straightforward mapping of values to their encoded forms. But there are common cases where a Go type doesn't have a straightforward path to serialization, either because the encoded form needs a custom layout, or because bindnode doesn't have enough information to infer a serialization pattern. Custom Go type converters for bindnode allow a user to supply a pair of converter functions for a Go type that dictate how to map that type to an IPLD Data Model kind. See the **[bindnode documentation](https://pkg.go.dev/github.com/ipld/go-ipld-prime/node/bindnode)** for more information. * **Bindnode**: [Type registry](https://github.com/ipld/go-ipld-prime/pull/437) - Setting up Go type mappings with Bindnode involves some boilerplate. A basic type registry is now available that takes some of this boilerplate away; giving you a single place to register, and perform conversions to and from Go types, Data Model (`Node`) forms or directly through serialization. See the **[bindnode/registry documentation](https://pkg.go.dev/github.com/ipld/go-ipld-prime/node/bindnode/registry)** for more information. * **Bindnode** [Full `uint64` support](https://github.com/ipld/go-ipld-prime/pull/414/commits/87211682cb963ef1c98fa63909f67a8b02d1108c) - the `uint64` support introduced in go-ipld-prime@v0.17.0 has been wired into Bindnode. The Data Model (`Node`) forms expose integers as `int64` values, which is lossy for unsigned 64-bit integers. Bindnode Go types using `uint64` values are now lossless in round-trips through serialization to codecs that support the full range (DAG-CBOR most notably). You can see all of these new features in action using Filecoin Go types, allowing a mapping between Go types, Data Model (`Node`) forms, and their DAG-CBOR serialized forms with [data-transfer vouchers](https://github.com/filecoin-project/go-fil-markets/pull/713). These features also allow us to interact with the original Go types, without modification, including `big.Int` serialization to `Bytes`, Filecoin `Signature` serialization to a byte-prefix discriminated `Bytes` and more. Since the Go types are unchanged, they can also simultaneously support [cbor-gen](https://github.com/whyrusleeping/cbor-gen) serialization, allowing an easier migration path. ### v0.17.0 _2022 Jun 15_ go-ipld-prime's release policy says that: > even numbers should be easy upgrades; odd numbers may change things In that spirit, this v0.17.0 release includes some potentially breaking changes. Although minor, they are marked below and they may lead to behavioral changes in your use of this library. #### 🛠 Breaking Changes * **Codecs**: * DAG-CBOR, DAG-JSON: [Error on `cid.Undef` links in dag{json,cbor} encoding](https://github.com/ipld/go-ipld-prime/pull/433) - previously, encoding Link nodes that were empty CIDs (uninitialized zero-value or explicitly `cid.Undef`) would have passed through the DAG-CBOR or DAG-JSON codecs, silently producing erroneous output that wouldn't successfully pass back through a decode. (Rod Vagg) * **Bindnode**: * [Panic early if API has been passed ptr-to-ptr](https://github.com/ipld/go-ipld-prime/pull/427) - previous usage of bindnode using pointers-to-pointers may have deferred (or in some cases avoided) panics until deeper usage of the API, this change makes it earlier to make it clear that pointer-to-pointer is not appropriate usage. (Rod Vagg) * **Build**: * [Drop Go 1.16.x testing & begin testing Go 1.18.x](https://github.com/ipld/go-ipld-prime/pull/394) (Daniel Martí) * Note also that in this release, the [github.com/ipfs/**go-cid**](https://github.com/ipfs/go-cid) dependency is upgraded from 0.0.4 to 0.2.0 which includes a breaking change with the removal of the `cid.Codecs` and `cid.CodecToStr` maps which may disruptive. See [the go-cid@0.2.0 release page for details](https://github.com/ipfs/go-cid/releases/tag/v0.2.0). #### 🔦 Highlights * **Data Model**: * [Introduce `UIntNode` interface, used within DAG-CBOR codec to quietly support full uint64 range](https://github.com/ipld/go-ipld-prime/pull/413) (Rod Vagg) * **Bindnode**: * Fuzzing and hardening for production use (Daniel Martí) * Refuse to decode empty union values (Daniel Martí) * [Allow nilable types for IPLD `optional`/`nullable`](https://github.com/ipld/go-ipld-prime/pull/401) (Daniel Martí) * [More helpful error message for common enum value footgun](https://github.com/ipld/go-ipld-prime/pull/430) (Rod Vagg) * [Infer links and `Any` from Go types](https://github.com/ipld/go-ipld-prime/pull/432) (Rod Vagg) * **Schemas**: * DMT: Proper checking for unknown union members (Daniel Martí) * DMT: Enum representations must be valid members (Daniel Martí) * DMT: Reject duplicate or missing union representation members (Daniel Martí) * DSL: [Support `stringjoin` struct representation and `stringprefix` union representation](https://github.com/ipld/go-ipld-prime/pull/397) (Eric Evenchick) * DMT, DSL: [Enable inline types](https://github.com/ipld/go-ipld-prime/pull/404) (Rod Vagg) * **Patch**: * [Add initial version of IPLD Patch feature](https://github.com/ipld/go-ipld-prime/pull/350) (Eric Myhre) *(helped across the line by mauve and Rod Vagg)* * **Codecs**: * DAG-CBOR: [Reject extraneous content after valid (complete) CBOR object](https://github.com/ipld/go-ipld-prime/pull/386) (Rod Vagg) * DAG-CBOR: [add `DecodeOptions.ExperimentalDeterminism`](https://github.com/ipld/go-ipld-prime/pull/390) (currently only checking map sorting order) (Daniel Martí) * Printer: [Fix printing of floats](https://github.com/ipld/go-ipld-prime/pull/412) (Dustin Long) * DAG-JSON: [Add option to not parse beyond end of structure](https://github.com/ipld/go-ipld-prime/pull/435) (Petar Maymounkov) * **Build**: * Fix [macOS](https://github.com/ipld/go-ipld-prime/pull/400) and [Windows](https://github.com/ipld/go-ipld-prime/pull/405) testing (Rod Vagg) * [Fix 32-bit build support](https://github.com/ipld/go-ipld-prime/pull/407) (Rod Vagg) * [Make staticcheck and govet happy across codebase](https://github.com/ipld/go-ipld-prime/pull/406) (Rod Vagg) * Enable full [unified-ci](https://github.com/protocol/.github) GitHub Actions suite, including auto-updating (Rod Vagg) * [Enable dependabot, with monthly checks](https://github.com/ipld/go-ipld-prime/pull/417) (and update all dependencies) (Rod Vagg) Special thanks to **Daniel Martí** for many bindnode improvements and hardening, fuzzing across the library and improvements to the Schema DMT and DSL. ### v0.16.0 _2022 March 09_ - New: `traversal.WalkTransforming` is finally implemented! (It's been a stub for quite a while.) This works similarly to the other transform features, but can do more than change to the structure during a single walk. - New: Selectors support partial/ranged match on bytes or strings nodes. (This is also a new feature for Selectors, recently specified.) [[#375](https://github.com/ipld/go-ipld-prime/pull/375); seealso specs in [ipld#184](https://github.com/ipld/ipld/pull/184)] - New: there's a `datamodel.LargeBytesNode` interface, which makes it possible to handle "large" blobs of bytes as a `Node`, without necessarily forcing them all into memory at once. (This is optional; you add the methods to match the interface if your Node implementation supports the feature.) [[#372](https://github.com/ipld/go-ipld-prime/pull/372)] - Slightly more specifically: this interface is `Node` plus a method that returns an `io.ReadSeeker`. (Pretty standard golang I/O and byte slice management concepts should carry you from there in the usual ways.) - This is a **really big deal** -- for example, this means that an [ADL](https://ipld.io/docs/advanced-data-layouts/) can support reading of arbitrarily large bytes without an issue. (Hello, transparently readable large sharded blobs!) - New: there's a "resume" (or, skipahead) mechanism for traversals and selectors. Engage it by simply setting the `traversal.Config.StartAtPath` field. [[#358](https://github.com/ipld/go-ipld-prime/pull/358)] - New: `dagcbor` now has a `EncodedLength(Node) int` function, which can calculate the expected serial message length without actually encoding. (The usefulness of this may be situational, but it's there if you want it.) - Improved: `bindnode`, yet again, in more ways that can easily be summarized. - Better support for pointers in more places in your golang types. - Many panics either fixed or routed into calmer errors. - Unsigned intergers are now supported in your golang types. - Some fixes for AssignNode working correctly (e.g. at the type or representation level, as appropriate; sometimes previously it would use the type level incorrectly). - Various fixes to handling absent fields correctly. - A `datamodel.Node` can now be used for an `any` field. - Fixed: selectors now behave correctly for a recursion clause that just contains a recursion edge immedately. (It's still not a sensible selector, really, but it's valid.) Previously this would panic, which was nasty. - Fixed: `bindnode` now correctly doesn't include absent fields in the count of length when looking at the representation-level view of structs. - Improved: all our batteries-included codecs double check while encoding that the number iterator steps over a map matches its self-reported length. (This doesn't matter in many cases, but does defend you a little better against a `Node` implementation with a bug, if you happen to be so unlucky.) - Improved: miscellaneous performance work in the `schema/*` area. Thank you to @mvdan, @warpfork, @hannahhoward, @rvagg, @willscott, @arajasek and others for all their work that went into making this release (as well as all the point releases in v0.14.x leading up to it) happen. Finally, please note that we're starting to try out some new (and slightly more formal) governance and review and merge processes. Check out https://github.com/ipld/go-ipld-prime/issues/370 for more information. The aim is to make things generally more inclusive and involve more contributors! This is still experimental and may be subject to change, but if you'd like to have better expectations about who can review and what the process should be like, we hope this will be a step in a helpful direction. (Feedback about this experiment welcome!) ### v0.14.x (There were releases `v0.14.1`, `v0.14.2`, `v0.14.3`, and `v0.14.4` -- but all were in rapid succession, very minor, and hitting the same areas; we'll keep the notes brief and condensed.) - New: Selectors can include clauses for signalling the use of ADLs! [[#301](https://github.com/ipld/go-ipld-prime/pull/301); seealso specs in [ipld#149](https://github.com/ipld/ipld/pull/149)+[ipld#170](https://github.com/ipld/ipld/pull/170)] - Also kindly note that there are expected to be many ways of signalling ADL invocations -- this is only one of them. See the IPLD website for more on this topic as a whole: https://ipld.io/docs/advanced-data-layouts/signalling/ - Improved: `bindnode`, in ways more various than can easily be summarized. - The `cidlink.Link` type can be bound to links in the data. - Enums are now supported. - The `any` typekind is now supported. - Improved: both the `schema/dmt` and `schema/dsl` packages (and in some cases, the `schema` package itself) continue to be improved and become more complete. - Structs with tuple representation are now supported. - Enums with int representation are now supported. - The `any` typekind is now supported. - Changed: the dag-json codec will tolerate padded base64 in bytes content upon read. It does so silently. (It is not still possible to emit this kind of serial data with this library; it is noncanonical.) [[#309](https://github.com/ipld/go-ipld-prime/pull/309)] - Changed: the cbor and dag-cbor codec will now tolerate CBOR's "undef" token. It will coerce it to a null token when reading. Previously, encountering the undef token would result in a parse error. (It is still not possible to emit this token with this library.) [[#308](https://github.com/ipld/go-ipld-prime/pull/308)] - New: the `traversal` package gained a `WalkLocal` function. This simply does a walk that does not cross any links. ### v0.14.0 _2021 November 11_ This release is a smooth-sailing release, and mostly contains new features, quality-of-life improvements, and some significant improvements to the completeness and usability of features that have been in development across previous releases. There shouldn't be a lot of surprises, and upgrading should be easy. Some of the biggest improvements include: `bindnode` now supports most IPLD features and is increasingly stable; the `schema` system now has functioning `schema/dmt` and `schema/dsl` packages, and can parse schema documents smoothly(!); if you haven't seen the `printer` package that first quietly appeared in `v0.12.2`, you should definitely check it out now; and we have some new `storage` APIs that might be worth checking out, too. There are also many, many other smaller improvements. See the complete list and further deatils below (and don't forget to check out the notes under the other `v0.12.*` headings, if you haven't absorbed those updates already, too!): - New: `datamodel.Copy`: a helper function to do a shallow copy from one node to another. - You don't often need this, because nodes are supposed to be immutable! But it still sometimes comes in handy, for example, if you want to change the memory layout you're using by moving data into a different node implementation. - Improved: documentation of APIs. (Especially, for subtler bits like `NodeAssembler.AssignNode`.) - New: `datamodel.Link` now requires a `Binary()` function. In contrast to `Link.String()` (which is supposed to return something printable), `Link.Binary()` should give you the rawest thing possible. (It's equivalent to `go-cid.CID.KeyString`.) - New: **a new storage API**, including one **batteries-included** filesystem storage implementation, and **adapters** to several other different storage APIs. [[#265](https://github.com/ipld/go-ipld-prime/pull/265), [#279](https://github.com/ipld/go-ipld-prime/pull/279)] - The primary goal of this is the "batteries included" part: using the new `storage/fsstore` package, you should now be able to make simple applications with IPLD and use a simple sharded disk storage system (it'll look vaguely like a `.git/objects` directory), and do it in about five minutes and without pulling in any additional complex dependencies. - If you want to develop new storage systems or make adapters to them: the APIs in `storage` package are designed to be implemented easily. - The `storage` APIs are designed entirely around types found in the golang standard library. You do not need to import anything in the `storage` package in order to implement its interfaces! - The minimal APIs that a storage system has to implement are _very_ small. Two functions. Every additional feature, or optimization that you can offer: those all have their own interfaces, and we use feature-detection on them. You can implement as much or as little as you like. - As a user of the storage APIs: use the functions in the `storage` package. Those functions take a storage system as a parameter, and will do feature detection _for you_. - This means you can always write your code to call the APIs you _want_, and the `storage` functions will figure out how to map it onto the storage system that you _have_ (whatever it supports) in the most efficient way it can. - As a user of the `LinkSystem` API: you can ignore most of this! If you want to use the new `storage` APIs, there are setup methods on `LinkSystem` that will take them as a parameter. If you have existing code wired up with the previous APIs, it still works too. - As someone who already has code and wonders how to migrate: - If you're using the `linking.Storage*Opener` API: you don't have to do anything. Those still work too. - If you were using code from other repos like `ipfs/go-ipfs-blockstore` or `ipfs/go-datastore` or so on: those have adapters now in the `storage/*adapter` packages! You should now be able to use those more easily, with less custom glue code. (There's also now a migration readme in the repo root: check that out.) - If you would like to ask: "is it fast?" -- yes. You'll find that the new `storage/fsstore`, our batteries-included filesystem storage system, is comparable (or beating) the `go-ds-flatfs` package that you may have been using in the past. (More benchmarks and any performance improvement patches will of course be welcome -- but at the very least, there's no reason to hold back on using the new system.) - New: `LinkSystem` has some new methods: `LoadRaw` and `LoadPlusRaw` give you the ability to get data model nodes loaded, and _also_ receive the raw binary blobs. - This can be useful if you're building an application that's piping data around to other serial APIs without necessarily transforming it. (No need to reserialize if that's your journey.) - New: a CLI tool has begun development! - ... and almost immediately been removed again, to live in its own repo: check out https://github.com/ipld/go-ipldtool . - Improved: many more things about `bindnode`. - `bindnode` now understands `go-cid.CID` fields. - Kinded unions are much more completely supported. - Many TODO panics have gone away, replaced by finished features. - `bindnode` will increasingly check that the golang types you give it can be structurally matched to the schema if you provide one, which gives better errors earlier, and increases the ease and safety of use drastically. - Improved: the `schema/dmt` and `schema/dsl` packages are increasingly complete. - There are also now helper functions in the root package which will do the whole journey of "load a file, parse the Schema DSL, compile and typecheck the DMT, and give you the type info in handy golang interfaces", all at once! Check out `ipld.LoadSchema`! - New: there is a codegen feature for `bindnode` which will produce very terse golang structs matching a schema and ready to be bound back to `bindnode`! - This competes with the older `gengo` code generator -- by comparison, the `bindnode` code generator produces much, _much_ less code. (However, be advised that the performance characteristics are probably also markedly different; and we do not have sufficient benchmarks to comment on this at this time.) - Internal: many tests are being ported to `quicktest`. There should be no external impact to this, but we look forward to removing some of the other test libraries from our dependency tree in the near future. - Improved: `printer` now supports links and bytes! - Improved: `printer` is now more resilient and works even on relatively misbehaved `Node` implementations, such as those which implement `schema.TypedNode` but then rudely and nonsensically return nil type info. (We don't expect all code to be resilient against misbehaved `Node` implementations... but for a debug tool in particular? It's good to have it handle as much as it can.) This, and the last few releases tagged in the `v0.12.*` series, include invaluable contributions from @mvdan, @warpfork, @rvagg, @willscott, @masih, @hannahhoward, @aschmahmann, @ribasushi, and probably yet more others who have contributed through code and design reviews, or by using these libraries and demanding they continue to become better. Thanks to each and every one of the people who carry this project forward! ### v0.12.3 _2021 September 30_ (This is a minor release; we'll keep the notes brief.) - Fixed: using `SkipMe` in a traversal now skips only that subtree of nodes, not the remainder of the block! [[#251](https://github.com/ipld/go-ipld-prime/pull/251)] - New: `traversal` features now have budgets! You can set a "budget" value, and watch it monotonically decrement as your operations procede. This makes it easy to put limits on the amount of work you'll do. [[#260](https://github.com/ipld/go-ipld-prime/pull/260)] - New: `traversal` features can be configured to visit links they encounter only once (and ignore them if seen again). [[#252](https://github.com/ipld/go-ipld-prime/pull/252)] - Note that this is not without caveats: this is not merely an optimization; enabling it _may_ produce logically different outcomes, depending on what your selector is. This is because links are ignored when seen again, even if they're seen for a different _reason_, via a different path, etc. - Fixed: a very nasty off-by-one in unions produced by the "gogen" codegen. [[#257](https://github.com/ipld/go-ipld-prime/pull/257)] - Improved: the test suites for typed nodes now provide much better coverage (to prevent something like the above from happening again, even in other implementations). - New: `schema/dsl`! This package contains parsers for the IPLD Schema DSL, and produces data structures in `schema/dmt` form. - Removed: other misc partially-complete packages. (This will surely bother no one; it's just cleanup.) - Removed: `codec/jst`. If you were using that, [`jst` has its own repo](https://github.com/warpfork/go-jst/) now. - Improved: `traversal` now uses the error wrapping ("`%w`") feature in more places. - Changed: `printer` keeps empty maps and lists and strings on a single line. - Changed: `schema.TypeName` is now just an alias of `string`. This may result in somewhat less casting; or, you might not notice it. - Improved: the `schema/dmt` package continues to be improved and become more complete. - Some changes also track fixes in the schema spec, upstream. (Or caused those fixes!) - New/Improved: the `schema` package describes several more things which it always should have. Enums, for example. ### v0.12.2 _2021 September 8_ (This is a minor release; we'll keep the notes brief.) - New: the `printer` package has appeared, and aims to provide an information-rich, debug-readable, human-friendly output of data from an IPLD node tree. [[#238](https://github.com/ipld/go-ipld-prime/pull/238/)] - This works for both plain data model data, and for typed data, and annotates type information if present. - Note that this is _not_ a codec: it's specifically _richer_ than that. Conversely, this printer format is not designed to be parsed back to data model data. Use a codec for a codec's job; use the printer for debugging and inspection jobs. - Fixed/Improved: more things about the `bindnode` system. (It's still early and improving fast.) - Fixed: json codec, cbor codec, and their dag variants all now return ErrUnexpectedEOF in the conditions you'd expect. (Previously they sometimes just returned EOF, which could be surprising.) - Changed/Improved: the `schema/dmt` package is now implemented using `bindnode`, and there's a more complete `Compile()` feature. (This is still very early, in this tag. More to come here soon.) ### v0.12.1 _2021 August 30_ (This is a minor release; we'll keep the notes brief.) - Fixed/Improved: many things about the `bindnode` system. (It's still early and improving fast.) - Changed: the strings for `schema.TypeKind_*` are lowercase. (The docs and specs all act this way, and always have; it was a strange error for this code to have titlecase.) - New: the root package contains more helper methods for encoding and decoding operations ### v0.12.0 _2021 August 19_ This release is a momentous one. It contains a sizable refactor: we've extracted some of the most key interfaces to a new package, called `datamodel`! It's also an even numbered release tag, which we generally use to indicate "upgrading should be smooth sailing". Surprisingly, despite the magnitude of the refactor, we mean that, too. Golang's "alias" feature has been used _heavily_ for this change process, and downstream code that worked on the previous release should continue to work on this release too, without syntactic changes. Why did we do this? The root package, `ipld`, is now going to be a place where we can put helpful functions. Synthesis functions that put all the pieces of IPLD together for you. The functions you're probably looking for; the high-level stuff that gets work done. Previously, the root package was _guts_: the lowest level interfaces, the more core stuff... which was cool to see (arguably), but tended not to be the things you'd want to see _first_ as a new user. And because everything _else_ in the world depended on those interface, we could never put interesting high-level functions in the same package (or if we tried, compilation would fail, because of import cycles)... which meant any time we wanted to add helper functions for getting useful work done, we'd be stuck cramming them off into subpackages somewhere. While this worked, the discoverability for a new user was terribly arduous. We hope this pivot to how we organize the code helps you find your way through IPLD! We haven't yet added many of the new helper features to the updated root package. Those will come in the very near future. (Follow along with commits on the master branch if you want to try the new APIs early!) This release is being made just to cover the refactor, before we steam along any further. Your existing code should continue working without changes because the root `ipld` package still contains all the same types -- just as aliases. You can choose to update your code to use the types where they've moved to (which is mostly the `datamodel` package), or, if you prefer... just leave it as-is. Some aliases may be removed over time; if so, they'll be marked with a comment to that effect, and there should be plenty of warning and time to change. In some cases, continuing to use the `ipld` package directly will remain acceptable indefinitely. The new intention is that common work should often be possible to do only by importing the `ipld` package, and users should only need to dive into the more specific subpackages if they been to need direct access to more detailed APIs for performance or other reasons. That's it for the big refactor news. There's also some sweet new features in bindnode, and a few other important fixes to recently introduced features. In detail: - Changed: that massive refactor, described above. Gosh it's big. [[#228](https://github.com/ipld/go-ipld-prime/pull/228)] - New: the selectors system is tested against the language-agnostic selector specs, from the IPLD specs+docs repo! [[#231](https://github.com/ipld/go-ipld-prime/pull/231)] - This uses a new fixture format, called [testmark](https://github.com/warpfork/go-testmark#what-is-the-testmark-format), which is managed by a library called [go-testmark](https://pkg.go.dev/github.com/warpfork/go-testmark). - The fixtures are drawn in by a git submodule. The actual fixture content remains in the [ipld/ipld](https://github.com/ipld/ipld/) repo. - These new tests will be run if you have cloned the git submodule (and of course, by CI). If you do not clone the submodule that contains the fixtures, the tests will quietly skip. - We hope this will be a template for how to do more testing in the future, while keeping it closely coordinated with specs, and in sync with other implementations of IPLD in other languages! - Improved: bindnode: in a variety of ways. [[#226](https://github.com/ipld/go-ipld-prime/pull/226)] - Several error messages are improved. - Kinded unions support complex recipients even for string kinds. (E.g., putting a struct with stringjoin representation inside a kinded union now works correctly.) - Stringprefix unions now work even with no explicit delimiter. - Please note that bindnode is, and remains, considered experimental. While we're improving it, it's still something to use at your own risk. - Changed/Improved: bindnode: unions are now handled completely differently (and much better). [[#223](https://github.com/ipld/go-ipld-prime/pull/223)] - In short: now they expect a golang struct which has a field for each of the possible members, and each of them should be a pointer. This is type safe and works reasonably idiomatically in golang. - This is a fairly huge improvement, because it fixes the "bindnode unions force downshift into anonymous types" problem, which was tracked as [issue#210](https://github.com/ipld/go-ipld-prime/issues/210). - Fixed: the selector `ExploreRecursive.stopAt` feature now actually... works. It was completely broken when it was introduced in the last release. (Tests. They're important.) [[#229](https://github.com/ipld/go-ipld-prime/pull/229)] - Notice how we've also now got selector tests driven by fixtures appearing in this release. Hopefully that decreases the odds of something like this happening again. ### v0.11.0 _2021 August 12_ This release is an odd numbered release, which means it may contain breaking changes. Unfortunately, the changes here may be particularly, tricky, as well -- for the most part, they're not compile-time detectable. They're behavioral changes. Much more subtle. Run tests on your systems before accepting these changes. Specifically: several codecs now enforce sorting when emitting serial data. There's also some details of what's changing that makes it milder than it first sounds: most of the changes are around codecs becoming *more* spec-compliant. So, for example, if you were using another IPLD library that always enforced sorting on e.g. DAG-CBOR, you won't be surprised or experience it much like a "change" when using this version of go-ipld-prime, which now also enforces such sorting in that codec. Also! At least one huge and awesome new feature: `bindnode`. This is a new implementation of `ipld.Node` which can bind to native golang structures using reflection, which provides a new and easy-to-use way to move data in and out of golang structures (or traverse them, etc!) with IPLD interfaces and codecs. See the full change list for details: - New: some new helpful constructors for making Selectors out of serial forms can now be found in the `traversal/selector/parse` package. [[#199](https://github.com/ipld/go-ipld-prime/pull/199)] - Some constants are also included which show some examples of creating common selectors from JSON. - Fixed: cbor, dag-cbor, json, and dag-json codecs now all accept parsing a block that contains just a null token alone. (Previously, this returned an "unexpected EOF" error, which was silly.) [[#217](https://github.com/ipld/go-ipld-prime/pull/217)] - Fixed (upstream): json floats are actually supported. (You might've had this already, if anything dragged in a newer version of the `refmt` library. We just make sure to require this ourselves in our `go.mod` file now.) [[#215](https://github.com/ipld/go-ipld-prime/pull/215)] - New: Selectors now support some kinds of conditions. Specifically, `ExploreRecursive` clauses can contain a `stopAt` condition, and the condition system now supports `Condition_IsLink`, which can be used to do an equality check for CIDs. [[#214](https://github.com/ipld/go-ipld-prime/pull/214)] - Fixed: in codegen'd types, the `LinkTargetNodePrototype` on links was returning the wrong prototype; now it returns the right one. [[#211](https://github.com/ipld/go-ipld-prime/pull/211)] - New: `schema.TypedPrototype` interface, which is like `ipld.NodePrototype` but also has methods for asking `Type() schema.Type` and `Representation() ipld.NodePrototype`, both of which should probably instantly make sense to you. [[#195](https://github.com/ipld/go-ipld-prime/pull/195)] - Changed: the dag-json and dag-cbor codecs now apply sorting. [[#203](https://github.com/ipld/go-ipld-prime/pull/203), [#204](https://github.com/ipld/go-ipld-prime/pull/204)] - This means all serial data created with these codecs is sorted as advised by their respective specifications. Previously, the implementations of these codecs was order-preserving, and emitted data in whatever order the `ipld.Node` yielded it. - There may be new performance costs originating from this sorting. - The codecs do not reject other orderings when parsing serial data. The `ipld.Node` trees resulting from deserialization will still preserve the serialized order. However, it has now become impossible to re-encode data in that same preserved order. - If doing your own encoding, there are customization options in `dagcbor.EncodeOptions.MapSortMode` and `dagjson.EncodeOptions.MapSortMode`. (However, note that these options are not available to you while using any systems that only operate in terms of multicodec codes.) - _Be cautious of this change._ It is now extremely easy to write code which puts data into an `ipld.Node` in memory in one order, then save and load that data using these codecs, and end up with different data as a result because the sorting changes the order of data. For some applications, this may not be a problem; for others, it may be surprising. In particular, mind this carefully in the presense of other order-sensitive logic -- for example, such as when using Selectors, whose behaviors also depend on ordering of data returned when iterating over an `ipld.Node`. - Fixed/Changed: the dag-json codec no longer emits whitespace (!). It is now spec-compliant. [[#202](https://github.com/ipld/go-ipld-prime/pull/202)] - This means hashes of content produced by dag-json codec will change. This is unfortunate, but the previous implementation was woefully and wildly out of sync with the spec, and addressing that is a predominating concern. - Removed: `fluent/quip` has been dropped. `fluent/qp` is superior. `fluent/quip` was too easy to use incorrectly, so we no longer offer it. [[#197](https://github.com/ipld/go-ipld-prime/pull/197)] - This was an experimental package introduced a few releases ago, together with caveats that we may choose to drop it. The warning was purposeful! We don't believe that this will be too painful of a change; not many things depended on the `fluent/quip` variant, and those that did should not be difficult to rewrite to `fluent/qp`. - New: `node/basic.Chooser` is a function that implements `traversal.LinkTargetNodePrototypeChooser`. It's a small handy quality-of-life increase if you need to supply such a function, which is common. [[#198](https://github.com/ipld/go-ipld-prime/pull/198)] - New: `bindnode`! **This is a huge feature.** The beginnings of it may have been visible in v0.10.0, but it's grown into a usable thing we're ready to talk about. - Bindnode lets you write golang types and structures, and "bind" them into being IPLD Nodes and supporting Data Model operations by using golang reflection. - The result of working with `bindnode` is somewhere between using basicnode and using codegen: it's going to provide some structural constraints (like codegen) and provide moderate performance (it lets you use structs rather than memory-expensive maps; but reflection is still going to be slower than codegen). - However, most importantly, `bindnode` is *nice to use*. It doesn't have a huge barrier to entry like codegen does. - `bindnode` can be used with _or without_ IPLD Schemas. For basic golang types, a schema can be inferred automatically. For more advanced features (e.g. any representation customization), you can provide a Schema. - Please note that though it is now usable, bindnode remains _in development_. There is not yet any promise that it will be frozen against changes. - In fact, several changes are expected; in particular, be advised there is some sizable change expected around the shape of golang types expected for unions. - Improved: tests for behavior of schema typed nodes are now extracted to a package, where they are reusable. - The same tests now cover the `bindnode` implementation, as well as being used in tests of our codegen outputs. - Previously, these tests were already mostly agnostic of implementation, but had been thrown into packages in a way that made them hard to reuse. - Improved (or Fixed, depending on your point of view): dag-json codec now supports bytes as per the spec. [[#166](https://github.com/ipld/go-ipld-prime/pull/166),[#216](https://github.com/ipld/go-ipld-prime/pull/216)] - Bytes are encoded in roughly this form: `{"/":{"bytes":"base64data"}}`. - Note: the json codec does _not_ include this behavior; this is behavior specific to dag-json. ### v0.10.0 _2021 June 02_ v0.10.0 is a mild release, containing _no_ breaking changes, but lots of cool new stuff. Update at your earliest convenience. There's a bunch of cool new features in here, some of which are significant power-ups for the ecosystem (e.g. the `NodeReifier` API), so we recommend updating as soon as possible. There's also some sizable performance improvements available for generated code, so go forth and update your generated code too! Check out the full feature list: - New: an `ipld.DeepEqual` method lets you easily compare two `ipld.Node` for equality. (This is useful in case you have nodes with two different internal implementations, different memory layouts, etc, such that native golang equality would not be semantically correct.) [[#174](https://github.com/ipld/go-ipld-prime/pull/174)] - New: the multicodec package exposes a `multicodec.Registry` type, and also some `multicodec.List*` methods. [[#172](https://github.com/ipld/go-ipld-prime/pull/172), [#176](https://github.com/ipld/go-ipld-prime/pull/176)] - Please be cautious of using these `List*` methods. It's very possible to create race conditions with these, especially if using them on the global default registry instance. If we detect that these access methods seem to produce a source of bugs and design errors in downstream usage, they will be removed. Consider doing whatever you're doing by buildling your own registry systems, and attaching whatever semantics your system desires to those systems, rather than shoehorning this intentionally limited system into doing things it isn't made to do. - Improved: the dag-json codec now actually supports bytes! (Perhaps surprisingly, this was a relatively recent addition to the dag-json spec. We've now caught up with it.) [[#166](https://github.com/ipld/go-ipld-prime/pull/166)] - Improved: the codegen system now gofmt's the generated code immediately. You no longer need to do this manually in a separate step. [[#163](https://github.com/ipld/go-ipld-prime/pull/163)] - Improved: the codegen system is slightly faster at emitting code (due to use of more buffering during writes). [[#161](https://github.com/ipld/go-ipld-prime/pull/161)] - Improved: the codegen system will now avoid pointers in the generated "Maybe" types, if they're known to be small in memory (and thus, reasonable to inline). [[#160](https://github.com/ipld/go-ipld-prime/pull/160)] - This is quite likely to result in performance improvements for most programs, as it decreases the number of small memory allocations done, and amount of time spent on dereferencing, cache misses, etc. Some workloads demonstrated over 10% speed increases, and 40% decreases in allocation counts. (Of course, run your own benchmarks; not all workloads are equal.) - New: `ipld.LinkSystem` now contains a "reification" hook system. **This is really cool.** - The center of this is the `ipld.LinkSystem.NodeReifier` field, and the `ipld.NodeReifier` function type. - The `ipld.NodeReifier` function type is simply `func(LinkContext, Node, *LinkSystem) (Node, error)`. - The purpose and intention of this is: you can use this hooking point in order to decide where to engage advanced IPLD features like [ADLs](https://ipld.io/glossary/#adl). One can use a `NodeReifier` to decide what ADLs to use and when... even when in the middle of a traversal. - For example: one could write a NodeReifier that says "when I'm in a path that ends with '`foosys/*/hamt`', i'm going to try to load that as if it's a HAMT ADL". With that hook in place, you'd then be able to walks over whole forests of data with `traversal.*` functions, and they would automatically load the relevant ADL for you transparently every time that pattern is encountered, without disrupting or complicating the walk. - In the future, we might begin to offer more structural and declaratively configurable approaches to this, and eventually, attempt to standardize them. For now: you can build any solution you like using this hook system. (And we'll probably plug in any future declarative systems via these same hooks, too.) - All this appeared in [#158](https://github.com/ipld/go-ipld-prime/pull/158). - New: `ipld.LinkSystem` now contains a boolean flag for `TrustedStorage`. If set to true, it will cause methods like `Load` to _skip hashing_ when loading content. **_Do not do this unless you know what you're doing._** [[#149](https://github.com/ipld/go-ipld-prime/pull/149)] - New: a json (as opposed to dag-json) codec is now available from this repo. It does roughly what you'd expect. (It's like dag-json, but explicitly rejects encoding links and bytes, and correspondingly does not have dag-json's special decoding behaviors that produce those kinds.) [[#152](https://github.com/ipld/go-ipld-prime/pull/152)] - New: a cbor (as opposed to dag-cbor) codec is now available from this repo. Same story as the json codec: it just explicitly doesn't support links (because you should use dag-cbor if you want that). [[#153](https://github.com/ipld/go-ipld-prime/pull/153)] This contained a ton of contributions from lots of people: especially thanks to @mvdan, @hannahhoward, and @willscott for invaluable contributions. ### v0.9.0 _2021 March 15_ v0.9.0 is a pretty significant release, including several neat new convenience features, but most noticeably, significantly reworking how linking works. Almost any code that deals with storing and linking data will need some adaptation to handle this release. We're sorry about the effort this may require, but it should be worth it. The new LinkSystem API should let us introduce a lot more convenience features in the future, and do so *without* pushing additional breakage out to downstream users; this is an investment in the future. The bullet points below contain all the fun details. Note that a v0.8.0 release version has been skipped. We use odd numbers to indicate the existence of significant changes; and while usually we try to tag an even-number release between each odd number release so that migrations can be smoothed out, in this case there simply weren't enough interesting points in between to be worth doing so. - Change: linking has been significantly reworked, and now primarily works through the `ipld.LinkSystem` type. - This is cool, because it makes a lot of things less circuitous. Previously, working with links was a complicated combination of Loader and Storer functions, the Link interface contained the Load method, it was just... complicated to figure out where to start. Now, the answer is simple and constant: "Start with LinkSystem". Clearer to use; clearer to document; and also coincidentally a lot clearer to develop for, internally. - The PR can be found here: https://github.com/ipld/go-ipld-prime/pull/143 - `Link.Load` -> `LinkSystem.Load` (or, new: `LinkSystem.Fill`, which lets you control memory allocation more explicitly). - `LinkBuilder.Build` -> `LinkSystem.Store`. - `LinkSystem.ComputeLink` is a new feature that produces a Link without needing to store the data anywhere. - The `ipld.Loader` function is now most analogous to `ipld.BlockReadOpener`. You now put it into use by assigning it to a `LinkLoader`'s `StorageReadOpener` field. - The `ipld.Storer` function is now most analogous to `ipld.BlockWriteOpener`. You now put it into use by assigning it to a `LinkLoader`'s `StorageWriteOpener` field. - 99% of the time, you'll probably start with `linking/cid.DefaultLinkSystem()`. You can assign to fields of this to customize it further, but it'll get you started with multihashes and multicodecs and all the behavior you expect when working with CIDs. - (So, no -- the `cidlink` package hasn't gone anywhere. Hopefully it's a bit less obtrusive now, but it's still here.) - The `traversal` package's `Config` struct now uses a `LinkSystem` instead of a `Loader` and `Storer` pair, as you would now probably expect. - If you had code that was also previously passing around `Loader` and `Storer`, it's likely a similar pattern of change will be the right direction for that code. - In the _future_, further improvements will come from this: we're now much, much closer to making a bunch of transitive dependencies become optional (especially, various hashers, which currently, whenever you pull in the `linking/cid` package, come due to `go-cid`, and are quite large). When these improvements land (again, they're not in this release), you'll need to update your applications to import hashers you need if they're not in the golang standard library. For now: there's no change. - Change: multicodec registration is now in the `go-ipld-prime/multicodec` package. - Previously, this registry was in the `linking/cid` package. These things are now better decoupled. - This will require packages which register codecs to make some very small updates: e.g. `s/cidlink.RegisterMulticodecDecoder/multicodec.RegisterDecoder/`, and correspondingly, update the package imports at the top of the file. - New: some pre-made storage options (e.g. satisfying the `ipld.StorageReadOpener` and `ipld.StorageWriteOpener` function interfaces) have appeared! Find these in the `go-ipld-prime/storage` package. - Currently this only includes a simple in-memory storage option. This may be useful for testing and examples, but probably not much else :) - These are mostly intended to be illustrative. You should still expect to find better storage mechanisms in other repos. - Change: some function names in codec packages are ever-so-slightly updated. (They're verbs now, instead of nouns, which makes sense because they're functions. I have no idea what I was thinking with the previous naming. Sorry.) - `s/dagjson.Decoder/dagjson.Decode/g` - `s/dagjson.Decoder/dagjson.Encode/g` - `s/dagcbor.Decoder/dagcbor.Decode/g` - `s/dagcbor.Encoder/dagcbor.Encode/g` - If you've only been using these indirectly, via their multicodec indicators, you won't have to update anything at all to account for this change. - New: several new forms of helpers to make it syntactically easy to create new IPLD data trees with golang code! - Check out the `go-ipld-prime/fluent/quip` package! See https://github.com/ipld/go-ipld-prime/pull/134, where it was introduced, for more details. - Check out the `go-ipld-prime/fluent/qp` package! See https://github.com/ipld/go-ipld-prime/pull/138, where it was introduced, for more details. - Both of these offer variations on `fluent` which have much lower costs to use. (`fluent` incurs allocations during operation, which has a noticable impact on performance if used in a "hot" code path. Neither of these two new solutions do!) - For now, both `quip` and `qp` will be maintained. They have similar goals, but different syntaxes. If one is shown drastically more popular over time, we might begin to consider deprecating one in favor of the other, but we'll need lots of data before considering that. - We won't be removing the `fluent` package anytime soon, but we probably wouldn't recommend building new stuff on it. `qp` and `quip` are both drastically preferable for performance reasons. - New: there is now an interface called `ipld.ADL` which can be used for a certain kind of feature detection. - This is an experimental new concept and likely subject to change. - The one key trait we've found all ADLs tend to share right now is that they have a "synthesized" view and "substrate" view of their data. So: the `ipld.ADL` interface states that a thing is an `ipld.Node` (for the synthesized view), and from it you should be able to access a `Substrate() ipld.Node`, and that's about it. ### v0.7.0 _2020 December 31_ v0.7.0 is a small release that makes a couple of breaking changes since v0.6.0. However, the good news is: they're all very small changes, and we've kept them in a tiny group, so if you're already on v0.6.0, this update should be easy. And we've got scripts to help you. There's also one cool new feature: `traversal.FocusedTransform` is now available to help you make mutations to large documents conveniently. - Change: all interfaces and APIs now use golang `int64` rather than golang `int`. [#125](https://github.com/ipld/go-ipld-prime/pull/125) - This is necessary because the IPLD Data Model specifies that integers must be "at least 2^53" in range, and so since go-ipld-prime may also be used on 32-bit architectures, it is necessary that we not use the `int` type, which would fail to be Data Model-compliant on those architectures. - The following GNU sed lines should assist this transition in your code, although some other changes that are more difficult automate may also be necessary: ``` sed -ri 's/(func.* AsInt.*)\/\1int64/g' **/*.go sed -ri 's/(func.* AssignInt.*)\/\1int64/g' **/*.go sed -ri 's/(func.* Length.*)\/\1int64/g' **/*.go sed -ri 's/(func.* LookupByIndex.*)\/\1int64/g' **/*.go sed -ri 's/(func.* Next.*)\/\1int64/g' **/*.go sed -ri 's/(func.* ValuePrototype.*)\/\1int64/g' **/*.go ``` - Change: we've renamed the types talking about "kinds" for greater clarity. `ipld.ReprKind` is now just `ipld.Kind`; `schema.Kind` is now `schema.TypeKind`. We expect to use "kind" and "typekind" consistently in prose and documentation from now on, as well. [#127](https://github.com/ipld/go-ipld-prime/pull/127) - Pretty much everyone who's used this library has said "ReprKind" didn't really make sense as a type name, so, uh, yeah. You were all correct. It's fixed now. - "kind" now always means "IPLD Data Model kind", and "typekind" now always means "the kinds which an IPLD Schema type can have". - You can find more examples of how we expect to use this in a sentence from now on in the discussion that lead to the rename: https://github.com/ipld/go-ipld-prime/issues/94#issuecomment-745307919 - The following GNU sed lines should assist this transition in your code: ``` sed -ri 's/\/TypeKind/g' **/*.go sed -i 's/ReprKind/Kind/g' **/*.go ``` - Feature: `traversal.FocusedTransform` works now! :tada: You can use this to take a node, say what path inside it you want to update, and then give it an updated value. Super handy. [#130](https://github.com/ipld/go-ipld-prime/pull/130) ### v0.6.0 _2020 December 14_ v0.6.0 is a feature-packed release and has a few bugfixes, and _no_ significant breaking changes. Update at your earliest convenience. Most of the features have to do with codegen, which we now consider to be in **alpha** -- go ahead and use it! (We're starting to self-host some things in it, so any changes will definitely be managed from here on out.) A few other small handy helper APIs have appeared as well; see the detailed notes for those. Like with the last couple of releases, our intent is to follow this smooth-sailing change with another release shortly which will include some minor but noticable API changes, and that release may require you to make some code changes. Therefore, we suggest upgrading to this one first, beacuse it's an easy waypoint before the next change. - Feature: codegen is a reasonably usable alpha! We now encourage trying it out (but still only for those willing to experience an "alpha" level of friction -- UX still rough, and we know it). - Consult the feature table in the codegen package readme: many major features of IPLD Schemas are now supported. - Structs with tuple representations? Yes. - Keyed unions? Yes. - Structs with stringjoin representations? Yes. Including nested? _Yes_. - Lots of powerful stuff is now available to use. - See [the feature table in the codegen readme](https://github.com/ipld/go-ipld-prime/blob/v0.6.0/schema/gen/go/README.md#completeness) for details. - Many generated types now have more methods for accessing them in typed ways (in addition to the usual `ipld.Node` interfaces, which can access the same data, but lose explicit typing). [#106](https://github.com/ipld/go-ipld-prime/pull/106) - Maps and lists now have both lookup methods and iterators which know the type of the child keys and values explicitly. - Cool: when generating unions, you can choose between different implementation strategies (favoring either interfaces, or embedded values) by using Adjunct Config. This lets you tune for either speed (reduced allocation count) or memory footprint (less allocation size, but more granular allocations). - See notes in [#60](https://github.com/ipld/go-ipld-prime/pull/60) for more detail on this. We'll be aiming to make configurability of this more approachable and better documented in future releases, as we move towards codegen tools usable as CLI tools. - Cyclic references in types are now supported. - ... mostly. Some manual configuration may sometimes be required to make sure the generated structure wouldn't have an infinite memory size. We'll keep working on making this smoother in the future. - Field symbol overrides now work properly. (E.g., if you have a schema with a field called "type", you can make that work now. Just needs a field symbol override in the Adjunct Config when doing codegen!) - Codegen'd link types now implemented the `schema.TypedLinkNode` interface where applicable. - Structs now actually validate all required fields are present before allowing themselves to finish building. Ditto for their map representations. - Much more testing. And we've got a nice new declarative testcase system that makes it easier to write descriptions of how data should behave (at both the typed and representation view levels), and then just call one function to run exhaustive tests to make sure it looks the same from every inspectable API. - Change: codegen now outputs a fixed set of files. (Previously, it output one file per type in your schema.) This makes codegen much more managable; if you remove a type from your schema, you don't have to chase down the orphaned file. It's also just plain less clutter to look at on the filesystem. - Demo: as proof of the kind of work that can be done now with codegen, we've implemented the IPLD Schema schema -- the schema that describes IPLD Schema declarations -- using codegen. It's pretty neat. - Future: we'll be replacing most of the current current `schema` package with code based on this generated stuff. Not there yet, though. Taking this slow. - You can see the drafts of this, along with new features based on it, in [#107](https://github.com/ipld/go-ipld-prime/pull/107). - Feature: the `schema` typesystem info packages are improved. - Cyclic references in types are now supported. - (Mind that there are still some caveats about this when fed to codegen, though.) - Graph completeness is now validated (e.g. missing type references emit useful errors)! - Feature: there's a `traversal.Get` function. It's like `traversal.Focus`, but just returns the reached data instead of dragging you through a callback. Handy. - Feature/bugfix: the DAG-CBOR codec now includes resource budgeting limits. This means it's a lot harder for a badly-formed (or maliciously formed!) message to cause you to run out of memory while processing it. [#85](https://github.com/ipld/go-ipld-prime/pull/85) - Bugfix: several other panics from the DAG-CBOR codec on malformed data are now nice politely-returned errors, as they should be. - Bugfix: in codegen, there was a parity break between the AssembleEntry method and AssembleKey+AssembleValue in generated struct NodeAssemblers. This has been fixed. - Minor: ErrNoSuchField now uses PathSegment instead of a string. You probably won't notice (but this was important interally: we need it so we're able to describe structs with tuple representations). - Bugfix: an error path during CID creation is no longer incorrectly dropped. (I don't think anyone ever ran into this; it only handled situations where the CID parameters were in some way invalid. But anyway, it's fixed now.) - Performance: when `cidlink.Link.Load` is used, it will do feature detection on its `io.Reader`, and if it looks like an already-in-memory buffer, take shortcuts that do bulk operations. I've heard this can reduce memory pressure and allocation counts nicely in applications where that's a common scenario. - Feature: there's now a `fluent.Reflect` convenience method. Its job is to take some common golang structs like maps and slices of primitives, and flip them into an IPLD Node tree. [#81](https://github.com/ipld/go-ipld-prime/pull/81) - This isn't very high-performance, so we don't really recommend using it in production code (certainly not in any hot paths where performance matters)... but it's dang convenient sometimes. - Feature: there's now a `traversal.SelectLinks` convenience method. Its job is to walk a node tree and return a list of all the link nodes. [#110](https://github.com/ipld/go-ipld-prime/pull/110) - This is both convenient, and faster than doing the same thing using general-purpose Selectors (we implemented it as a special case). - Demo: you can now find a "rot13" ADL in the `adl/rot13adl` package. This might be useful reference material if you're interested in writing an ADL and wondering what that entails. [#98](https://github.com/ipld/go-ipld-prime/pull/98) - In progress: we've started working on some new library features for working with data as streams of "tokens". You can find some of this in the new `codec/codectools` package. - Functions are available for taking a stream of tokens and feeding them into a NodeAssembler; and for taking a Node and reading it out as a stream of tokens. - The main goal in mind for this is to provide reusable components to make it easier to implement new codecs. But maybe there will be other uses for this feature too! - These APIs are brand new and are _extremely subject to change_, much more so than any other packages in this repo. If you work with them at this stage, _do_ expect to need to update your code when things shift. ### v0.5.0 _2020 July 2_ v0.5.0 is a small release -- it just contains a bunch of renames. There are _no_ semantic changes bundled with this (it's _just_ renames) so this should be easy to absorb. - Renamed: `NodeStyle` -> `NodePrototype`. - Reason: it seems to fit better! See https://github.com/ipld/go-ipld-prime/issues/54 for a full discussion. - This should be a "sed refactor" -- the change is purely naming, not semantics, so it should be easy to update your code for. - This also affects some package-scoped vars named `Style`; they're accordingly also renamed to `Prototype`. - This also affects several methods such as `KeyStyle` and `ValueStyle`; they're accordingly also renamed to `KeyPrototype` and `ValuePrototype`. - Renamed: `(Node).Lookup{Foo}` -> `(Node).LookupBy{Foo}`. - Reason: The former phrasing makes it sound like the "{Foo}" component of the name describes what it returns, but in fact what it describes is the type of the param (which is necessary, since Golang lacks function overloading parametric polymorphism). Adding the preposition should make this less likely to mislead (even though it does make the method name moderately longer). - This should be a "sed refactor" -- the change is purely naming, not semantics, so it should be easy to update your code for. - Renamed: `(Node).Lookup` -> `(Node).LookupNode`. - Reason: The shortest and least-qualified name, 'Lookup', should be reserved for the best-typed variant of the method, which is only present on codegenerated types (and not present on the Node interface at all, due to Golang's limited polymorphism). - This should be a "sed refactor" -- the change is purely naming, not semantics, so it should be easy to update your code for. (The change itself in the library was fairly literally `s/Lookup(/LookupNode(/g`, and then `s/"Lookup"/"LookupNode"/g` to catch a few error message strings, so consumers shouldn't have it much harder.) - Note: combined with the above rename, this method overall becomes `(Node).LookupByNode`. - Renamed: `ipld.Undef` -> `ipld.Absent`, and `(Node).IsUndefined` -> `(Node).IsAbsent`. - Reason: "absent" has emerged as a much, much better description of what this value means. "Undefined" sounds nebulous and carries less meaning. In long-form prose docs written recently, "absent" consistently fits the sentence flow much better. Let's just adopt "absent" consistently and do away with "undefined". - This should be a "sed refactor" -- the change is purely naming, not semantics, so it should be easy to update your code for. ### v0.4.0 v0.4.0 contains some misceleanous features and documentation improvements -- perhaps most notably, codegen is re-introduced and more featureful than previous rounds -- but otherwise isn't too shocking. This tag mostly exists as a nice stopping point before the next version coming up (which is planned to include several API changes). - Docs: several new example functions should now appear in the godoc for how to use the linking APIs. - Feature: codegen is back! Use it if you dare. - Generated code is now up to date with the present versions of the core interfaces (e.g., it's updated for the NodeAssembler world). - We've got a nice big feature table in the codegen package readme now! Consult that to see which features of IPLD Schemas now have codegen support. - There are now several implemented and working (and robustly tested) examples of codegen for various representation strategies for the same types. (For example, struct-with-stringjoin-representation.) Neat! - This edition of codegen uses some neat tricks to not just maintain immutability contracts, but even prevent the creation of zero-value objects which could potentially be used to evade validation phases on objects that have validation rules. (This is a bit experimental; we'll see how it goes.) - There are oodles and oodles of deep documentation of architecture design choices recorded in "HACKME_*" documents in the codegen package that you may enjoy if you want to contribute or understand why generated things are the way they are. - Testing infrastructure for codegen is now solid. Running tests for the codegen package will: exercise the generation itself; AND make sure the generated code compiles; AND run behavioral tests against it: the whole gamut, all from regular `go test`. - The "node/gendemo" package contains a real example of codegen output... and it's connected to the same tests and benchmarks as other node implementations. (Are the gen'd types fast? yes. yes they are.) - There's still lots more to go: interacting with the codegen system still requires writing code to interact with as a library, as we aren't shipping a CLI frontend to it yet; and many other features are still in development as well. But you're welcome to take it for a spin if you're eager! - Feature: introduce JSON Tables Codec ("JST"), in the `codec/jst` package. This is a codec that emits bog-standard JSON, but leaning in on the non-semantic whitespace to produce aligned output, table-like, for pleasant human reading. (If you've used `column -t` before in the shell: it's like that.) - This package may be a temporary guest in this repo; it will probably migrate to its own repo soon. (It's a nice exercise of our core interfaces, though, so it incubated here.) - I'm quietly shifting the versioning up to the 0.x range. (Honestly, I thought it was already there, heh.) That makes this this "v0.4". ### v0.0.3 v0.0.3 contained a massive rewrite which pivoted us to using NodeAssembler patterns. Code predating this version will need significant updates to match; but, the performance improvements that result should be more than worth it. - Constructing new nodes has a major pivot towards using "NodeAssembler" pattern: https://github.com/ipld/go-ipld-prime/pull/49 - This was a massively breaking change: it pivoted from bottom-up composition to top-down assembly: allocating large chunks of structured memory up front and filling them in, rather than stitching together trees over fragmented heap memory with lots of pointers - "NodeStyle" and "NodeBuilder" and "NodeAssembler" are all now separate concepts: - NodeStyle is more or less a builder factory (forgive me -- but it's important: you can handle these without causing allocations, and that matters). Use NodeStyle to tell library functions what kind of in-memory representation you want to use for your data. (Typically `basicnode.Style.Any` will do -- but you have the control to choose others.) - NodeBuilder allocates and begins the assembly of a value (or a whole tree of values, which may be allocated all at once). - NodeAssembler is the recursive part of assembling a value (NodeBuilder implements NodeAssembler, but everywhere other than the root, you only use the NodeAssembler interface). - Assembly of trees of values now simply involves asking the assembler for a recursive node to give you assemblers for the keys and/or values, and then simply... using them. - This is much simpler (and also faster) to use than the previous system, which involved an awkward dance to ask about what kind the child nodes were, get builders for them, use those builders, then put the result pack in the parent, and so forth. - Creating new maps and lists now accepts a size hint argument. - This isn't strictly enforced (you can provide zero, or even a negative number to indicate "I don't know", and still add data to the assembler), but may improve efficiency by reducing reallocation costs to grow structures if the size can be estimated in advance. - Expect **sizable** performance improvements in this version, due to these interface changes. - Some packages were renamed in an effort to improve naming consistency and feel: - The default node implementations have moved: expect to replace `impl/free` in your package imports with `node/basic` (which is an all around better name, anyway). - The codecs packages have moved: replace `encoding` with `codec` in your package imports (that's all there is to it; nothing else changed). - Previous demos of code generation are currently broken / disabled / removed in this tag. - ...but they'll return in future versions, and you can follow along in branches if you wish. - Bugfix: dag-cbor codec now correctly handles marshalling when bytes come after a link in the same object. [[53](https://github.com/ipld/go-ipld-prime/pull/53)] ### v0.0.2 - Many various performance improvements, fixes, and docs improvements. - Many benchmarks and additional tests introduced. - Includes early demos of parts of the schema system, and early demos of code generation. - Mostly a checkpoint before beginning v0.0.3, which involved many large API reshapings. ### v0.0.1 - Our very first tag! - The central `Node` and `NodeBuilder` interfaces are already established, as is `Link`, `Loader`, and so forth. You can already build generic data handling using IPLD Data Model concepts with these core interfaces. - Selectors and traversals are available. - Codecs for dag-cbor and dag-json are batteries-included in the repo. - There was quite a lot of work done before we even started tagging releases :) ================================================ FILE: HACKME.md ================================================ hackme ====== Design rational are documented here. This doc is not necessary reading for users of this package, but if you're considering submitting patches -- or just trying to understand why it was written this way, and check for reasoning that might be dated -- then it might be useful reading. It may also be an incomplete doc. It's been written opportunistically. If you don't understand the rationale for some things, try checking git history (many of the commit messages are downright bookish), or get in touch via a github issue, irc, matrix, etc and ask! about NodeAssembler and NodeBuilder ----------------------------------- See the godoc on these types. In short, a `NodeBuilder` is for creating a new piece of memory; a `NodeAssembler` is for instantiating some memory which you already have. Generally, you'll start any function using a `NodeBuilder`, but then continue and recurse by passing on the `NodeAssembler`. See the `./HACKME_builderBehaviors.md` doc for more details on high level rules and implementation patterns to look out for. about NodePrototype --------------- ### NodePrototype promises information without allocations You'll notice nearly every `ipld.NodePrototype` implementation is a golang struct type with _zero fields_. This is important. Getting a NodePrototype is generally expected to be "free" (i.e., zero allocations), while `NewBuilder` is allowed to be costly (usually causes at least one allocation). Zero-member structs can be referred to by an interface without requiring an allocation, which is how it's possible ensure `NodePrototype` are always "free" to refer to. (Note that a `NodePrototype` that bundles some information like ADL configuration will subvert this pattern -- but these are an exception, not the rule.) ### NodePrototype reported by a Node `ipld.NodePrototype` is a type that opaquely represents some information about how a node was constructed and is implemented. The general contract for what should happen when asking a node about its prototype (via the `ipld.Node.Prototype() NodePrototype` interface) is that prototype should contain effective instructions for how one could build a copy of that node, using the same implementation details. By example, if some node `n` was made as a `basicnode.plainString`, then `n.Prototype()` will be `basicnode.Prototype.String`, and `n.Prototype().NewBuilder().AssignString("xyz")` can be presumed to work. Note there are also limits to this: if a node was built in a flexible way, the prototype it reports later may only report what it is now, and not return that same flexibility again. By example, if something was made as an "any" -- i.e., via `basicnode.Prototype.Any.NewBuilder()`, and then *happened* to be assigned a string value -- the resulting node will still carry a `Prototype()` property that returns `basicnode.Prototype.String` -- **not** `basicnode.Prototype.Any`. #### NodePrototype meets generic transformation One of the core purposes of the `NodePrototype` interface (and all the different ways you can get it from existing data) is to enable the `traversal` package (or other user-written packages like it) to do transformations on data. // work-in-progress warning: generic transformations are not fully implemented. When implementing a transformation that works over unknown data, the signiture of function a user provides is roughly: `func(oldValue Node, acceptableValues NodePrototype) (Node, error)`. (This signiture may vary by the strategy taken by the transformation -- this signiture is useful because it's capable of no-op'ing; an alternative signiture might give the user a `NodeAssembler` instead of the `NodePrototype`.) In this situation, the transformation system determines the `NodePrototype` (or `NodeAssembler`) to use by asking the parent value of the one we're visiting. This is because we want to give the update function the ability to create any kind of value that would be accepted in this position -- not just create a value of the same prototype as the one currently there! It is for this reason the `oldValue.Prototype()` property can't be used directly. At the root of such a transformation, we use the `node.Prototype()` property to determine how to get started building a new value. #### NodePrototype meets recursive assemblers Asking for a NodePrototype in a recursive assembly process tells you about what kind of node would be accepted in an `AssignNode(Node)` call. It does *not* make any remark on the fact it's a key assembler or value assembler and might be wrapped with additional rules (such as map key uniqueness, field name expectations, etc). (Note that it's also not an exclusive statement about what `AssignNode(Node)` will accept; e.g. in many situations, while a `Prototype.MyStringType` might be the prototype returned, any string kinded node can be used in `AssignNode(Node)` and will be appropriately converted.) Any of these paths counts as "recursive assembly process": - `MapAssembler.KeyPrototype()` - `MapAssembler.ValuePrototype(string)` - `MapAssembler.AssembleKey().Prototype()` - `MapAssembler.AssembleValue().Prototype()` - `ListAssembler.ValuePrototype()` - `ListAssembler.AssembleValue().Prototype()` ### NodePrototype for carrying ADL configuration // work-in-progress warning: this is an intention of the design, but not implemented. ================================================ FILE: HACKME_builderBehaviors.md ================================================ hackme: NodeBuilder and NodeAssembler behaviors =============================================== high level rules of builders and assemblers ------------------------------------------- - Errors should be returned as soon as possible. - That means an error like "repeated key in map" should be returned by the key assembler! - Either 'NodeAssembler.AssignString' should return this (for simple keys on untyped maps, or on structs, etc)... - ... or 'MapAssembler.Finish' (in the case of complex keys in a typed map). - Logical integrity checks must be done locally -- recursive types rely on their contained types to report errors, and the recursive type wraps the assemblers of their contained type in order to check and correctly invalidate/rollback the recursive construction. - Recursive types tend to have a value assembler that wraps the child type's assembler in order to intercept relevant "finish" methods. - This is generally where that logic integrity check mentioned above is tracked; we need explicit confirmation that it *passes* before the parent's assembly should proceed. - Implementations may also need this moment to complete any assignment of the child value into position in the parent value. But not all implementations need this -- some will have had all the child assembler effects applying directly to the final memory positions. - Assemblers should invalidate themselves as soon as they become "finished". - For maps and lists, that means the "Finish" methods. - For all the other scalars, the "Assign*" method itself means finished. - Or in other words: whatever method returns an `error`, that's what makes that assembler "finished". - The purpose of this is to prevent accidental mutation after any validations have been performed during the "finish" processing. - Many methods must be called in the right order, and the user must not hold onto references after calling "finish" methods on them. - The reason this is important is to enable assembler systems to aggressively reuse memory, thus increasing performance. - Thus, if you hold onto NodeAssembler reference after being finished with it... you can't assume it'll explicitly error if you call further methods on it, because it might now be operating again... _on a different target_. - In recursive structures, calling AssembleKey or AssembleValue might return pointer-identical assemblers (per warning in previous bullet), but the memory their assembly is targeted to should always advance -- it should never target already-assembled memory. - (If you're thinking "the Rust memory model would be able to greatly enhance safety here!"... yes. Yes it would.) - When misuses of order are detected, these may cause panics (rather than error returns) (not all methods that can be so misused have error returns). detailed rules and expectations for implementers ------------------------------------------------ The expectations in the "happy path" are often clear. Here are also collected some details of exactly what should happen when an error has been reached, but the caller tries to continue anyway. - while building maps: - assigning a key with 'AssembleKey': - in case of success: clearly 'AssembleValue' should be ready to use next. - in case of failure from repeated key: - the error must be returned immediately from either the 'NodeAssembler.AssignString' or the 'MapAssembler.Finish' method. - 'AssignString' for any simple keys; 'MapAssembler.Finish' may be relevant in the case of complex keys in a typed map. - implementers take note: this implies the `NodeAssembler` returned by `AssembleKey` has some way to refer to the map assembler that spawned it. - no side effect should be visible if 'AssembleKey' is called again next. - (typically this doesn't require extra code for the string case, but it may require some active zeroing in the complex key case.) - (remember to reset any internal flag for expecting 'AssembleValue' to be used next, and decrement any length pointers that were optimistically incremented!) - n.b. the "no side effect" rule here is for keys, not for values. - TODO/REVIEW: do we want the no-side-effect rule for values? it might require nontrivial volumes of zeroing, and often in practice, this might be wasteful. - invalidation of assemblers: - is typically implemented by nil'ing the wip node they point to. - this means you get nil pointer dereference panics when attempting to use an assembler after it's finished... which is not the greatest error message. - but it does save us a lot of check code for a situation that the user certainly shouldn't get into in the first place. - (worth review: can we add that check code without extra runtime cost? possibly, because the compiler might then skip its own implicit check branches. might still increase SLOC noticeably in codegen output, though.) - worth noting there's a limit to how good this can be anyway: it's "best effort" error reporting: see the remarks on reuse of assembler memory in "overall rules" above. - it's systemically critical to not yield an assembler _ever again in the future_ that refers to some memory already considered finished. - even though we no longer return intermediate nodes, there's still many ways this could produce problems. For example, complicating (if not outright breaking) COW sharing of segments of data. - in most situations, we get this for free, because the child assembler methods only go "forward" -- there's no backing up, lists have no random index insertion or update support, and maps actively reject dupe keys. - if you *do* make a system which exposes any of those features... be very careful; you will probably need to start tracking "freeze" flags on the data in order to retain systemic sanity. ================================================ FILE: HACKME_mergeStrategy.md ================================================ hacking: merge strategies ========================= This is a short document about how the maintainers of this repo handle branches and merging. It's useful information for a developer wanting to contribute, but otherwise unimportant. --- We prefer to: 1. Do development on a branch; 2. Before merge, rebase onto master (if at all possible); - if there are individual commit hashes that should be preserved because they've been referenced outside the project, say so; we don't want to have to presume this by default. - rebasing your commits (or simply staging them carefully the first time; `git add -p` is your friend) for clarity of later readers is greatly appreciated. 3. Merge, using the "--no-ff" strategy. The github UI does fine at this. There are a couple of reasons we prefer this: - Squashing, if appropriate, can be done by the author. We don't use github's squash button because it's sometimes quite difficult to make a good combined commit message without effort to do so by the diff's author, so it's best left to that author to do themselves. - Generating a merge commit gives a good place for github to insert the PR link, if a PR has been used. This is good info to have if someone is later reading git history and wants to see links to where other discussion may have taken place. - We *do* like fairly linearly history. Emphasis on "fairly" -- it doesn't have to be perfectly lock-step ridgidly linear: but when doing `git log --graph`, we also want to not see more than a handful of lines running in parallel at once. (Too many parallel branches at once is both unpleasant to read and review later, and can indicative of developmental process issues, so it's a good heuristic to minimize for multiple reasons.) Rebasing before generating a merge commit does this: if consistently done, `git log --graph` will yield two parallel lines at all times. - Generating a merge commit, when combined with rebasing the commits on the branch right before merge, means `git log --graph` will group up the branch commits in a visually clear way. Preserving this relation can be useful. (Neither squashing nor rebase-without-merge approaches preserve this information.) Mind, all of these rules are heuristics and "rules of thumb". Some small changes are also perfectly reasonable to land with either a squash or rebase that appends them linearly onto history. The maintainers may choose strategies as they see fit depending on the size of the content and the level of interest in preserving individual commits and their messages and relational history. What does this mean for PRs? ---------------------------- - Please keep PRs rebased on top of master as much as possible. - If you decide you want multiple commits with distinct messages, fine. If you want to squash, also fine. - If you are linking to commit hashes and don't want them rebased, please comment about this; otherwise, it should not be presumed we'll keep exact commit hashes reachable. The maintainers also reserve the right to rebase or squash things at their own option; we'll comment explicitly if committing to not do so, and it should not otherwise be presumed. - If you're not comfortable with rebase: fine. Just be aware that if a PR branches off master for quite some time, and it does become ready for merge later, the maintainers are likely to squash/rebase your work for you before merging. ================================================ FILE: HACKME_releases.md ================================================ # Making go-ipld-prime Releases ## Versioning strategy go-ipld-prime follows **[WarpVer](https://gist.github.com/warpfork/98d2f4060c68a565e8ad18ea4814c25f)**, a form of SemVer that never bumps the major version number and uses minor version numbers to indicate degree of *changeness*: **even numbers should be easy upgrades; odd numbers may change things**. The patch version number is rarely used in this scheme. ## CHANGELOG.md There is a CHANGELOG.md, it should be relevant and updated. Notable items in the commit history since the last release should be included. Where possible and practical, links to relevant pull requests or other issues with discussions on the items should be included. To find the list of commits, it is recommended that you use a tool that can provide some extra metadata to help with matching commits to pull requests. [changelog-maker](https://github.com/nodejs/changelog-maker) can help with this (requires Node.js be installed and the `npx` command be available): ``` npx changelog-maker --start-ref=v0.16.0 --reverse=true --find-matching-prs=true --md=true ipld go-ipld-prime ``` Alternatively, you can use `git log` and perform mapping to pull requests manually, e.g.: ``` git log --all --graph --date-order --abbrev-commit --decorate --oneline ``` *(where `--start-ref` points to name of the previous release tag)* ### Curate and summarize The CHANGELOG should be informative for developers wanting to know what changes may pose a risk (highlight these!) and what changes introduce features they may be interested in using. 1. Group commits to subsystem to create a two-level list. Subsections can include "Data Model", "Schemas", "Bindnode", "Selectors", "Codecs", and the meta-category of "Build" to describe changes local to the repository and not necessarily relevant to API consumers. 2. If there are breaking, or potentially breaking changes, list them under a `#### 🛠 Breaking Changes` section. 3. Otherwise, prune the list of commits down to the set of changes relevant to users, and list them under a `#### 🔦 Highlights` section. Note that there is also a **Planned/Upcoming Changes** section near the top of the CHANGELOG.md. Update this to remove _done_ items and add other items that may be nearing completion but not yet released. ### Call-outs Add "special thanks" call-outs to individuals who have contributed meaningful changes to the release. ## Propose a release After updating the CHANGELOG.md entry, also bump the version number appropriately in **version.json** file so the auto-release procedure can take care of tagging for you. Commit and propose the changes via a pull request to ipld/go-ipld-prime. ## Release After a reasonable amount of time for feedback (usually at least a full global business day), the changes can be merged and a release tag will be created by the GitHub Actions. Use the GitHub UI to make a [release](https://github.com/ipld/go-ipld-prime/releases), copying in the contents of the CHANGELOG.md for that release. Drop in a note to the appropriate Matrix/Discord/Slack channel(s) for IPLD about the release. Optional: Protocol Labs staff can send an email to shipped@protocol.ai to describe the release, these are typically well-read and appreciated. ## Checklist Prior to opening a release proposal pull request, create an issue with the following markdown checklist to help ensure the requisite steps are taken. The issue can also be used to alert subscribed developers to the timeframe and the approximate scope of changes in the release. ```markdown * [ ] Add new h3 to `CHANGELOG.md` under **Released Changes** with curated and subsystem-grouped list of changes and links to relevant PRs * [ ] Highlight any potentially breaking or disruptive changes under "🛠 Breaking Changes", including extended descriptions to help users make compatibility judgements * [ ] Add special-thanks call-outs to contributors making significant contributions * [ ] Update **Planned/Upcoming Changes** section to remove completed items and add newly upcoming, but incomplete items * [ ] Bump version number appropriately in `version.json` * [ ] Propose release via pull request, merge after enough time for async global feedback * [ ] Create GitHub [release](https://github.com/ipld/go-ipld-prime/releases) with the new tag, copying the new `CHANGELOG.md` contents * [ ] Announce on relevant Discord/Matrix/Slack channel(s) * [ ] (Optional) Announce to shipped@protocol.ai ``` ================================================ FILE: LICENSE ================================================ The MIT License (MIT) Copyright (c) 2018 Eric Myhre Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: README.md ================================================ go-ipld-prime ============= `go-ipld-prime` is an implementation of the IPLD spec interfaces, a batteries-included codec implementations of IPLD for CBOR and JSON, and tooling for basic operations on IPLD objects (traversals, etc). API --- The API is split into several packages based on responsibly of the code. The most central interfaces are the base package, but you'll certainly need to import additional packages to get concrete implementations into action. Roughly speaking, the core package interfaces are all about the IPLD Data Model; the `codec/*` packages contain functions for parsing serial data into the IPLD Data Model, and converting Data Model content back into serial formats; the `traversal` package is an example of higher-order functions on the Data Model; concrete `ipld.Node` implementations ready to use can be found in packages in the `node/*` directory; and several additional packages contain advanced features such as IPLD Schemas. (Because the codecs, as well as higher-order features like traversals, are implemented in a separate package from the core interfaces or any of the Node implementations, you can be sure they're not doing any funky "magic" -- all this stuff will work the same if you want to write your own extensions, whether for new Node implementations or new codecs, or new higher-order order functions!) - `github.com/ipld/go-ipld-prime` -- imported as just `ipld` -- contains the core interfaces for IPLD. The most important interfaces are `Node`, `NodeBuilder`, `Path`, and `Link`. - `github.com/ipld/go-ipld-prime/node/basicnode` -- provides concrete implementations of `Node` and `NodeBuilder` which work for any kind of data, using unstructured memory. - `github.com/ipld/go-ipld-prime/node/bindnode` -- provides concrete implementations of `Node` and `NodeBuilder` which store data in native golang structures, interacting with it via reflection. Also supports IPLD Schemas! - `github.com/ipld/go-ipld-prime/traversal` -- contains higher-order functions for traversing graphs of data easily. - `github.com/ipld/go-ipld-prime/traversal/selector` -- contains selectors, which are sort of like regexps, but for trees and graphs of IPLD data! - `github.com/ipld/go-ipld-prime/codec` -- parent package of all the codec implementations! - `github.com/ipld/go-ipld-prime/codec/dagcbor` -- implementations of marshalling and unmarshalling as CBOR (a fast, binary serialization format). - `github.com/ipld/go-ipld-prime/codec/dagjson` -- implementations of marshalling and unmarshalling as JSON (a popular human readable format). - `github.com/ipld/go-ipld-prime/linking/cid` -- imported as `cidlink` -- provides concrete implementations of `Link` as a CID. Also, the multicodec registry. - `github.com/ipld/go-ipld-prime/schema` -- contains the `schema.Type` and `schema.TypedNode` interface declarations, which represent IPLD Schema type information. - `github.com/ipld/go-ipld-prime/node/typed` -- provides concrete implementations of `schema.TypedNode` which decorate a basic `Node` at runtime to have additional features described by IPLD Schemas. Getting Started --------------- Let's say you want to create some data programmatically, and then serialize it, or save it as [blocks]. You've got a ton of different options, depending on what golang convention you want to use: - the `qp` package -- [example](https://pkg.go.dev/github.com/ipld/go-ipld-prime/fluent/qp#example-package) - the `bindnode` system, if you want to use golang types -- [example](https://pkg.go.dev/github.com/ipld/go-ipld-prime/node/bindnode#example-Wrap-NoSchema), [example with schema](https://pkg.go.dev/github.com/ipld/go-ipld-prime/node/bindnode#example-Wrap-WithSchema) - or the [`NodeBuilder`](https://pkg.go.dev/github.com/ipld/go-ipld-prime/datamodel#NodeBuilder) interfaces, raw (verbose; not recommended) - or even some codegen systems! Once you've got a Node full of data, you can serialize it: https://pkg.go.dev/github.com/ipld/go-ipld-prime#example-package-CreateDataAndMarshal But probably you want to do more than that; probably you want to store this data as a block, and get a CID that links back to it. For this you use `LinkSystem`: https://pkg.go.dev/github.com/ipld/go-ipld-prime/linking#example-LinkSystem.Store Hopefully these pointers give you some useful getting-started focal points. The API docs should help from here on out. We also highly recommend scanning the [godocs](https://pkg.go.dev/github.com/ipld/go-ipld-prime) for other pieces of example code, in various packages! Let us know in [issues](https://github.com/ipld/go-ipld-prime/issues), [chat, or other community spaces](https://ipld.io/docs/intro/community/) if you need more help, or have suggestions on how we can improve the getting-started experiences! Other IPLD Libraries -------------------- The IPLD specifications are designed to be language-agnostic. Many implementations exist in a variety of languages. For overall behaviors and specifications, refer to the IPLD website, or its source, in IPLD meta repo: - https://ipld.io/ - https://github.com/ipld/ipld/ You should find specs in the `specs/` dir there, human-friendly docs in the `docs/` dir, and information about _why_ things are designed the way they are mostly in the `design/` directories. There are also pages in the IPLD website specifically about golang IPLD libraries, and your alternatives: https://ipld.io/libraries/golang/ ### distinctions from go-ipld-interface&go-ipld-cbor This library ("go ipld prime") is the current head of development for golang IPLD, and we recommend new developments in golang be done using this library as the basis. However, several other libraries exist in golang for working with IPLD data. Most of these predate go-ipld-prime and no longer receive active development, but since they do support a lot of other software, you may continue to seem them around for a while. go-ipld-prime is generally **serially compatible** with these -- just like it is with IPLD libraries in other languages. In terms of programmatic API and features, go-ipld-prime is a clean take on the IPLD interfaces, and chose to address several design decisions very differently than older generation of libraries: - **The Node interfaces map cleanly to the IPLD Data Model**; - Many features known to be legacy are dropped; - The Link implementations are purely CIDs (no "name" nor "size" properties); - The Path implementations are provided in the same box; - The JSON and CBOR implementations are provided in the same box; - Several odd dependencies on blockstore and other interfaces that were closely coupled with IPFS are replaced by simpler, less-coupled interfaces; - New features like IPLD Selectors are only available from go-ipld-prime; - New features like ADLs (Advanced Data Layouts), which provide features like transparent sharding and indexing for large data, are only available from go-ipld-prime; - Declarative transformations can be applied to IPLD data (defined in terms of the IPLD Data Model) using go-ipld-prime; - and many other small refinements. In particular, the clean and direct mapping of "Node" to concepts in the IPLD Data Model ensures a much more consistent set of rules when working with go-ipld-prime data, regardless of which codecs are involved. (Codec-specific embellishments and edge-cases were common in the previous generation of libraries.) This clarity is also what provides the basis for features like Selectors, ADLs, and operations such as declarative transformations. Many of these changes had been discussed for the other IPLD codebases as well, but we chose clean break v2 as a more viable project-management path. Both go-ipld-prime and these legacy libraries can co-exist on the same import path, and both refer to the same kinds of serial data. Projects wishing to migrate can do so smoothly and at their leisure. We now consider many of the earlier golang IPLD libraries to be defacto deprecated, and you should expect new features *here*, rather than in those libraries. (Those libraries still won't be going away anytime soon, but we really don't recommend new construction on them.) ### migrating **For recommendations on where to start when migrating:** see [README_migrationGuide](./README_migrationGuide.md). That document will provide examples of which old concepts and API names map to which new APIs, and should help set you on the right track. ### unixfsv1 Lots of people who hear about IPLD have heard about it through IPFS. IPFS has IPLD-native APIs, but IPFS *also* makes heavy use of a specific system called "UnixFSv1", so people often wonder if UnixFSv1 is supported in IPLD libraries. The answer is "yes" -- but it's not part of the core. UnixFSv1 is now treated as an [ADL](https://ipld.io/glossary/#adl), and a go-ipld-prime compatible implementation can be found in the [ipfs/go-unixfsnode](https://github.com/ipfs/go-unixfsnode) repo. Additionally, the codec used in UnixFSv1 -- dag-pb -- can be found implemented in the [ipld/go-codec-dagpb](https://github.com/ipld/go-codec-dagpb) repo. A "some assembly required" advisory may still be in effect for these pieces; check the readmes in those repos for details on what they support. The move to making UnixFSv1 a non-core system has been an arduous retrofit. However, framing it as an ADL also provides many advantages: - it demonstrates that ADLs as a plugin system _work_, and others can develop new systems in this pattern! - it has made pathing over UnixFSv1 much more standard and well-defined - this standardization means systems like [Selectors](https://ipld.io/glossary/#selectors) work naturally over UnixFSv1... - ... which in turn means anything using them (ex: CAR export; graphsync; etc) can very easily be asked to produce a merkle-proof for a path over UnixFSv1 data, without requiring the querier to know about the internals. Whew! We hope users and developers alike will find value in how these systems are now layered. Change Policy ------------- The go-ipld-prime library is ready to use, and we value stability highly. We make releases periodically. However, using a commit hash to pin versions precisely when depending on this library is also perfectly acceptable. (Only commit hashes on the master branch can be expected to persist, however; depending on a commit hash in a branch is not recommended. See [development branches](#development-branches).) We maintain a [CHANGELOG](CHANGELOG.md)! Please read it, when updating! We do make reasonable attempts to minimize the degree of changes to the library which will create "breaking change" experiences for downstream consumers, and we do document these in the changelog (often, even with specific migration instructions). However, we do also still recommend running your own compile and test suites as a matter of course after updating. You can help make developing this library easier by staying up-to-date as a downstream consumer! When we do discover a need for API changes, we typically try to introduce the new API first, and do at least one release tag in which the old API is deprecated (but not yet removed). We will all be able to develop software faster, together, as an ecosystem, if libraries can keep reasonably closely up-to-date with the most recent tags. ### Version Names When a tag is made, version number steps in go-ipld-prime advance as follows: 1. the number bumps when the lead maintainer says it does. 2. even numbers should be easy upgrades; odd numbers may change things. 3. the version will start with `v0.` until further notice. [This is WarpVer](https://gist.github.com/warpfork/98d2f4060c68a565e8ad18ea4814c25f). These version numbers are provided as hints about what to expect, but ultimately, you should always invoke your compiler and your tests to tell you about compatibility, as well as read the [changelog](CHANGELOG.md). ### Updating **Read the [CHANGELOG](CHANGELOG.md).** Really, read it. We put exact migration instructions in there, as much as possible. Even outright scripts, when feasible. An even-number release tag is usually made very shortly before an odd number tag, so if you're cautious about absorbing changes, you should update to the even number first, run all your tests, and *then* upgrade to the odd number. Usually the step to the even number should go off without a hitch, but if you *do* get problems from advancing to an even number tag, A) you can be pretty sure it's a bug, and B) you didn't have to edit a bunch of code before finding that out. ### Development branches The following are norms you can expect of changes to this codebase, and the treatment of branches: - The `master` branch will not be force-pushed. - (exceptional circumstances may exist, but such exceptions will only be considered valid for about as long after push as the "$N-second-rule" about dropped food). - Therefore, commit hashes on master are gold to link against. - All other branches *can* be force-pushed. - Therefore, commit hashes not reachable from the master branch are inadvisable to link against. - If it's on master, it's understood to be good, in as much as we can tell. - Changes and features don't get merged until their tests pass! - Packages of "alpha" developmental status may exist, and be more subject to change than other more finalized parts of the repo, but their self-tests will at least pass. - Development proceeds -- both starting from and ending on -- the `master` branch. - There are no other long-running supported-but-not-master branches. - The existence of tags at any particular commit do not indicate that we will consider starting a long running and supported diverged branch from that point, nor start doing backports, etc. ================================================ FILE: README_migrationGuide.md ================================================ A short guide to migrating to go-ipld-prime from other repos ============================================================ Here are some quick notes on APIs that you might've been using if you worked with IPLD in golang before go-ipld-prime, and a pointer to what you should check out for an equivalent in order to upgrade your code to use go-ipld-prime. (Let us know if there are more pointers we should add to this list to ease your journey, or someone else's future journey!) - Were you using [ipfs/go-datastore](https://pkg.go.dev/github.com/ipfs/go-datastore) APIs? - You can wrap those in `storage/dsadapter` and keep using them. You can also plug that into `linking.LinkSystem` to get higher level IPLD operations. - Or if you were only using datastore because of some specific implementation of it, like, say `flatfs`? Then check out the possibility of moving all the way directly to new implementations like `storage/fsstore`. - Were you using [ipfs/go-ipfs-blockstore](https://pkg.go.dev/github.com/ipfs/go-ipfs-blockstore) APIs? - You can wrap those in `storage/bsadapter` and keep using them. You can also plug that into `linking.LinkSystem` to get higher level IPLD operations. (This is almost exactly the same; we've just simplified in the interface, made it easier to implement, and cleaned up inconsistencies with the other interfaces in this migration guide which were already very very similar.) - Were you using [ipfs/go-blockservice](https://pkg.go.dev/github.com/ipfs/go-blockservice) APIs? - You can wrap those in `storage/bsrvadapter` and keep using them. You can also plug that into `linking.LinkSystem` to get higher level IPLD operations. - Be judicious about whether you actually want to do this. Plugging in the potential to experience unknown network latencies into code that's expecting predictable local lookup speeds may have undesirable performance outcomes. (But if you want to do it, go ahead...) - Were you using [ipfs/go-ipld-format.DAGService](https://pkg.go.dev/github.com/ipfs/go-ipld-format#DAGService)? - If you're using the `Add` and `Get` methods: those are now on `linking.LinkSystem`, as `Store` and `Load`. - If you're using the `*Many` methods: those don't have direct replacements; we don't use APIs that force that style of future-aggregation anymore, because it's bad for pipelining. Just use `Store` and `Load`. - Were you using [ipfs/go-ipld-format.Node](https://pkg.go.dev/github.com/ipfs/go-ipld-format#Node)? - That's actually a semantic doozy -- we use the word "node" _much_ differently now. See https://ipld.io/glossary/#node for an introduction. (Long story short: the old "node" was often more like a whole block, and the new "node" is more like an AST node: there's lots of "node" in a "block" now.) - There's an `datamodel.Node` interface -- it's probably what you want. (You just might have to update how you think about it -- see above bullet point about semantics shift.) It's also aliased as the `ipld.Node` interface here right in the root package, because it's such an important and central interface that you'll be using it all the time. - Are you specifically looking for how to get a list of links out of a block? That's a `LinkSystem.Load` followed by applying `traversal.SelectLinks` on the node it gives you: now you've got the list of links from that node (and its children, recursively, as long as they came from the same block of raw data), tada! - Are you looking for the equivalent of [ipfs/go-ipld-format.Link](https://pkg.go.dev/github.com/ipfs/go-ipld-format#Link)? - That's actually a feature specific to dag-pb, and arguably even specific to unixfsv1. There is no equivalent in go-ipld-prime as a result. But you'll find it in other libraries that are modern and go-ipld-prime based: see below. - Were you using some feature specific to dag-pb? - There's an updated dag-pb codec found in https://github.com/ipld/go-codec-dagpb -- it should have what you need. - Does it turn out you actually meant you're using some feature specific to unixfsv1? (Yeah -- that comes up a lot. It's one of the reasons we're deprecating so many of the old libraries -- they make it _really_ easy to confuse this.) Then hang on for the next bullet point, which is for you! :) - Were you using some features of unixfsv1? - Check out the https://github.com/ipfs/go-unixfsnode library -- it should have what you need. - Pathing, like what used to be found in ipfs/go-path? That's here now. - Sharding? That's here now too. - You probably don't need incentives if you're here already, but can we quickly mention that... you can use Selectors over these? Neat! If you're encountering more questions about how to migrate some code, please jump in to the ipld chat via either [matrix](https://matrix.to/#/#ipld:ipfs.io) or discord (or any other bridge) and ask! We want to grow this list of pointers to be as encompassing and helpful as possible. ================================================ FILE: adl/interface.go ================================================ package adl import ( "github.com/ipld/go-ipld-prime/datamodel" ) // ADL is an interface denoting an Advanced Data Layout, // which is something that supports all the datamodel.Node operations, // but may be doing so using some custom internal logic. // // For more details, see the docs at // https://ipld.io/docs/advanced-data-layouts/ . // // This interface doesn't specify much new behavior, but it does include // the requirement of a way to tell an examiner about your "substrate", // since this concept does seem to be present in all ADLs. type ADL interface { datamodel.Node // Substrate returns the underlying Data Model node, which can be used // to encode an ADL's raw layout. // // Note that the substrate of an ADL can contain other ADLs! Substrate() datamodel.Node } ================================================ FILE: adl/rot13adl/example_test.go ================================================ package rot13adl_test import ( "bytes" "context" "fmt" "strings" "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime/adl/rot13adl" "github.com/ipld/go-ipld-prime/codec/dagjson" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/linking" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipld/go-ipld-prime/must" "github.com/ipld/go-ipld-prime/storage/memstore" ) func ExampleReify_unmarshallingToADL() { // Create a NodeBuilder for the ADL's substrate. // Unmarshalling into this memory structure is optimal, // because it immediately puts data into the right memory layout for the ADL code to work on, // but you could use any other kind of NodeBuilder just as well and still get correct results. nb := rot13adl.Prototype.SubstrateRoot.NewBuilder() // Unmarshal -- using the substrate's nodebuilder just like you'd unmarshal with any other nodebuilder. err := dagjson.Decode(nb, strings.NewReader(`"n pbby fgevat"`)) fmt.Printf("unmarshal error: %v\n", err) // Use `Reify` to get the synthetic high-level view of the ADL data. substrateNode := nb.Build() syntheticView, err := rot13adl.Reify(substrateNode) fmt.Printf("reify error: %v\n", err) // We can inspect the synthetic ADL node like any other node! fmt.Printf("adl node kind: %v\n", syntheticView.Kind()) fmt.Printf("adl view value: %q\n", must.String(syntheticView)) // Output: // unmarshal error: // reify error: // adl node kind: string // adl view value: "a cool string" } func ExampleReify_loadingToADL() { // Create a NodeBuilder for the ADL's substrate. // Unmarshalling into this memory structure is optimal, // because it immediately puts data into the right memory layout for the ADL code to work on, // but you could use any other kind of NodeBuilder just as well and still get correct results. nb := rot13adl.Prototype.SubstrateRoot.NewBuilder() // Unmarshal -- using the substrate's nodebuilder just like you'd unmarshal with any other nodebuilder. err := dagjson.Decode(nb, strings.NewReader(`"n pbby fgevat"`)) fmt.Printf("unmarshal error: %v\n", err) substrateNode := nb.Build() // now save the node to storage lp := cidlink.LinkPrototype{Prefix: cid.Prefix{ Version: 1, Codec: 0x129, MhType: 0x13, MhLength: 4, }} linkSystem := cidlink.DefaultLinkSystem() storage := &memstore.Store{} linkSystem.SetReadStorage(storage) linkSystem.SetWriteStorage(storage) linkSystem.NodeReifier = func(_ linking.LinkContext, nd datamodel.Node, _ *linking.LinkSystem) (datamodel.Node, error) { return rot13adl.Reify(nd) } lnk, err := linkSystem.Store(linking.LinkContext{Ctx: context.Background()}, lp, substrateNode) fmt.Printf("storage error: %v\n", err) // reload from storage, but this time the NodeReifier function should give us the ADL syntheticView, err := linkSystem.Load(linking.LinkContext{Ctx: context.Background()}, lnk, rot13adl.Prototype.SubstrateRoot) fmt.Printf("load error: %v\n", err) // We can inspect the synthetic ADL node like any other node! fmt.Printf("adl node kind: %v\n", syntheticView.Kind()) fmt.Printf("adl view value: %q\n", must.String(syntheticView)) // Output: // unmarshal error: // storage error: // load error: // adl node kind: string // adl view value: "a cool string" } func ExampleR13String_creatingViaADL() { // Create a NodeBuilder for the ADL -- the high-level synthesized thing (not the substrate). nb := rot13adl.Prototype.Node.NewBuilder() // Create a ADL node via its builder. This is just like creating any other node in IPLD. nb.AssignString("woohoo") n := nb.Build() // We can inspect the synthetic ADL node like any other node! fmt.Printf("adl node kind: %v\n", n.Kind()) fmt.Printf("adl view value: %q\n", must.String(n)) // We can get the substrate view and examine that as a node too. // (This requires a cast to see that we have an ADL, though. Not all IPLD nodes have a 'Substrate' property.) substrateNode := n.(rot13adl.R13String).Substrate() fmt.Printf("substrate node kind: %v\n", substrateNode.Kind()) fmt.Printf("substrate value: %q\n", must.String(substrateNode)) // To marshal the ADL, just use marshal methods on its substrate as normal: var marshalBuffer bytes.Buffer err := dagjson.Encode(substrateNode, &marshalBuffer) fmt.Printf("marshalled: %v\n", marshalBuffer.String()) fmt.Printf("marshal error: %v\n", err) // Output: // adl node kind: string // adl view value: "woohoo" // substrate node kind: string // substrate value: "jbbubb" // marshalled: "jbbubb" // marshal error: } // It's worth noting that the builders for an ADL substrate node still return the substrate. // (This is interesting in contrast to Schemas, where codegenerated representation-level builders // yield the type-level node values (and not the representation level node).) // // To convert the substrate node to the high level synthesized view of the ADL, // use Reify as normal -- it's the same whether you've used the substrate type // or if you've used any other node implementation to hold the data. // // Future work: unmarshalling which can invoke an ADL mid-structure, // and automatically places the reified ADL in place in the larger structure. // // There will be several ways to do this (it hinges around "the signalling problem", // discussed in https://github.com/ipld/specs/issues/130 ): // // The first way is to use IPLD Schemas, which provide a signalling mechanism // by leaning on the schema, and the matching of shape of surrounding data to the schema, // as a way to determine where an ADL is expected to appear. // // A second mechanism could involve new unmarshal function contracts // which would ake a (fairly complex) argument that says what NodePrototype to use in certain positions. // This could be accomplished by use of Selectors. // (This would also have many other potential purposes -- implementing this in terms of NodePrototype selection is very multi-purpose, // and could be used for efficiency and misc tuning purposes, // for expecting a *schema* thing part way through, and so forth.) // ================================================ FILE: adl/rot13adl/rot13logic.go ================================================ package rot13adl import ( "strings" ) var replaceTable = []string{ "A", "N", "B", "O", "C", "P", "D", "Q", "E", "R", "F", "S", "G", "T", "H", "U", "I", "V", "J", "W", "K", "X", "L", "Y", "M", "Z", "N", "A", "O", "B", "P", "C", "Q", "D", "R", "E", "S", "F", "T", "G", "U", "H", "V", "I", "W", "J", "X", "K", "Y", "L", "Z", "M", "a", "n", "b", "o", "c", "p", "d", "q", "e", "r", "f", "s", "g", "t", "h", "u", "i", "v", "j", "w", "k", "x", "l", "y", "m", "z", "n", "a", "o", "b", "p", "c", "q", "d", "r", "e", "s", "f", "t", "g", "u", "h", "v", "i", "w", "j", "x", "k", "y", "l", "z", "m", } var unreplaceTable = func() []string { v := make([]string, len(replaceTable)) for i := 0; i < len(replaceTable); i += 2 { v[i] = replaceTable[i+1] v[i+1] = replaceTable[i] } return v }() // rotate transforms from the logical content to the raw content. func rotate(s string) string { return strings.NewReplacer(replaceTable...).Replace(s) } // unrotate transforms from the raw content to the logical content. func unrotate(s string) string { return strings.NewReplacer(unreplaceTable...).Replace(s) } ================================================ FILE: adl/rot13adl/rot13node.go ================================================ /* rot13adl is a demo ADL -- its purpose is to show what an ADL and its public interface can look like. It implements a "rot13" string: when creating data through the ADL, the user gives it a regular string; the ADL will create aninternal representation of it which has the characters altered in a reversible way. It provides reference and example materal, but it's very unlikely you want to use it in real situations ;) There are several ways to move data in and out of the ADL: - treat it like a regular IPLD map: - using the exported NodePrototype can be used to get a NodeBuilder which can accept keys and values; - using the resulting Node and doing lookup operations on it like a regular map; - load up raw substrate data and `Reify()` it into the synthesized form, and *then* treat it like a regular map: - this is handy if the raw data already parsed into Nodes. - optionally, use `SubstrateRootPrototype` as the prototype for loading the raw substrate data; any kind of Node is a valid input to Reify, but this one will generally have optimal performance. - take the synthesized form and inspect its substrate data: - the `Substrate()` method will return another datamodel.Node which is the root of the raw substrate data, and can be walked normally like any other datamodel.Node. */ package rot13adl import ( "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/node/mixins" "github.com/ipld/go-ipld-prime/schema" ) // -- Node --> var _ datamodel.Node = (R13String)(nil) type R13String = *_R13String type _R13String struct { raw string // the raw content, before our ADL lens is applied to it. synthesized string // the content that the ADL presents. calculated proactively from the original, in this implementation (though you could imagine implementing it lazily, in either direction, too). } func (*_R13String) Kind() datamodel.Kind { return datamodel.Kind_String } func (*_R13String) LookupByString(string) (datamodel.Node, error) { return mixins.String{TypeName: "rot13adl.R13String"}.LookupByString("") } func (*_R13String) LookupByNode(datamodel.Node) (datamodel.Node, error) { return mixins.String{TypeName: "rot13adl.R13String"}.LookupByNode(nil) } func (*_R13String) LookupByIndex(idx int64) (datamodel.Node, error) { return mixins.String{TypeName: "rot13adl.R13String"}.LookupByIndex(0) } func (*_R13String) LookupBySegment(seg datamodel.PathSegment) (datamodel.Node, error) { return mixins.String{TypeName: "rot13adl.R13String"}.LookupBySegment(seg) } func (*_R13String) MapIterator() datamodel.MapIterator { return nil } func (*_R13String) ListIterator() datamodel.ListIterator { return nil } func (*_R13String) Length() int64 { return -1 } func (*_R13String) IsAbsent() bool { return false } func (*_R13String) IsNull() bool { return false } func (*_R13String) AsBool() (bool, error) { return mixins.String{TypeName: "rot13adl.R13String"}.AsBool() } func (*_R13String) AsInt() (int64, error) { return mixins.String{TypeName: "rot13adl.R13String"}.AsInt() } func (*_R13String) AsFloat() (float64, error) { return mixins.String{TypeName: "rot13adl.R13String"}.AsFloat() } func (n *_R13String) AsString() (string, error) { return n.synthesized, nil } func (*_R13String) AsBytes() ([]byte, error) { return mixins.String{TypeName: "rot13adl.R13String"}.AsBytes() } func (*_R13String) AsLink() (datamodel.Link, error) { return mixins.String{TypeName: "rot13adl.R13String"}.AsLink() } func (*_R13String) Prototype() datamodel.NodePrototype { return _R13String__Prototype{} } // -- NodePrototype --> var _ datamodel.NodePrototype = _R13String__Prototype{} type _R13String__Prototype struct { // There's no configuration to this ADL. // A more complex ADL might have some kind of parameters here. // // The general contract of a NodePrototype is supposed to be that: // when you get one from an existing Node, // it should have enough information to create a new Node that // could "replace" the previous one in whatever context it's in. // For ADLs, that means it should carry most of the configuration. // // An ADL that does multi-block stuff might also need functions like a LinkLoader passed in through here. } func (np _R13String__Prototype) NewBuilder() datamodel.NodeBuilder { return &_R13String__Builder{} } // -- NodeBuilder --> var _ datamodel.NodeBuilder = (*_R13String__Builder)(nil) type _R13String__Builder struct { _R13String__Assembler } func (nb *_R13String__Builder) Build() datamodel.Node { if nb.m != schema.Maybe_Value { panic("invalid state: cannot call Build on an assembler that's not finished") } return nb.w } func (nb *_R13String__Builder) Reset() { *nb = _R13String__Builder{} } // -- NodeAssembler --> var _ datamodel.NodeAssembler = (*_R13String__Assembler)(nil) type _R13String__Assembler struct { w *_R13String m schema.Maybe // REVIEW: if the package where this Maybe enum lives is maybe not the right home for it after all. Or should this line use something different? We're only using some of its values after all. } func (_R13String__Assembler) BeginMap(sizeHint int64) (datamodel.MapAssembler, error) { return mixins.StringAssembler{TypeName: "rot13adl.R13String"}.BeginMap(0) } func (_R13String__Assembler) BeginList(sizeHint int64) (datamodel.ListAssembler, error) { return mixins.StringAssembler{TypeName: "rot13adl.R13String"}.BeginList(0) } func (na *_R13String__Assembler) AssignNull() error { // REVIEW: unclear how this might compose with some other context (like a schema) which does allow nulls. Probably a wrapper type? return mixins.StringAssembler{TypeName: "rot13adl.R13String"}.AssignNull() } func (_R13String__Assembler) AssignBool(bool) error { return mixins.StringAssembler{TypeName: "rot13adl.R13String"}.AssignBool(false) } func (_R13String__Assembler) AssignInt(int64) error { return mixins.StringAssembler{TypeName: "rot13adl.R13String"}.AssignInt(0) } func (_R13String__Assembler) AssignFloat(float64) error { return mixins.StringAssembler{TypeName: "rot13adl.R13String"}.AssignFloat(0) } func (na *_R13String__Assembler) AssignString(v string) error { switch na.m { case schema.Maybe_Value: panic("invalid state: cannot assign into assembler that's already finished") } na.w = &_R13String{ raw: rotate(v), synthesized: v, } na.m = schema.Maybe_Value return nil } func (_R13String__Assembler) AssignBytes([]byte) error { return mixins.StringAssembler{TypeName: "rot13adl.R13String"}.AssignBytes(nil) } func (_R13String__Assembler) AssignLink(datamodel.Link) error { return mixins.StringAssembler{TypeName: "rot13adl.R13String"}.AssignLink(nil) } func (na *_R13String__Assembler) AssignNode(v datamodel.Node) error { if v.IsNull() { return na.AssignNull() } if v2, ok := v.(*_R13String); ok { switch na.m { case schema.Maybe_Value: panic("invalid state: cannot assign into assembler that's already finished") } na.w = v2 na.m = schema.Maybe_Value return nil } if v2, err := v.AsString(); err != nil { return err } else { return na.AssignString(v2) } } func (_R13String__Assembler) Prototype() datamodel.NodePrototype { return _R13String__Prototype{} } ================================================ FILE: adl/rot13adl/rot13node_test.go ================================================ package rot13adl import ( "testing" qt "github.com/frankban/quicktest" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/node/basicnode" ) func TestLogicalNodeRoundtrip(t *testing.T) { // Build high level node. nb := Prototype.Node.NewBuilder() err := nb.AssignString("abcd") qt.Assert(t, err, qt.IsNil) n := nb.Build() // Inspect the high level node. s, err := n.AsString() qt.Check(t, err, qt.IsNil) qt.Check(t, s, qt.Equals, "abcd") } func TestNodeInternal(t *testing.T) { // Build high level node. nb := Prototype.Node.NewBuilder() err := nb.AssignString("abcd") qt.Assert(t, err, qt.IsNil) n := nb.Build() // Poke its insides directly to see that the transformation occurred. qt.Check(t, n.(*_R13String).synthesized, qt.Equals, "abcd") qt.Check(t, n.(*_R13String).raw, qt.Equals, "nopq") } func TestReify(t *testing.T) { t.Run("using unspecialized raw node", func(t *testing.T) { // Build substrate-shaped data using basicnode. sn := basicnode.NewString("nopq") // Reify it. synth, err := Reify(sn) // Inspect the resulting high level node. qt.Assert(t, err, qt.IsNil) qt.Check(t, synth.Kind(), qt.Equals, datamodel.Kind_String) s, err := synth.AsString() qt.Check(t, err, qt.IsNil) qt.Check(t, s, qt.Equals, "abcd") }) t.Run("using substrate node", func(t *testing.T) { // Build substrate-shaped data, in the substrate type right from the start. snb := Prototype.SubstrateRoot.NewBuilder() snb.AssignString("nopq") sn := snb.Build() // Reify it. synth, err := Reify(sn) // Inspect the resulting high level node. qt.Assert(t, err, qt.IsNil) qt.Check(t, synth.Kind(), qt.Equals, datamodel.Kind_String) s, err := synth.AsString() qt.Check(t, err, qt.IsNil) qt.Check(t, s, qt.Equals, "abcd") }) } func TestInspectingSubstrate(t *testing.T) { // Build high level node. nb := Prototype.Node.NewBuilder() err := nb.AssignString("abcd") qt.Assert(t, err, qt.IsNil) n := nb.Build() // Ask it about its substrate, and inspect that. sn := n.(R13String).Substrate() // TODO: It's unfortunate this is only available as a concrete type cast: we should probably make a standard feature detection interface with `Substrate()`. // Is it reasonable to make this part of a standard feature detection pattern, // and make that interface reside in the main IPLD package? Or in an `adl` package that contains such standard interfaces? ss, err := sn.AsString() qt.Check(t, err, qt.IsNil) qt.Check(t, ss, qt.Equals, "nopq") } ================================================ FILE: adl/rot13adl/rot13prototypes.go ================================================ package rot13adl // Prototype embeds a NodePrototype for every kind of Node implementation in this package. // This includes both the synthesized node as well as the substrate root node // (other substrate interior node prototypes are not exported here; // it's unlikely they'll be useful outside of the scope of the ADL's implementation package.) // // You can use it like this: // // rot13adl.Prototype.Node.NewBuilder() //... // // and: // // rot13adl.Prototype.SubstrateRoot.NewBuilder() // ... var Prototype prototype // This may seem a tad mundane, but we do it for consistency with the way // other Node implementation packages (like basicnode) do this: // it's a convention for "pkgname.Prototype.SpecificThing.NewBuilder()" to be defined. type prototype struct { Node _R13String__Prototype SubstrateRoot _Substrate__Prototype } // REVIEW: In ADLs that use codegenerated substrate types defined by an IPLD Schema, the `Prototype.SubstrateRoot` field... // should it be a `_SubstrateRoot__Prototype`, or a `_SubstrateRoot__ReprPrototype`? // Probably the latter, because the only thing an external user of this package should be interested in is how to read data into memory in an optimal path. // But does this answer all questions? Codegen ReprPrototypes currently still return the type-level node from their Build method! // This probably would functionally work -- we could make the Reify methods check for either the type-level or repr-level types -- but would it be weird/surprising/hard-to-use? ================================================ FILE: adl/rot13adl/rot13reification.go ================================================ package rot13adl import ( "fmt" "github.com/ipld/go-ipld-prime/datamodel" ) // Reify examines data in a Node to see if it matches the shape for valid substrate data for this ADL, // and if so, synthesizes and returns the high-level view of the ADL. // If it succeeds in recognizing the raw data as this ADL, // Reify returns a new Node which exhibits the logical behaviors of the ADL; // otherwise, it returns an error. // // The input data can be any implementation of ipld.Node; // it will be considered purely through that interface. // // If your application is expecting ADL data, this pipeline can be optimized // by using the SubstratePrototype right from the start when unmarshalling; // then, Reify can detect if the rawRoot parameter is of that implementation, // and it can save some processing work internally that can be known to already be done. // // Reification will generally operate on the data in a single block // (e.g. this function will not do any additional block loads and unmarshalling). // This is important because some ADLs handle data so large that loading it all // eagerly would be impractical (and in some cases outright impossible). // However, it also necessarily implies that invalid data may lie beyond // one of those lazy loads, and it won't be discovered at the time of Reify. // // In this demo ADL, we don't have multi-block content at all, // so of course we don't have any additional block loads! // However, ADL implementations may vary in their approaches to lazy vs eager loading. // All ADLs should document their exact semantics regarding this -- // especially if it has any implications for boundaries of data validity checking. // // REVIEW: this function is currently not conforming to any particular interface; // if we evolve the contract for ADLs to include an interface for reficiation functions, // might we need to add context and link loader systems as parameters to it? // Not all implementations might need it, as per previous paragraph; but some might. // Reification for multiblock ADLs might also need link loader systems as a parameter here // so they can capture them as config and hold them for use in future operations that do lazy loading. func Reify(maybeSubstrateRoot datamodel.Node) (datamodel.Node, error) { // Reify is often very easy to implement, // especially if you have an IPLD Schema that specifies the shape of the substrate data: // We can just check if the data in maybeSubstrateRoot happens to already be exactly the right type, // and if so, take very direct shortcuts because we already know its been validated in shape; // otherwise, we create a new piece of memory for our native substrate memory layout, // and assign into it from the raw node, validating in the process, // which again just leans directly on the shape validation logic already given to us by the schema logic on that type. // (Checking the concrete type of maybeSubstrateRoot in search of a shortcut is seemingly a tad redundant, // because the AssignNode path later also has such a check! // However, doing it earlier allows us to avoid an allocation; // the AssignNode path doesn't become available until after NewBuilder is invoked, and NewBuilder is where allocations happen.) // Check if we can recognize the maybeSubstrateRoot as being our own substrate types; // if it is, we can shortcut pretty drastically. if x, ok := maybeSubstrateRoot.(*_Substrate); ok { // In this ADL implementation, the high level node has the exact same memory layout as the substrate root, // and so our only remaining processing here is just to cast them, so that // the node we return has the correct methodset exposed. return (*_R13String)(x), nil } // Shortcut didn't work. Process via the data model. // The AssignNode method on the substrate type already contains all the logic necessary for this, so we use that. nb := Prototype.SubstrateRoot.NewBuilder() if err := nb.AssignNode(maybeSubstrateRoot); err != nil { return nil, fmt.Errorf("rot13adl.Reify failed: data does not match expected shape for substrate: %w", err) } return (*_R13String)(nb.Build().(*_Substrate)), nil } ================================================ FILE: adl/rot13adl/rot13substrate.go ================================================ package rot13adl import ( "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/node/mixins" "github.com/ipld/go-ipld-prime/schema" ) // Substrate returns the root node of the raw internal data form of the ADL's content. func (n *_R13String) Substrate() datamodel.Node { // This is a very minor twist in the case of the rot13 ADL. // However, for larger ADLs (especially those relating to multi-block collections), // this could be quite a bit more involved, and would almost certainly be the root node of a larger tree. return (*_Substrate)(n) } // -- Node --> var _ datamodel.Node = (*_Substrate)(nil) // Somewhat unusually for an ADL, there's only one substrate node type, // and we actually made it have the same in-memory structure as the synthesized view node. // // When implementing other more complex ADLs, it will probably be more common to have // the synthesized high-level node type either embed or have a pointer to the substrate root node. type _Substrate _R13String // REVIEW: what on earth we think the "TypeName" strings in error messages and other references to this node should be. // At the moment, it shares a prefix with the synthesized node, which is potentially confusing (?), // and I'm not sure what, if any, suffix actually makes meaningful sense to a user either. // I added the segment ".internal." to the middle of the name mangle; does this seem helpful? func (*_Substrate) Kind() datamodel.Kind { return datamodel.Kind_String } func (*_Substrate) LookupByString(string) (datamodel.Node, error) { return mixins.String{TypeName: "rot13adl.internal.Substrate"}.LookupByString("") } func (*_Substrate) LookupByNode(datamodel.Node) (datamodel.Node, error) { return mixins.String{TypeName: "rot13adl.internal.Substrate"}.LookupByNode(nil) } func (*_Substrate) LookupByIndex(idx int64) (datamodel.Node, error) { return mixins.String{TypeName: "rot13adl.internal.Substrate"}.LookupByIndex(0) } func (*_Substrate) LookupBySegment(seg datamodel.PathSegment) (datamodel.Node, error) { return mixins.String{TypeName: "rot13adl.internal.Substrate"}.LookupBySegment(seg) } func (*_Substrate) MapIterator() datamodel.MapIterator { return nil } func (*_Substrate) ListIterator() datamodel.ListIterator { return nil } func (*_Substrate) Length() int64 { return -1 } func (*_Substrate) IsAbsent() bool { return false } func (*_Substrate) IsNull() bool { return false } func (*_Substrate) AsBool() (bool, error) { return mixins.String{TypeName: "rot13adl.internal.Substrate"}.AsBool() } func (*_Substrate) AsInt() (int64, error) { return mixins.String{TypeName: "rot13adl.internal.Substrate"}.AsInt() } func (*_Substrate) AsFloat() (float64, error) { return mixins.String{TypeName: "rot13adl.internal.Substrate"}.AsFloat() } func (n *_Substrate) AsString() (string, error) { return n.raw, nil } func (*_Substrate) AsBytes() ([]byte, error) { return mixins.String{TypeName: "rot13adl.internal.Substrate"}.AsBytes() } func (*_Substrate) AsLink() (datamodel.Link, error) { return mixins.String{TypeName: "rot13adl.internal.Substrate"}.AsLink() } func (*_Substrate) Prototype() datamodel.NodePrototype { return _Substrate__Prototype{} } // -- NodePrototype --> var _ datamodel.NodePrototype = _Substrate__Prototype{} type _Substrate__Prototype struct { // There's no configuration to this ADL. } func (np _Substrate__Prototype) NewBuilder() datamodel.NodeBuilder { return &_Substrate__Builder{} } // -- NodeBuilder --> var _ datamodel.NodeBuilder = (*_Substrate__Builder)(nil) type _Substrate__Builder struct { _Substrate__Assembler } func (nb *_Substrate__Builder) Build() datamodel.Node { if nb.m != schema.Maybe_Value { panic("invalid state: cannot call Build on an assembler that's not finished") } return nb.w } func (nb *_Substrate__Builder) Reset() { *nb = _Substrate__Builder{} } // -- NodeAssembler --> var _ datamodel.NodeAssembler = (*_Substrate__Assembler)(nil) type _Substrate__Assembler struct { w *_Substrate m schema.Maybe // REVIEW: if the package where this Maybe enum lives is maybe not the right home for it after all. Or should this line use something different? We're only using some of its values after all. } func (_Substrate__Assembler) BeginMap(sizeHint int64) (datamodel.MapAssembler, error) { return mixins.StringAssembler{TypeName: "rot13adl.internal.Substrate"}.BeginMap(0) } func (_Substrate__Assembler) BeginList(sizeHint int64) (datamodel.ListAssembler, error) { return mixins.StringAssembler{TypeName: "rot13adl.internal.Substrate"}.BeginList(0) } func (na *_Substrate__Assembler) AssignNull() error { // REVIEW: unclear how this might compose with some other context (like a schema) which does allow nulls. Probably a wrapper type? return mixins.StringAssembler{TypeName: "rot13adl.internal.Substrate"}.AssignNull() } func (_Substrate__Assembler) AssignBool(bool) error { return mixins.StringAssembler{TypeName: "rot13adl.internal.Substrate"}.AssignBool(false) } func (_Substrate__Assembler) AssignInt(int64) error { return mixins.StringAssembler{TypeName: "rot13adl.internal.Substrate"}.AssignInt(0) } func (_Substrate__Assembler) AssignFloat(float64) error { return mixins.StringAssembler{TypeName: "rot13adl.internal.Substrate"}.AssignFloat(0) } func (na *_Substrate__Assembler) AssignString(v string) error { switch na.m { case schema.Maybe_Value: panic("invalid state: cannot assign into assembler that's already finished") } na.w = &_Substrate{ raw: v, synthesized: unrotate(v), } na.m = schema.Maybe_Value return nil } func (_Substrate__Assembler) AssignBytes([]byte) error { return mixins.StringAssembler{TypeName: "rot13adl.internal.Substrate"}.AssignBytes(nil) } func (_Substrate__Assembler) AssignLink(datamodel.Link) error { return mixins.StringAssembler{TypeName: "rot13adl.internal.Substrate"}.AssignLink(nil) } func (na *_Substrate__Assembler) AssignNode(v datamodel.Node) error { if v.IsNull() { return na.AssignNull() } if v2, ok := v.(*_Substrate); ok { switch na.m { case schema.Maybe_Value: panic("invalid state: cannot assign into assembler that's already finished") } na.w = v2 na.m = schema.Maybe_Value return nil } if v2, err := v.AsString(); err != nil { return err } else { return na.AssignString(v2) } } func (_Substrate__Assembler) Prototype() datamodel.NodePrototype { return _Substrate__Prototype{} } ================================================ FILE: adl.go ================================================ package ipld import ( "github.com/ipld/go-ipld-prime/adl" ) type ADL = adl.ADL ================================================ FILE: codec/README.md ================================================ Codecs ====== The `go-ipld-prime/codec` package is a grouping package. The subpackages contains some codecs which reside in this repo. The codecs included here are our "batteries included" codecs, but they are not otherwise special. It is not necessary for a codec to be a subpackage here to be a valid codec to use with go-ipld; anything that implements the `codec.Encoder` and `codec.Decoder` interfaces is fine. Terminology ----------- We generally refer to "codecs" as having an "encode" function and "decode" function. We consider "encoding" to be the process of going from {Data Model} to {serial data}, and "decoding" to be the process of going from {serial data} to {Data Model}. ### Codec vs Multicodec A "codec" is _any_ function that goes from {Data Model} to {serial data}, or vice versa. A "multicodec" is a function which does that and is _also_ specifically recognized and described in the tables in https://github.com/multiformats/multicodec/ . Multicodecs generally leave no further room for customization and configuration, because their entire behavior is supposed to be specified by a multicodec indicator code number. Our codecs, in the child packages of this one, usually offer configuration options. They also usually offer exactly one function, which does *not* allow configuration, which is supplying a multicodec-compatible behavior. You'll see this marked in the docs on those functions. ### Marshal vs Encode It's common to see the terms "marshal" and "unmarshal" used in golang. Those terms are usually describing when structured data is transformed into linearized, tokenized data (and then, perhaps, all the way to serially encoded data), or vice versa. We would use the words the same way... except we don't end up using them, because that feature doesn't really come up in our codec layer. In IPLD, we would describe mapping some typed data into Data Model as "marshalling". (It's one step shy of tokenizing, but barely: Data Model does already have defined ordering for every element of data.) And we do have systems that do this: `bindnode` and our codegen systems both do this, implicitly, when they give you an `ipld.Node` of the representation of some data. We just don't end up talking about it as "marshalling" because of how it's done implicitly by those systems. As a result, all of our features relating to codecs only end up speaking about "encoding" and "decoding". ### Legacy code There are some appearances of the words "marshal" and "unmarshal" in some of our subpackages here. That verbiage is generally on the way out. For functions and structures with those names, you'll notice their docs marking them as deprecated. Why have "batteries-included" codecs? ------------------------------------- These codecs live in this repo because they're commonly used, highly supported, and general-purpose codecs that we recommend for widespread usage in new developments. Also, it's just plain nice to have something in-repo for development purposes. It makes sure that if we try to make any API changes, we immediately see if they'd make codecs harder to implement. We also use the batteries-included codecs for debugging, for test fixtures, and for benchmarking. Further yet, the batteries-included codecs let us offer getting-started APIs. For example, we offer some helper APIs which use codecs like e.g. JSON to give consumers of the libraries one-step helper methods that "do the right thing" with zero config... so long as they happen to use that codec. Even for consumers who don't use those codecs, such functions then serve as natural documentation and examples for what to do to put their codec of choice to work. ================================================ FILE: codec/api.go ================================================ package codec import ( "io" "github.com/ipld/go-ipld-prime/datamodel" ) // The following two types define the two directions of transform that a codec can be expected to perform: // from Node to serial stream (aka "encoding", sometimes also described as "marshalling"), // and from serial stream to Node (via a NodeAssembler) (aka "decoding", sometimes also described as "unmarshalling"). // // You'll find a couple of implementations matching this shape in subpackages of 'codec'. // (These are the handful of encoders and decoders we ship as "batteries included".) // Other encoder and decoder implementations can be found in other repositories/modules. // It should also be easy to implement encodecs and decoders of your own! // // Encoder and Decoder functions can be used on their own, but are also often used via the `ipld/linking.LinkSystem` construction, // which handles all the other related operations necessary for a content-addressed storage system at once. // // Encoder and Decoder functions can be registered in the multicodec table in the `ipld/multicodec` package // if they're providing functionality that matches the expectations for a multicodec identifier. // This table will be used by some common EncoderChooser and DecoderChooser implementations // (namely, the ones in LinkSystems produced by the `linking/cid` package). // It's not strictly necessary to register functions there, though; you can also just use them directly. // // There are furthermore several conventions that codec packages are recommended to follow, but are only conventions: // // Most codec packages should have a ReusableEncoder and ResuableDecoder type, // which contain any working memory needed by the implementation, as well as any configuration options, // and those types should have an Encode and Decode function respectively which match these function types. // They may alternatively have EncoderConfig and DecoderConfig types, which have similar purpose, // but aren't promising memory reuse if kept around. // // By convention, a codec package that expects to fulfill a multicodec contract will also have // a package-scope exported function called Encode or Decode which also matches this interface, // and is the equivalent of creating a zero-value ReusableEncoder or ReusableDecoder (aka, default config) // and using its Encode or Decode methods. // This package-scope function may also internally use a sync.Pool // to keep some ReusableEncoder values on hand to avoid unnecessary allocations. // // Note that an EncoderConfig or DecoderConfig type that supports configuration options // does not functionally expose those options when invoked by the multicodec system -- // multicodec indicator codes do not provide room for extended configuration info. // Codecs that expose configuration options are doing so for library users to enjoy; // it does not mean those non-default configurations will necessarily be available // in all scenarios that use codecs indirectly. // There is also no standard interface for such configurations: by nature, // if they exist at all, they tend to vary per codec. type ( // Encoder defines the shape of a function which traverses a Node tree // and emits its data in a serialized form into an io.Writer. // // The dual of Encoder is a Decoder, which takes a NodeAssembler // and fills it with deserialized data consumed from an io.Reader. // Typically, Decoder and Encoder functions will be found in pairs, // and will be expected to be able to round-trip each other's data. // // Encoder functions can be used directly. // Encoder functions are also often used via a LinkSystem when working with content-addressed storage. // LinkSystem methods will helpfully handle the entire process of traversing a Node tree, // encoding this data, hashing it, streaming it to the writer, and committing it -- all as one step. // // An Encoder works with Nodes. // If you have a native golang structure, and want to serialize it using an Encoder, // you'll need to figure out how to transform that golang structure into an ipld.Node tree first. // // It may be useful to understand "multicodecs" when working with Encoders. // In IPLD, a system called "multicodecs" is typically used to describe encoding foramts. // A "multicodec indicator" is a number which describes an encoding; // the Link implementations used in IPLD (CIDs) store a multicodec indicator in the Link; // and in this library, a multicodec registry exists in the `codec` package, // and can be used to associate a multicodec indicator number with an Encoder function. // The default EncoderChooser in a LinkSystem will use this multicodec registry to select Encoder functions. // However, you can construct a LinkSystem that uses any EncoderChooser you want. // It is also possible to have and use Encoder functions that aren't registered as a multicodec at all... // we just recommend being cautious of this, because it may make your data less recognizable // when working with other systems that use multicodec indicators as part of their communication. Encoder func(datamodel.Node, io.Writer) error // Decoder defines the shape of a function which produces a Node tree // by reading serialized data from an io.Reader. // (Decoder doesn't itself return a Node directly, but rather takes a NodeAssembler as an argument, // because this allows the caller more control over the Node implementation, // as well as some control over allocations.) // // The dual of Decoder is an Encoder, which takes a Node and // emits its data in a serialized form into an io.Writer. // Typically, Decoder and Encoder functions will be found in pairs, // and will be expected to be able to round-trip each other's data. // // Decoder functions can be used directly. // Decoder functions are also often used via a LinkSystem when working with content-addressed storage. // LinkSystem methods will helpfully handle the entire process of opening block readers, // verifying the hash of the data stream, and applying a Decoder to build Nodes -- all as one step. // // A Decoder works with Nodes. // If you have a native golang structure, and want to populate it with data using a Decoder, // you'll need to either get a NodeAssembler which proxies data into that structure directly, // or assemble a Node as intermediate storage and copy the data to the native structure as a separate step. // // It may be useful to understand "multicodecs" when working with Decoders. // See the documentation on the Encoder function interface for more discussion of multicodecs, // the multicodec table, and how this is typically connected to linking. Decoder func(datamodel.NodeAssembler, io.Reader) error ) // ------------------- // Errors // type ErrBudgetExhausted struct{} func (e ErrBudgetExhausted) Error() string { return "decoder resource budget exhausted (message too long or too complex)" } // --------------------- // Other valuable and reused constants // type MapSortMode uint8 const ( MapSortMode_None MapSortMode = iota MapSortMode_Lexical MapSortMode_RFC7049 ) ================================================ FILE: codec/cbor/multicodec.go ================================================ package cbor import ( "io" "github.com/ipld/go-ipld-prime/codec" "github.com/ipld/go-ipld-prime/codec/dagcbor" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/multicodec" ) var ( _ codec.Decoder = Decode _ codec.Encoder = Encode ) func init() { multicodec.RegisterEncoder(0x51, Encode) multicodec.RegisterDecoder(0x51, Decode) } // Decode deserializes data from the given io.Reader and feeds it into the given datamodel.NodeAssembler. // Decode fits the codec.Decoder function interface. // // This is the function that will be registered in the default multicodec registry during package init time. func Decode(na datamodel.NodeAssembler, r io.Reader) error { return dagcbor.DecodeOptions{ AllowLinks: false, }.Decode(na, r) } // Encode walks the given datamodel.Node and serializes it to the given io.Writer. // Encode fits the codec.Encoder function interface. // // This is the function that will be registered in the default multicodec registry during package init time. func Encode(n datamodel.Node, w io.Writer) error { return dagcbor.EncodeOptions{ AllowLinks: false, }.Encode(n, w) } ================================================ FILE: codec/cbor/roundtrip_test.go ================================================ package cbor import ( "bytes" "strings" "testing" qt "github.com/frankban/quicktest" "github.com/ipld/go-ipld-prime/fluent" "github.com/ipld/go-ipld-prime/node/basicnode" nodetests "github.com/ipld/go-ipld-prime/node/tests" ) var n = fluent.MustBuildMap(basicnode.Prototype.Map, 4, func(na fluent.MapAssembler) { na.AssembleEntry("plain").AssignString("olde string") na.AssembleEntry("map").CreateMap(2, func(na fluent.MapAssembler) { na.AssembleEntry("one").AssignInt(1) na.AssembleEntry("two").AssignInt(2) }) na.AssembleEntry("list").CreateList(2, func(na fluent.ListAssembler) { na.AssembleValue().AssignString("three") na.AssembleValue().AssignString("four") }) na.AssembleEntry("nested").CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry("deeper").CreateList(1, func(na fluent.ListAssembler) { na.AssembleValue().AssignString("things") }) }) }) var serial = "\xa4eplainkolde stringcmap\xa2cone\x01ctwo\x02dlist\x82ethreedfourfnested\xa1fdeeper\x81fthings" func TestRoundtrip(t *testing.T) { t.Run("encoding", func(t *testing.T) { var buf bytes.Buffer err := Encode(n, &buf) qt.Assert(t, err, qt.IsNil) qt.Check(t, buf.String(), qt.Equals, serial) }) t.Run("decoding", func(t *testing.T) { buf := strings.NewReader(serial) nb := basicnode.Prototype.Map.NewBuilder() err := Decode(nb, buf) qt.Assert(t, err, qt.IsNil) qt.Check(t, nb.Build(), nodetests.NodeContentEquals, n) }) } func TestRoundtripScalar(t *testing.T) { nb := basicnode.Prototype__String{}.NewBuilder() nb.AssignString("applesauce") simple := nb.Build() t.Run("encoding", func(t *testing.T) { var buf bytes.Buffer err := Encode(simple, &buf) qt.Assert(t, err, qt.IsNil) qt.Check(t, buf.String(), qt.Equals, `japplesauce`) }) t.Run("decoding", func(t *testing.T) { buf := strings.NewReader(`japplesauce`) nb := basicnode.Prototype__String{}.NewBuilder() err := Decode(nb, buf) qt.Assert(t, err, qt.IsNil) qt.Check(t, nb.Build(), nodetests.NodeContentEquals, simple) }) } ================================================ FILE: codec/dagcbor/common.go ================================================ package dagcbor const linkTag = 42 ================================================ FILE: codec/dagcbor/doc.go ================================================ /* The dagcbor package provides a DAG-CBOR codec implementation. The Encode and Decode functions match the codec.Encoder and codec.Decoder function interfaces, and can be registered with the go-ipld-prime/multicodec package for easy usage with systems such as CIDs. Importing this package will automatically have the side-effect of registering Encode and Decode with the go-ipld-prime/multicodec registry, associating them with the standard multicodec indicator numbers for DAG-CBOR. This implementation follows most of the rules of DAG-CBOR, namely: - by and large, it does emit and parse CBOR! - only explicit-length maps and lists will be emitted by Encode; - only tag 42 is accepted, and it must parse as a CID; - only 64 bit floats will be emitted by Encode. This implementation is also not strict about certain rules: - Encode is order-passthrough when emitting maps (it does not sort, nor abort in error if unsorted data is encountered). To emit sorted data, the node should be sorted before applying the Encode function. - Decode is order-passthrough when parsing maps (it does not sort, nor abort in error if unsorted data is encountered). To be strict about the ordering of data, additional validation must be applied to the result of the Decode function. - Decode will accept indeterminate length lists and maps without complaint. (These should not be allowed according to the DAG-CBOR spec, nor will the Encode function re-emit such values, so this behavior should almost certainly be seen as a bug.) - Decode does not consistently verify that ints and floats use the smallest representation possible (or, the 64-bit version, in the float case). (Only these numeric encodings should be allowed according to the DAG-CBOR spec, and the Encode function will not re-emit variations, so this behavior should almost certainly be seen as a bug.) A note for future contributors: some functions in this package expose references to packages from the refmt module, and/or use them internally. Please avoid adding new code which expands the visibility of these references. In future work, we'd like to reduce or break this relationship entirely (in part, to reduce dependency sprawl, and in part because several of the imprecisions noted above stem from that library's lack of strictness). */ package dagcbor ================================================ FILE: codec/dagcbor/marshal.go ================================================ package dagcbor import ( "fmt" "io" "sort" "github.com/polydawn/refmt/cbor" "github.com/polydawn/refmt/shared" "github.com/polydawn/refmt/tok" "github.com/ipld/go-ipld-prime/codec" "github.com/ipld/go-ipld-prime/datamodel" cidlink "github.com/ipld/go-ipld-prime/linking/cid" ) // This file should be identical to the general feature in the parent package, // except for the `case datamodel.Kind_Link` block, // which is dag-cbor's special sauce for schemafree links. // EncodeOptions can be used to customize the behavior of an encoding function. // The Encode method on this struct fits the codec.Encoder function interface. type EncodeOptions struct { // If true, allow encoding of Link nodes as CBOR tag(42); // otherwise, reject them as unencodable. AllowLinks bool // Control the sorting of map keys, using one of the `codec.MapSortMode_*` constants. MapSortMode codec.MapSortMode } // Encode walks the given datamodel.Node and serializes it to the given io.Writer. // Encode fits the codec.Encoder function interface. // // The behavior of the encoder can be customized by setting fields in the EncodeOptions struct before calling this method. func (cfg EncodeOptions) Encode(n datamodel.Node, w io.Writer) error { // Probe for a builtin fast path. Shortcut to that if possible. type detectFastPath interface { EncodeDagCbor(io.Writer) error } if n2, ok := n.(detectFastPath); ok { return n2.EncodeDagCbor(w) } // Okay, generic inspection path. return Marshal(n, cbor.NewEncoder(w), cfg) } // Future work: we would like to remove the Marshal function, // and in particular, stop seeing types from refmt (like shared.TokenSink) be visible. // Right now, some kinds of configuration (e.g. for whitespace and prettyprint) are only available through interacting with the refmt types; // we should improve our API so that this can be done with only our own types in this package. // Marshal is a deprecated function. // Please consider switching to EncodeOptions.Encode instead. func Marshal(n datamodel.Node, sink shared.TokenSink, options EncodeOptions) error { var tk tok.Token return marshal(n, &tk, sink, options) } func marshal(n datamodel.Node, tk *tok.Token, sink shared.TokenSink, options EncodeOptions) error { switch n.Kind() { case datamodel.Kind_Invalid: return fmt.Errorf("cannot traverse a node that is absent") case datamodel.Kind_Null: tk.Type = tok.TNull _, err := sink.Step(tk) return err case datamodel.Kind_Map: return marshalMap(n, tk, sink, options) case datamodel.Kind_List: // Emit start of list. tk.Type = tok.TArrOpen l := n.Length() tk.Length = int(l) // TODO: overflow check if _, err := sink.Step(tk); err != nil { return err } // Emit list contents (and recurse). for i := int64(0); i < l; i++ { v, err := n.LookupByIndex(i) if err != nil { return err } if err := marshal(v, tk, sink, options); err != nil { return err } } // Emit list close. tk.Type = tok.TArrClose _, err := sink.Step(tk) return err case datamodel.Kind_Bool: v, err := n.AsBool() if err != nil { return err } tk.Type = tok.TBool tk.Bool = v _, err = sink.Step(tk) return err case datamodel.Kind_Int: if uin, ok := n.(datamodel.UintNode); ok { v, err := uin.AsUint() if err != nil { return err } tk.Type = tok.TUint tk.Uint = v } else { v, err := n.AsInt() if err != nil { return err } tk.Type = tok.TInt tk.Int = v } _, err := sink.Step(tk) return err case datamodel.Kind_Float: v, err := n.AsFloat() if err != nil { return err } tk.Type = tok.TFloat64 tk.Float64 = v _, err = sink.Step(tk) return err case datamodel.Kind_String: v, err := n.AsString() if err != nil { return err } tk.Type = tok.TString tk.Str = v _, err = sink.Step(tk) return err case datamodel.Kind_Bytes: v, err := n.AsBytes() if err != nil { return err } tk.Type = tok.TBytes tk.Bytes = v _, err = sink.Step(tk) return err case datamodel.Kind_Link: if !options.AllowLinks { return fmt.Errorf("cannot Marshal ipld links to CBOR") } v, err := n.AsLink() if err != nil { return err } switch lnk := v.(type) { case cidlink.Link: if !lnk.Cid.Defined() { return fmt.Errorf("encoding undefined CIDs are not supported by this codec") } tk.Type = tok.TBytes tk.Bytes = append([]byte{0}, lnk.Bytes()...) tk.Tagged = true tk.Tag = linkTag _, err = sink.Step(tk) tk.Tagged = false return err default: return fmt.Errorf("schemafree link emission only supported by this codec for CID type links") } default: panic("unreachable") } } func marshalMap(n datamodel.Node, tk *tok.Token, sink shared.TokenSink, options EncodeOptions) error { // Emit start of map. tk.Type = tok.TMapOpen expectedLength := int(n.Length()) tk.Length = expectedLength // TODO: overflow check if _, err := sink.Step(tk); err != nil { return err } if options.MapSortMode != codec.MapSortMode_None { // Collect map entries, then sort by key type entry struct { key string value datamodel.Node } entries := []entry{} for itr := n.MapIterator(); !itr.Done(); { k, v, err := itr.Next() if err != nil { return err } keyStr, err := k.AsString() if err != nil { return err } entries = append(entries, entry{keyStr, v}) } if len(entries) != expectedLength { return fmt.Errorf("map Length() does not match number of MapIterator() entries") } // Apply the desired sort function. switch options.MapSortMode { case codec.MapSortMode_Lexical: sort.Slice(entries, func(i, j int) bool { return entries[i].key < entries[j].key }) case codec.MapSortMode_RFC7049: sort.Slice(entries, func(i, j int) bool { // RFC7049 style sort as per DAG-CBOR spec li, lj := len(entries[i].key), len(entries[j].key) if li == lj { return entries[i].key < entries[j].key } return li < lj }) } // Emit map contents (and recurse). for _, e := range entries { tk.Type = tok.TString tk.Str = e.key if _, err := sink.Step(tk); err != nil { return err } if err := marshal(e.value, tk, sink, options); err != nil { return err } } } else { // no sorting // Emit map contents (and recurse). var entryCount int for itr := n.MapIterator(); !itr.Done(); { k, v, err := itr.Next() if err != nil { return err } entryCount++ tk.Type = tok.TString tk.Str, err = k.AsString() if err != nil { return err } if _, err := sink.Step(tk); err != nil { return err } if err := marshal(v, tk, sink, options); err != nil { return err } } if entryCount != expectedLength { return fmt.Errorf("map Length() does not match number of MapIterator() entries") } } // Emit map close. tk.Type = tok.TMapClose _, err := sink.Step(tk) return err } // EncodedLength will calculate the length in bytes that the encoded form of the // provided Node will occupy. // // Note that this function requires a full walk of the Node's graph, which may // not necessarily be a trivial cost and will incur some allocations. Using this // method to calculate buffers to pre-allocate may not result in performance // gains, but rather incur an overall cost. Use with care. func EncodedLength(n datamodel.Node) (int64, error) { switch n.Kind() { case datamodel.Kind_Invalid: return 0, fmt.Errorf("cannot traverse a node that is absent") case datamodel.Kind_Null: return 1, nil // 0xf6 case datamodel.Kind_Map: length := uintLength(uint64(n.Length())) // length prefixed major 5 for itr := n.MapIterator(); !itr.Done(); { k, v, err := itr.Next() if err != nil { return 0, err } keyLength, err := EncodedLength(k) if err != nil { return 0, err } length += keyLength valueLength, err := EncodedLength(v) if err != nil { return 0, err } length += valueLength } return length, nil case datamodel.Kind_List: nl := n.Length() length := uintLength(uint64(nl)) // length prefixed major 4 for i := int64(0); i < nl; i++ { v, err := n.LookupByIndex(i) if err != nil { return 0, err } innerLength, err := EncodedLength(v) if err != nil { return 0, err } length += innerLength } return length, nil case datamodel.Kind_Bool: return 1, nil // 0xf4 or 0xf5 case datamodel.Kind_Int: v, err := n.AsInt() if err != nil { return 0, err } if v < 0 { v = -v - 1 // negint is stored as one less than actual } return uintLength(uint64(v)), nil // major 0 or 1, as small as possible case datamodel.Kind_Float: return 9, nil // always major 7 and 64-bit float case datamodel.Kind_String: v, err := n.AsString() if err != nil { return 0, err } return uintLength(uint64(len(v))) + int64(len(v)), nil // length prefixed major 3 case datamodel.Kind_Bytes: v, err := n.AsBytes() if err != nil { return 0, err } return uintLength(uint64(len(v))) + int64(len(v)), nil // length prefixed major 2 case datamodel.Kind_Link: v, err := n.AsLink() if err != nil { return 0, err } switch lnk := v.(type) { case cidlink.Link: length := int64(2) // tag,42: 0xd82a bl := int64(len(lnk.Bytes())) + 1 // additional 0x00 in front of the CID bytes length += uintLength(uint64(bl)) + bl // length prefixed major 2 return length, err default: return 0, fmt.Errorf("schemafree link emission only supported by this codec for CID type links") } default: panic("unreachable") } } // Calculate how many bytes an integer, and therefore also the leading bytes of // a length-prefixed token. CBOR will pack it up into the smallest possible // uint representation, even merging it with the major if it's <=23. type boundaryLength struct { upperBound uint64 length int64 } var lengthBoundaries = []boundaryLength{ {24, 1}, // packed major|minor {256, 2}, // major, 8-bit length {65536, 3}, // major, 16-bit length {4294967296, 5}, // major, 32-bit length {0, 9}, // major, 64-bit length } func uintLength(ii uint64) int64 { for _, lb := range lengthBoundaries { if ii < lb.upperBound { return lb.length } } // maximum number of bytes to pack this int // if this int is used as a length prefix for a map, list, string or bytes // then we likely have a very bad Node that shouldn't be encoded, but the // encoder may raise problems with that if the memory allocator doesn't first. return lengthBoundaries[len(lengthBoundaries)-1].length } ================================================ FILE: codec/dagcbor/marshal_test.go ================================================ package dagcbor import ( "bytes" "math/rand" "testing" "time" qt "github.com/frankban/quicktest" "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/fluent/qp" cidlink "github.com/ipld/go-ipld-prime/linking/cid" basicnode "github.com/ipld/go-ipld-prime/node/basic" "github.com/ipld/go-ipld-prime/testutil/garbage" ) func calculateActualLength(t *testing.T, n datamodel.Node) int64 { var buf bytes.Buffer err := Encode(n, &buf) qt.Assert(t, err, qt.IsNil) return int64(buf.Len()) } func verifyEstimatedSize(t *testing.T, n datamodel.Node) { estimatedLength, err := EncodedLength(n) qt.Assert(t, err, qt.IsNil) actualLength := calculateActualLength(t, n) qt.Assert(t, estimatedLength, qt.Equals, actualLength) } func TestEncodedLength(t *testing.T) { t.Run("int boundaries", func(t *testing.T) { for ii := 0; ii < 4; ii++ { verifyEstimatedSize(t, basicnode.NewInt(int64(lengthBoundaries[ii].upperBound))) verifyEstimatedSize(t, basicnode.NewInt(int64(lengthBoundaries[ii].upperBound)-1)) verifyEstimatedSize(t, basicnode.NewInt(int64(lengthBoundaries[ii].upperBound)+1)) verifyEstimatedSize(t, basicnode.NewInt(-1*int64(lengthBoundaries[ii].upperBound))) verifyEstimatedSize(t, basicnode.NewInt(-1*int64(lengthBoundaries[ii].upperBound)-1)) verifyEstimatedSize(t, basicnode.NewInt(-1*int64(lengthBoundaries[ii].upperBound)+1)) } }) t.Run("small garbage", func(t *testing.T) { seed := time.Now().Unix() t.Logf("randomness seed: %v\n", seed) rnd := rand.New(rand.NewSource(seed)) for i := 0; i < 1000; i++ { gbg := garbage.Generate(rnd, garbage.TargetBlockSize(1<<6)) verifyEstimatedSize(t, gbg) } }) t.Run("medium garbage", func(t *testing.T) { seed := time.Now().Unix() t.Logf("randomness seed: %v\n", seed) rnd := rand.New(rand.NewSource(seed)) for i := 0; i < 100; i++ { gbg := garbage.Generate(rnd, garbage.TargetBlockSize(1<<16)) verifyEstimatedSize(t, gbg) } }) t.Run("large garbage", func(t *testing.T) { seed := time.Now().Unix() t.Logf("randomness seed: %v\n", seed) rnd := rand.New(rand.NewSource(seed)) for i := 0; i < 10; i++ { gbg := garbage.Generate(rnd, garbage.TargetBlockSize(1<<20)) verifyEstimatedSize(t, gbg) } }) } func TestMarshalUndefCid(t *testing.T) { link, err := cid.Decode("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi") qt.Assert(t, err, qt.IsNil) node, err := qp.BuildMap(basicnode.Prototype.Any, -1, func(ma datamodel.MapAssembler) { qp.MapEntry(ma, "UndefCid", qp.Link(cidlink.Link{Cid: cid.Undef})) qp.MapEntry(ma, "DefCid", qp.Link(cidlink.Link{Cid: link})) }) qt.Assert(t, err, qt.IsNil) _, err = ipld.Encode(node, Encode) qt.Assert(t, err, qt.ErrorMatches, "encoding undefined CIDs are not supported by this codec") } ================================================ FILE: codec/dagcbor/multicodec.go ================================================ package dagcbor import ( "io" "github.com/ipld/go-ipld-prime/codec" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/multicodec" ) var ( _ codec.Decoder = Decode _ codec.Encoder = Encode ) func init() { multicodec.RegisterEncoder(0x71, Encode) multicodec.RegisterDecoder(0x71, Decode) } // Decode deserializes data from the given io.Reader and feeds it into the given datamodel.NodeAssembler. // Decode fits the codec.Decoder function interface. // // A similar function is available on DecodeOptions type if you would like to customize any of the decoding details. // This function uses the defaults for the dag-cbor codec // (meaning: links (indicated by tag 42) are decoded). // // This is the function that will be registered in the default multicodec registry during package init time. func Decode(na datamodel.NodeAssembler, r io.Reader) error { return DecodeOptions{ AllowLinks: true, }.Decode(na, r) } // Encode walks the given datamodel.Node and serializes it to the given io.Writer. // Encode fits the codec.Encoder function interface. // // A similar function is available on EncodeOptions type if you would like to customize any of the encoding details. // This function uses the defaults for the dag-cbor codec // (meaning: links are encoded, and map keys are sorted (with RFC7049 ordering!) during encode). // // This is the function that will be registered in the default multicodec registry during package init time. func Encode(n datamodel.Node, w io.Writer) error { return EncodeOptions{ AllowLinks: true, MapSortMode: codec.MapSortMode_RFC7049, }.Encode(n, w) } ================================================ FILE: codec/dagcbor/nongreedy_test.go ================================================ package dagcbor import ( "bytes" "encoding/hex" "testing" qt "github.com/frankban/quicktest" "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/fluent/qp" "github.com/ipld/go-ipld-prime/node/basicnode" ) func TestNonGreedy(t *testing.T) { // same as JSON version of this test: {"a": 1}{"b": 2} buf, err := hex.DecodeString("a1616101a1616202") qt.Assert(t, err, qt.IsNil) r := bytes.NewReader(buf) opts := DecodeOptions{ DontParseBeyondEnd: true, } // first object nb1 := basicnode.Prototype.Map.NewBuilder() err = opts.Decode(nb1, r) qt.Assert(t, err, qt.IsNil) expected, err := qp.BuildMap(basicnode.Prototype.Any, 1, func(ma datamodel.MapAssembler) { qp.MapEntry(ma, "a", qp.Int(1)) }) qt.Assert(t, err, qt.IsNil) qt.Assert(t, ipld.DeepEqual(nb1.Build(), expected), qt.IsTrue) // second object nb2 := basicnode.Prototype.Map.NewBuilder() err = opts.Decode(nb2, r) qt.Assert(t, err, qt.IsNil) expected, err = qp.BuildMap(basicnode.Prototype.Any, 1, func(ma datamodel.MapAssembler) { qp.MapEntry(ma, "b", qp.Int(2)) }) qt.Assert(t, err, qt.IsNil) qt.Assert(t, ipld.DeepEqual(nb2.Build(), expected), qt.IsTrue) } ================================================ FILE: codec/dagcbor/roundtripCidlink_test.go ================================================ package dagcbor import ( "bytes" "io" "testing" qt "github.com/frankban/quicktest" cid "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/linking" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipld/go-ipld-prime/node/basicnode" nodetests "github.com/ipld/go-ipld-prime/node/tests" ) func TestRoundtripCidlink(t *testing.T) { lp := cidlink.LinkPrototype{Prefix: cid.Prefix{ Version: 1, Codec: 0x71, MhType: 0x13, MhLength: 4, }} lsys := cidlink.DefaultLinkSystem() buf := bytes.Buffer{} lsys.StorageWriteOpener = func(lnkCtx linking.LinkContext) (io.Writer, linking.BlockWriteCommitter, error) { return &buf, func(lnk datamodel.Link) error { return nil }, nil } lsys.StorageReadOpener = func(lnkCtx linking.LinkContext, lnk datamodel.Link) (io.Reader, error) { return bytes.NewReader(buf.Bytes()), nil } lnk, err := lsys.Store(linking.LinkContext{}, lp, n) qt.Assert(t, err, qt.IsNil) n2, err := lsys.Load(linking.LinkContext{}, lnk, basicnode.Prototype.Any) qt.Assert(t, err, qt.IsNil) qt.Check(t, n2, nodetests.NodeContentEquals, nSorted) } ================================================ FILE: codec/dagcbor/roundtrip_test.go ================================================ package dagcbor import ( "bytes" "crypto/rand" "encoding/hex" "math" "strings" "testing" qt "github.com/frankban/quicktest" cid "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/fluent" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipld/go-ipld-prime/node/basicnode" nodetests "github.com/ipld/go-ipld-prime/node/tests" ) var n = fluent.MustBuildMap(basicnode.Prototype.Map, 4, func(na fluent.MapAssembler) { na.AssembleEntry("plain").AssignString("olde string") na.AssembleEntry("map").CreateMap(2, func(na fluent.MapAssembler) { na.AssembleEntry("one").AssignInt(1) na.AssembleEntry("two").AssignInt(2) }) na.AssembleEntry("list").CreateList(2, func(na fluent.ListAssembler) { na.AssembleValue().AssignString("three") na.AssembleValue().AssignString("four") }) na.AssembleEntry("nested").CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry("deeper").CreateList(1, func(na fluent.ListAssembler) { na.AssembleValue().AssignString("things") }) }) }) var nSorted = fluent.MustBuildMap(basicnode.Prototype.Map, 4, func(na fluent.MapAssembler) { na.AssembleEntry("map").CreateMap(2, func(na fluent.MapAssembler) { na.AssembleEntry("one").AssignInt(1) na.AssembleEntry("two").AssignInt(2) }) na.AssembleEntry("list").CreateList(2, func(na fluent.ListAssembler) { na.AssembleValue().AssignString("three") na.AssembleValue().AssignString("four") }) na.AssembleEntry("plain").AssignString("olde string") na.AssembleEntry("nested").CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry("deeper").CreateList(1, func(na fluent.ListAssembler) { na.AssembleValue().AssignString("things") }) }) }) var serial = "\xa4cmap\xa2cone\x01ctwo\x02dlist\x82ethreedfoureplainkolde stringfnested\xa1fdeeper\x81fthings" func TestRoundtrip(t *testing.T) { t.Run("encoding", func(t *testing.T) { var buf bytes.Buffer err := Encode(n, &buf) qt.Assert(t, err, qt.IsNil) qt.Check(t, buf.String(), qt.Equals, serial) }) t.Run("length", func(t *testing.T) { length, err := EncodedLength(n) qt.Assert(t, err, qt.IsNil) qt.Check(t, length, qt.Equals, int64(len(serial))) }) t.Run("decoding", func(t *testing.T) { buf := strings.NewReader(serial) nb := basicnode.Prototype.Map.NewBuilder() err := Decode(nb, buf) qt.Assert(t, err, qt.IsNil) qt.Check(t, nb.Build(), nodetests.NodeContentEquals, nSorted) }) } func TestRoundtripScalar(t *testing.T) { nb := basicnode.Prototype__String{}.NewBuilder() nb.AssignString("applesauce") simple := nb.Build() t.Run("encoding", func(t *testing.T) { var buf bytes.Buffer err := Encode(simple, &buf) qt.Assert(t, err, qt.IsNil) qt.Check(t, buf.String(), qt.Equals, `japplesauce`) }) t.Run("decoding", func(t *testing.T) { buf := strings.NewReader(`japplesauce`) nb := basicnode.Prototype__String{}.NewBuilder() err := Decode(nb, buf) qt.Assert(t, err, qt.IsNil) qt.Check(t, nb.Build(), nodetests.NodeContentEquals, simple) }) } func TestRoundtripLinksAndBytes(t *testing.T) { lnk := cidlink.LinkPrototype{Prefix: cid.Prefix{ Version: 1, Codec: 0x71, MhType: 0x13, MhLength: 4, }}.BuildLink([]byte{1, 2, 3, 4}) // dummy value, content does not matter to this test. var linkByteNode = fluent.MustBuildMap(basicnode.Prototype.Map, 4, func(na fluent.MapAssembler) { nva := na.AssembleEntry("Link") nva.AssignLink(lnk) nva = na.AssembleEntry("Bytes") bytes := make([]byte, 100) _, _ = rand.Read(bytes) nva.AssignBytes(bytes) }) buf := bytes.Buffer{} err := Encode(linkByteNode, &buf) qt.Assert(t, err, qt.IsNil) nb := basicnode.Prototype.Map.NewBuilder() err = Decode(nb, &buf) qt.Assert(t, err, qt.IsNil) reconstructed := nb.Build() qt.Check(t, reconstructed, nodetests.NodeContentEquals, linkByteNode) } func TestInts(t *testing.T) { data := []struct { name string hex string value uint64 intValue int64 intErr string decodeErr string }{ {"max uint64", "1bffffffffffffffff", math.MaxUint64, 0, "unsigned integer out of range of int64 type", ""}, {"max int64", "1b7fffffffffffffff", math.MaxInt64, math.MaxInt64, "", ""}, {"1", "01", 1, 1, "", ""}, {"0", "00", 0, 0, "", ""}, {"-1", "20", 0, -1, "", ""}, {"min int64", "3b7fffffffffffffff", 0, math.MinInt64, "", ""}, {"~min uint64", "3bfffffffffffffffe", 0, 0, "", "cbor: negative integer out of rage of int64 type"}, // TODO: 3bffffffffffffffff isn't properly handled by refmt, it's coerced to zero // MaxUint64 gets overflowed here: https://github.com/polydawn/refmt/blob/30ac6d18308e584ca6a2e74ba81475559db94c5f/cbor/cborDecoderTerminals.go#L75 } for _, td := range data { t.Run(td.name, func(t *testing.T) { buf, err := hex.DecodeString(td.hex) // max uint64 qt.Assert(t, err, qt.IsNil) nb := basicnode.Prototype.Any.NewBuilder() err = Decode(nb, bytes.NewReader(buf)) if td.decodeErr != "" { qt.Assert(t, err, qt.IsNotNil) qt.Assert(t, err.Error(), qt.Equals, td.decodeErr) return } qt.Assert(t, err, qt.IsNil) n := nb.Build() ii, err := n.AsInt() if td.intErr != "" { qt.Assert(t, err.Error(), qt.Equals, td.intErr) } else { qt.Assert(t, err, qt.IsNil) qt.Assert(t, ii, qt.Equals, int64(td.intValue)) } // if the number is outside of the positive int64 range, we should be able // to access it as a UintNode and be able to access the full int64 range uin, ok := n.(datamodel.UintNode) if td.value <= math.MaxInt64 { qt.Assert(t, ok, qt.IsFalse) } else { qt.Assert(t, ok, qt.IsTrue) val, err := uin.AsUint() qt.Assert(t, err, qt.IsNil) qt.Assert(t, val, qt.Equals, uint64(td.value)) } var byts bytes.Buffer err = Encode(n, &byts) qt.Assert(t, err, qt.IsNil) qt.Assert(t, hex.EncodeToString(byts.Bytes()), qt.Equals, td.hex) }) } } ================================================ FILE: codec/dagcbor/unmarshal.go ================================================ package dagcbor import ( "errors" "fmt" "io" "math" cid "github.com/ipfs/go-cid" "github.com/polydawn/refmt/cbor" "github.com/polydawn/refmt/shared" "github.com/polydawn/refmt/tok" "github.com/ipld/go-ipld-prime/datamodel" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipld/go-ipld-prime/node/basicnode" ) var ( ErrInvalidMultibase = errors.New("invalid multibase on IPLD link") ErrAllocationBudgetExceeded = errors.New("message structure demanded too many resources to process") ErrDecodeDepthExceeded = errors.New("message structure exceeded maximum nesting depth") ErrTrailingBytes = errors.New("unexpected content after end of cbor object") ) const ( mapEntryCost = 8 listEntryCost = 4 ) // This file should be identical to the general feature in the parent package, // except for the `case tok.TBytes` block, // which has dag-cbor's special sauce for detecting schemafree links. // DecodeOptions can be used to customize the behavior of a decoding function. // The Decode method on this struct fits the codec.Decoder function interface. type DecodeOptions struct { // If true, parse DAG-CBOR tag(42) as Link nodes, otherwise reject them AllowLinks bool // TODO: ExperimentalDeterminism enforces map key order, but not the other parts // of the spec such as integers or floats. See the fuzz failures spotted in // https://github.com/ipld/go-ipld-prime/pull/389. // When we're done implementing strictness, deprecate the option in favor of // StrictDeterminism, but keep accepting both for backwards compatibility. // ExperimentalDeterminism requires decoded DAG-CBOR bytes to be canonical as per // the spec. For example, this means that integers and floats be encoded in // a particular way, and map keys be sorted. // // The decoder does not enforce this requirement by default, as the codec // was originally implemented without these rules. Because of that, there's // a significant amount of published data that isn't canonical but should // still decode with the default settings for backwards compatibility. // // Note that this option is experimental as it only implements partial strictness. ExperimentalDeterminism bool // If true, the decoder stops reading from the stream at the end of a full, // valid CBOR object. This may be useful for parsing a stream of undelimited // CBOR objects. // As per standard IPLD behavior, in the default mode the parser considers the // entire block to be part of the CBOR object and will error if there is // extraneous data after the end of the object. DontParseBeyondEnd bool // AllocationBudget sets the maximum budget for the decoder. The budget is // decremented as the decoder allocates resources (nodes, map entries, list // elements, string/bytes content). If the budget is exhausted, the decoder // returns ErrAllocationBudgetExceeded. // // When zero, a default budget is used which is generous for typical IPLD // block sizes. AllocationBudget int64 // MaxCollectionPrealloc sets the maximum size hint passed to // BeginMap/BeginList. CBOR headers declare collection sizes upfront; // this caps the initial allocation while collections grow dynamically // as entries are decoded. // // When zero, a default of 1024 is used. MaxCollectionPrealloc int64 // MaxDepth sets the maximum nesting depth for decoded structures. If the // decoder encounters a map or list nested beyond this depth, it returns // ErrDecodeDepthExceeded. // // When zero, a default of 1024 is used. MaxDepth int64 } const ( defaultAllocationBudget int64 = 1048576 * 10 defaultMaxCollectionPrealloc int64 = 1024 defaultMaxDepth int64 = 1024 ) // Decode deserializes data from the given io.Reader and feeds it into the given datamodel.NodeAssembler. // Decode fits the codec.Decoder function interface. // // The behavior of the decoder can be customized by setting fields in the DecodeOptions struct before calling this method. func (cfg DecodeOptions) Decode(na datamodel.NodeAssembler, r io.Reader) error { // Probe for a builtin fast path. Shortcut to that if possible. // Note: when an assembler implements this interface, it receives only the // reader and none of the fields set on cfg. Implementations are responsible // for enforcing their own equivalents of AllocationBudget, // MaxCollectionPrealloc, MaxDepth and the other DecodeOptions fields. type detectFastPath interface { DecodeDagCbor(io.Reader) error } if na2, ok := na.(detectFastPath); ok { return na2.DecodeDagCbor(r) } // Okay, generic builder path. err := Unmarshal(na, cbor.NewDecoder(cbor.DecodeOptions{ CoerceUndefToNull: true, }, r), cfg) if err != nil { return err } if cfg.DontParseBeyondEnd { return nil } var buf [1]byte _, err = io.ReadFull(r, buf[:]) switch err { case io.EOF: return nil case nil: return ErrTrailingBytes default: return err } } // Future work: we would like to remove the Unmarshal function, // and in particular, stop seeing types from refmt (like shared.TokenSource) be visible. // Right now, some kinds of configuration (e.g. for whitespace and prettyprint) are only available through interacting with the refmt types; // we should improve our API so that this can be done with only our own types in this package. // Unmarshal is a deprecated function. // Please consider switching to DecodeOptions.Decode instead. func Unmarshal(na datamodel.NodeAssembler, tokSrc shared.TokenSource, options DecodeOptions) error { budget := options.AllocationBudget if budget == 0 { budget = defaultAllocationBudget } return unmarshal1(na, tokSrc, &budget, 0, options) } func (cfg DecodeOptions) maxPrealloc() int64 { if cfg.MaxCollectionPrealloc > 0 { return cfg.MaxCollectionPrealloc } return defaultMaxCollectionPrealloc } func (cfg DecodeOptions) maxDepth() int64 { if cfg.MaxDepth > 0 { return cfg.MaxDepth } return defaultMaxDepth } func unmarshal1(na datamodel.NodeAssembler, tokSrc shared.TokenSource, budget *int64, depth int64, options DecodeOptions) error { var tk tok.Token done, err := tokSrc.Step(&tk) if err == io.EOF { return io.ErrUnexpectedEOF } if err != nil { return err } if done && !tk.Type.IsValue() && tk.Type != tok.TNull { return fmt.Errorf("unexpected eof") } return unmarshal2(na, tokSrc, &tk, budget, depth, options) } // starts with the first token already primed. Necessary to get recursion // // to flow right without a peek+unpeek system. func unmarshal2(na datamodel.NodeAssembler, tokSrc shared.TokenSource, tk *tok.Token, budget *int64, depth int64, options DecodeOptions) error { // FUTURE: check for schema.TypedNodeBuilder that's going to parse a Link (they can slurp any token kind they want). switch tk.Type { case tok.TMapOpen: if depth >= options.maxDepth() { return ErrDecodeDepthExceeded } expectLen := int64(tk.Length) allocLen := int64(tk.Length) if tk.Length == -1 { expectLen = math.MaxInt64 allocLen = 0 } else { *budget -= allocLen if *budget < 0 { return ErrAllocationBudgetExceeded } if allocLen > options.maxPrealloc() { allocLen = options.maxPrealloc() } } ma, err := na.BeginMap(allocLen) if err != nil { return err } var observedLen int64 lastKey := "" for { _, err := tokSrc.Step(tk) if err != nil { return err } switch tk.Type { case tok.TMapClose: if expectLen != math.MaxInt64 && observedLen != expectLen { return fmt.Errorf("unexpected mapClose before declared length") } return ma.Finish() case tok.TString: *budget -= int64(len(tk.Str) + mapEntryCost) if *budget < 0 { return ErrAllocationBudgetExceeded } // continue default: return fmt.Errorf("unexpected %s token while expecting map key", tk.Type) } observedLen++ if observedLen > expectLen { return fmt.Errorf("unexpected continuation of map elements beyond declared length") } if observedLen > 1 && options.ExperimentalDeterminism { if len(lastKey) > len(tk.Str) || lastKey > tk.Str { return fmt.Errorf("map key %q is not after %q as per RFC7049", tk.Str, lastKey) } } lastKey = tk.Str mva, err := ma.AssembleEntry(tk.Str) if err != nil { // return in error if the key was rejected return err } err = unmarshal1(mva, tokSrc, budget, depth+1, options) if err != nil { // return in error if some part of the recursion errored return err } } case tok.TMapClose: return fmt.Errorf("unexpected mapClose token") case tok.TArrOpen: if depth >= options.maxDepth() { return ErrDecodeDepthExceeded } expectLen := int64(tk.Length) allocLen := int64(tk.Length) if tk.Length == -1 { expectLen = math.MaxInt64 allocLen = 0 } else { *budget -= allocLen if *budget < 0 { return ErrAllocationBudgetExceeded } if allocLen > options.maxPrealloc() { allocLen = options.maxPrealloc() } } la, err := na.BeginList(allocLen) if err != nil { return err } var observedLen int64 for { _, err := tokSrc.Step(tk) if err != nil { return err } switch tk.Type { case tok.TArrClose: if expectLen != math.MaxInt64 && observedLen != expectLen { return fmt.Errorf("unexpected arrClose before declared length") } return la.Finish() default: *budget -= listEntryCost if *budget < 0 { return ErrAllocationBudgetExceeded } observedLen++ if observedLen > expectLen { return fmt.Errorf("unexpected continuation of array elements beyond declared length") } err := unmarshal2(la.AssembleValue(), tokSrc, tk, budget, depth+1, options) if err != nil { // return in error if some part of the recursion errored return err } } } case tok.TArrClose: return fmt.Errorf("unexpected arrClose token") case tok.TNull: return na.AssignNull() case tok.TString: *budget -= int64(len(tk.Str)) if *budget < 0 { return ErrAllocationBudgetExceeded } return na.AssignString(tk.Str) case tok.TBytes: *budget -= int64(len(tk.Bytes)) if *budget < 0 { return ErrAllocationBudgetExceeded } if !tk.Tagged { return na.AssignBytes(tk.Bytes) } switch tk.Tag { case linkTag: if !options.AllowLinks { return fmt.Errorf("unhandled cbor tag %d", tk.Tag) } if len(tk.Bytes) < 1 || tk.Bytes[0] != 0 { return ErrInvalidMultibase } elCid, err := cid.Cast(tk.Bytes[1:]) if err != nil { return err } return na.AssignLink(cidlink.Link{Cid: elCid}) default: return fmt.Errorf("unhandled cbor tag %d", tk.Tag) } case tok.TBool: *budget -= 1 if *budget < 0 { return ErrAllocationBudgetExceeded } return na.AssignBool(tk.Bool) case tok.TInt: *budget -= 1 if *budget < 0 { return ErrAllocationBudgetExceeded } return na.AssignInt(tk.Int) case tok.TUint: *budget -= 1 if *budget < 0 { return ErrAllocationBudgetExceeded } // note that this pushes any overflow errors up the stack when AsInt() may // be called on a UintNode that is too large to cast to an int64 if tk.Uint > math.MaxInt64 { return na.AssignNode(basicnode.NewUint(tk.Uint)) } return na.AssignInt(int64(tk.Uint)) case tok.TFloat64: *budget -= 1 if *budget < 0 { return ErrAllocationBudgetExceeded } return na.AssignFloat(tk.Float64) default: panic("unreachable") } } ================================================ FILE: codec/dagcbor/unmarshal_test.go ================================================ package dagcbor import ( "bytes" "encoding/binary" "fmt" "runtime" "strings" "testing" "github.com/ipld/go-ipld-prime/datamodel" qt "github.com/frankban/quicktest" "github.com/ipld/go-ipld-prime/node/basicnode" ) func TestFunBlocks(t *testing.T) { t.Run("zero length link", func(t *testing.T) { // This fixture has a zero length link -- not even the multibase byte (which dag-cbor insists must be zero) is there. buf := strings.NewReader("\x8d\x8d\x97\xd8*@") nb := basicnode.Prototype.Any.NewBuilder() err := Decode(nb, buf) qt.Assert(t, err, qt.Equals, ErrInvalidMultibase) }) t.Run("fuzz001", func(t *testing.T) { // This fixture might cause an overly large allocation if you aren't careful to have resource budgets. buf := strings.NewReader("\x9a\xff000") nb := basicnode.Prototype.Any.NewBuilder() err := Decode(nb, buf) if runtime.GOARCH == "386" { // TODO: fix refmt to properly handle 64-bit ints on 32-bit runtime qt.Assert(t, err.Error(), qt.Equals, "cbor: positive integer is out of length") } else { qt.Assert(t, err, qt.Equals, ErrAllocationBudgetExceeded) } }) t.Run("fuzz002", func(t *testing.T) { // This fixture might cause an overly large allocation if you aren't careful to have resource budgets. buf := strings.NewReader("\x9f\x9f\x9f\x9f\x9f\x9f\x9f\x9f\x9f\x9f\x9f\x9f\x9f\x9f\x9f\x9f\x9f\x9f\x9f\x9f\x9a\xff000") nb := basicnode.Prototype.Any.NewBuilder() err := Decode(nb, buf) if runtime.GOARCH == "386" { // TODO: fix refmt to properly handle 64-bit ints on 32-bit qt.Assert(t, err.Error(), qt.Equals, "cbor: positive integer is out of length") } else { qt.Assert(t, err, qt.Equals, ErrAllocationBudgetExceeded) } }) t.Run("fuzz003", func(t *testing.T) { // This fixture might cause an overly large allocation if you aren't careful to have resource budgets. buf := strings.NewReader("\x9f\x9f\x9f\x9f\x9f\x9f\x9f\xbb00000000") nb := basicnode.Prototype.Any.NewBuilder() err := Decode(nb, buf) if runtime.GOARCH == "386" { // TODO: fix refmt to properly handle 64-bit ints on 32-bit qt.Assert(t, err.Error(), qt.Equals, "cbor: positive integer is out of length") } else { qt.Assert(t, err, qt.Equals, ErrAllocationBudgetExceeded) } }) t.Run("undef", func(t *testing.T) { // This fixture tests that we tolerate cbor's "undefined" token (even though it's noncanonical and you shouldn't use it), // and that it becomes a null in the data model level. buf := strings.NewReader("\xf7") nb := basicnode.Prototype.Any.NewBuilder() err := Decode(nb, buf) qt.Assert(t, err, qt.IsNil) qt.Assert(t, nb.Build().Kind(), qt.Equals, datamodel.Kind_Null) }) t.Run("extra bytes", func(t *testing.T) { buf := strings.NewReader("\xa0\x00") nb := basicnode.Prototype.Any.NewBuilder() err := Decode(nb, buf) qt.Assert(t, err, qt.Equals, ErrTrailingBytes) }) } func cborMapHeader(length uint32) []byte { var buf bytes.Buffer buf.WriteByte(0xBA) binary.Write(&buf, binary.BigEndian, length) return buf.Bytes() } func cborArrayHeader(length uint32) []byte { var buf bytes.Buffer buf.WriteByte(0x9A) binary.Write(&buf, binary.BigEndian, length) return buf.Bytes() } // TestDecodeOptions_AllocationBudget verifies that the configurable allocation // budget is respected, both with defaults and custom values. func TestDecodeOptions_AllocationBudget(t *testing.T) { t.Run("default budget rejects oversized structure", func(t *testing.T) { // A map header claiming more entries than the default budget allows payload := cborMapHeader(20_000_000) nb := basicnode.Prototype.Any.NewBuilder() err := Decode(nb, bytes.NewReader(payload)) qt.Assert(t, err, qt.Equals, ErrAllocationBudgetExceeded) }) t.Run("custom budget accepts within limit", func(t *testing.T) { // Build a small valid map: {"a": 1} var buf bytes.Buffer buf.Write(cborMapHeader(1)) buf.WriteByte(0x61) // text(1) buf.WriteByte('a') buf.WriteByte(0x01) // uint(1) nb := basicnode.Prototype.Any.NewBuilder() err := DecodeOptions{AllowLinks: true, AllocationBudget: 100}.Decode(nb, &buf) qt.Assert(t, err, qt.IsNil) node := nb.Build() qt.Assert(t, node.Kind(), qt.Equals, datamodel.Kind_Map) }) t.Run("custom budget rejects when exhausted", func(t *testing.T) { // A map claiming 50 entries with a budget of only 10 should be rejected payload := cborMapHeader(50) nb := basicnode.Prototype.Any.NewBuilder() err := DecodeOptions{AllowLinks: true, AllocationBudget: 10}.Decode(nb, bytes.NewReader(payload)) qt.Assert(t, err, qt.Equals, ErrAllocationBudgetExceeded) }) t.Run("budget accounts for declared collection sizes", func(t *testing.T) { // A list claiming 1000 entries consumes budget even if no entries follow payload := cborArrayHeader(1000) nb := basicnode.Prototype.Any.NewBuilder() err := DecodeOptions{AllowLinks: true, AllocationBudget: 500}.Decode(nb, bytes.NewReader(payload)) qt.Assert(t, err, qt.Equals, ErrAllocationBudgetExceeded) }) } // TestDecodeOptions_MaxCollectionPrealloc verifies that the preallocation cap // is respected and that large collections still decode correctly. func TestDecodeOptions_MaxCollectionPrealloc(t *testing.T) { t.Run("large map decodes correctly with default cap", func(t *testing.T) { const numEntries = 5_000 var buf bytes.Buffer buf.Write(cborMapHeader(numEntries)) for i := 0; i < numEntries; i++ { key := fmt.Sprintf("k%05d", i) buf.WriteByte(0x66) // text(6) buf.WriteString(key) if i < 24 { buf.WriteByte(byte(i)) } else if i < 256 { buf.WriteByte(0x18) buf.WriteByte(byte(i)) } else { buf.WriteByte(0x19) binary.Write(&buf, binary.BigEndian, uint16(i)) } } nb := basicnode.Prototype.Any.NewBuilder() err := Decode(nb, &buf) qt.Assert(t, err, qt.IsNil) node := nb.Build() qt.Assert(t, node.Kind(), qt.Equals, datamodel.Kind_Map) qt.Assert(t, node.Length(), qt.Equals, int64(numEntries)) v, err := node.LookupByString("k00000") qt.Assert(t, err, qt.IsNil) vi, err := v.AsInt() qt.Assert(t, err, qt.IsNil) qt.Assert(t, vi, qt.Equals, int64(0)) v, err = node.LookupByString("k04999") qt.Assert(t, err, qt.IsNil) vi, err = v.AsInt() qt.Assert(t, err, qt.IsNil) qt.Assert(t, vi, qt.Equals, int64(4999)) }) t.Run("large list decodes correctly with default cap", func(t *testing.T) { const numEntries = 5_000 var buf bytes.Buffer buf.Write(cborArrayHeader(numEntries)) for i := 0; i < numEntries; i++ { if i < 24 { buf.WriteByte(byte(i)) } else if i < 256 { buf.WriteByte(0x18) buf.WriteByte(byte(i)) } else { buf.WriteByte(0x19) binary.Write(&buf, binary.BigEndian, uint16(i)) } } nb := basicnode.Prototype.Any.NewBuilder() err := Decode(nb, &buf) qt.Assert(t, err, qt.IsNil) node := nb.Build() qt.Assert(t, node.Kind(), qt.Equals, datamodel.Kind_List) qt.Assert(t, node.Length(), qt.Equals, int64(numEntries)) v, err := node.LookupByIndex(0) qt.Assert(t, err, qt.IsNil) vi, err := v.AsInt() qt.Assert(t, err, qt.IsNil) qt.Assert(t, vi, qt.Equals, int64(0)) v, err = node.LookupByIndex(numEntries - 1) qt.Assert(t, err, qt.IsNil) vi, err = v.AsInt() qt.Assert(t, err, qt.IsNil) qt.Assert(t, vi, qt.Equals, int64(numEntries-1)) }) t.Run("custom prealloc cap with valid data", func(t *testing.T) { // 100-entry list with a prealloc cap of 10 should still decode fine const numEntries = 100 var buf bytes.Buffer buf.Write(cborArrayHeader(numEntries)) for i := 0; i < numEntries; i++ { buf.WriteByte(byte(i % 24)) } nb := basicnode.Prototype.Any.NewBuilder() err := DecodeOptions{AllowLinks: true, MaxCollectionPrealloc: 10}.Decode(nb, &buf) qt.Assert(t, err, qt.IsNil) node := nb.Build() qt.Assert(t, node.Length(), qt.Equals, int64(numEntries)) }) } // TestDecodeOptions_MaxDepth verifies that the configurable nesting-depth // limit is respected, both with defaults and custom values. func TestDecodeOptions_MaxDepth(t *testing.T) { // buildNestedArrays returns depth `0x81` bytes (array(1)) followed by a // single `0xF6` null, forming `depth` levels of nested single-element // arrays. buildNestedArrays := func(depth int) []byte { buf := make([]byte, 0, depth+1) for i := 0; i < depth; i++ { buf = append(buf, 0x81) } buf = append(buf, 0xF6) return buf } t.Run("default depth rejects deeply nested structure", func(t *testing.T) { payload := buildNestedArrays(2000) nb := basicnode.Prototype.Any.NewBuilder() err := Decode(nb, bytes.NewReader(payload)) qt.Assert(t, err, qt.Equals, ErrDecodeDepthExceeded) }) t.Run("structure at default depth decodes", func(t *testing.T) { payload := buildNestedArrays(1024) nb := basicnode.Prototype.Any.NewBuilder() err := Decode(nb, bytes.NewReader(payload)) qt.Assert(t, err, qt.IsNil) qt.Assert(t, nb.Build().Kind(), qt.Equals, datamodel.Kind_List) }) t.Run("custom depth rejects when exceeded", func(t *testing.T) { payload := buildNestedArrays(10) nb := basicnode.Prototype.Any.NewBuilder() err := DecodeOptions{MaxDepth: 5}.Decode(nb, bytes.NewReader(payload)) qt.Assert(t, err, qt.Equals, ErrDecodeDepthExceeded) }) t.Run("custom depth accepts within limit", func(t *testing.T) { payload := buildNestedArrays(5) nb := basicnode.Prototype.Any.NewBuilder() err := DecodeOptions{MaxDepth: 10}.Decode(nb, bytes.NewReader(payload)) qt.Assert(t, err, qt.IsNil) qt.Assert(t, nb.Build().Kind(), qt.Equals, datamodel.Kind_List) }) t.Run("nested maps also limited", func(t *testing.T) { // Build N nested single-entry maps each with key "x" wrapping a null. const depth = 2000 buf := make([]byte, 0, 3*depth+1) for i := 0; i < depth; i++ { buf = append(buf, 0xA1) // map(1) buf = append(buf, 0x61) // text(1) buf = append(buf, 'x') } buf = append(buf, 0xF6) // null nb := basicnode.Prototype.Any.NewBuilder() err := Decode(nb, bytes.NewReader(buf)) qt.Assert(t, err, qt.Equals, ErrDecodeDepthExceeded) }) t.Run("zero MaxDepth resolves to default", func(t *testing.T) { payload := buildNestedArrays(2000) nb := basicnode.Prototype.Any.NewBuilder() err := DecodeOptions{MaxDepth: 0}.Decode(nb, bytes.NewReader(payload)) qt.Assert(t, err, qt.Equals, ErrDecodeDepthExceeded) }) t.Run("indefinite-length collections also limited", func(t *testing.T) { // Stream of 0x9F (indefinite list open) markers then a null, then // matching 0xFF break bytes. Exercises the indefinite-length branch // of the decoder which has a separate code path to the definite one. const depth = 2000 buf := make([]byte, 0, 2*depth+1) for i := 0; i < depth; i++ { buf = append(buf, 0x9F) } buf = append(buf, 0xF6) for i := 0; i < depth; i++ { buf = append(buf, 0xFF) } nb := basicnode.Prototype.Any.NewBuilder() err := Decode(nb, bytes.NewReader(buf)) qt.Assert(t, err, qt.Equals, ErrDecodeDepthExceeded) }) } // TestDecoderBoundaries asserts that decoder-layer and tokenizer-layer limits // behave as expected for unusual or malformed inputs. These are sanity tests // around boundaries that callers sometimes need to reason about. func TestDecoderBoundaries(t *testing.T) { t.Run("oversized string declaration rejected by tokenizer", func(t *testing.T) { // Text header declaring 1 TiB; no following bytes. The underlying // refmt tokenizer caps string/bytes length before attempting to read. var buf bytes.Buffer buf.WriteByte(0x7B) // text(uint64 length) binary.Write(&buf, binary.BigEndian, uint64(1<<40)) nb := basicnode.Prototype.Any.NewBuilder() err := Decode(nb, bytes.NewReader(buf.Bytes())) qt.Assert(t, err, qt.Not(qt.IsNil)) }) t.Run("stacked CBOR tags rejected", func(t *testing.T) { // CBOR permits tagging a value, but the tokenizer refuses to stack // multiple tags on a single item. Link handling relies on this. payload := []byte{ 0xD8, 42, // tag(42) 0xD8, 42, // tag(42) 0x42, 0x00, 0x01, } nb := basicnode.Prototype.Any.NewBuilder() err := DecodeOptions{AllowLinks: true}.Decode(nb, bytes.NewReader(payload)) qt.Assert(t, err, qt.Not(qt.IsNil)) }) t.Run("indefinite collection exceeding budget rejected", func(t *testing.T) { // Indefinite-length arrays have no declared size, but per-entry // budget still applies as entries are read. const entries = 3_000_000 var buf bytes.Buffer buf.Grow(1 + entries + 1) buf.WriteByte(0x9F) // indefinite array buf.Write(make([]byte, entries)) // entries zero-valued uints buf.WriteByte(0xFF) // break nb := basicnode.Prototype.Any.NewBuilder() err := Decode(nb, bytes.NewReader(buf.Bytes())) qt.Assert(t, err, qt.Equals, ErrAllocationBudgetExceeded) }) } ================================================ FILE: codec/dagjson/marshal.go ================================================ package dagjson import ( "encoding/base64" "fmt" "io" "sort" "github.com/polydawn/refmt/json" "github.com/polydawn/refmt/shared" "github.com/polydawn/refmt/tok" "github.com/ipld/go-ipld-prime/codec" "github.com/ipld/go-ipld-prime/datamodel" cidlink "github.com/ipld/go-ipld-prime/linking/cid" ) // This should be identical to the general feature in the parent package, // except for the `case datamodel.Kind_Link` block, // which is dag-json's special sauce for schemafree links. // EncodeOptions can be used to customize the behavior of an encoding function. // The Encode method on this struct fits the codec.Encoder function interface. type EncodeOptions struct { // If true, will encode nodes with a Link kind using the DAG-JSON // `{"/":"cid string"}` form. EncodeLinks bool // If true, will encode nodes with a Bytes kind using the DAG-JSON // `{"/":{"bytes":"base64 bytes..."}}` form. EncodeBytes bool // Control the sorting of map keys, using one of the `codec.MapSortMode_*` constants. MapSortMode codec.MapSortMode } // Encode walks the given datamodel.Node and serializes it to the given io.Writer. // Encode fits the codec.Encoder function interface. // // The behavior of the encoder can be customized by setting fields in the EncodeOptions struct before calling this method. func (cfg EncodeOptions) Encode(n datamodel.Node, w io.Writer) error { return Marshal(n, json.NewEncoder(w, json.EncodeOptions{}), cfg) } // Future work: we would like to remove the Marshal function, // and in particular, stop seeing types from refmt (like shared.TokenSink) be visible. // Right now, some kinds of configuration (e.g. for whitespace and prettyprint) are only available through interacting with the refmt types; // we should improve our API so that this can be done with only our own types in this package. // Marshal is a deprecated function. // Please consider switching to EncodeOptions.Encode instead. func Marshal(n datamodel.Node, sink shared.TokenSink, options EncodeOptions) error { var tk tok.Token switch n.Kind() { case datamodel.Kind_Invalid: return fmt.Errorf("cannot traverse a node that is absent") case datamodel.Kind_Null: tk.Type = tok.TNull _, err := sink.Step(&tk) return err case datamodel.Kind_Map: // Emit start of map. tk.Type = tok.TMapOpen expectedLength := int(n.Length()) tk.Length = expectedLength // TODO: overflow check if _, err := sink.Step(&tk); err != nil { return err } if options.MapSortMode != codec.MapSortMode_None { // Collect map entries, then sort by key type entry struct { key string value datamodel.Node } entries := []entry{} for itr := n.MapIterator(); !itr.Done(); { k, v, err := itr.Next() if err != nil { return err } keyStr, err := k.AsString() if err != nil { return err } entries = append(entries, entry{keyStr, v}) } if len(entries) != expectedLength { return fmt.Errorf("map Length() does not match number of MapIterator() entries") } // Apply the desired sort function. switch options.MapSortMode { case codec.MapSortMode_Lexical: sort.Slice(entries, func(i, j int) bool { return entries[i].key < entries[j].key }) case codec.MapSortMode_RFC7049: sort.Slice(entries, func(i, j int) bool { // RFC7049 style sort as per DAG-CBOR spec li, lj := len(entries[i].key), len(entries[j].key) if li == lj { return entries[i].key < entries[j].key } return li < lj }) } // Emit map contents (and recurse). var entryCount int for _, e := range entries { tk.Type = tok.TString tk.Str = e.key entryCount++ if _, err := sink.Step(&tk); err != nil { return err } if err := Marshal(e.value, sink, options); err != nil { return err } } if entryCount != expectedLength { return fmt.Errorf("map Length() does not match number of MapIterator() entries") } } else { // Don't sort map, emit map contents (and recurse). for itr := n.MapIterator(); !itr.Done(); { k, v, err := itr.Next() if err != nil { return err } tk.Type = tok.TString tk.Str, err = k.AsString() if err != nil { return err } if _, err := sink.Step(&tk); err != nil { return err } if err := Marshal(v, sink, options); err != nil { return err } } } // Emit map close. tk.Type = tok.TMapClose _, err := sink.Step(&tk) return err case datamodel.Kind_List: // Emit start of list. tk.Type = tok.TArrOpen l := n.Length() tk.Length = int(l) // TODO: overflow check if _, err := sink.Step(&tk); err != nil { return err } // Emit list contents (and recurse). for i := int64(0); i < l; i++ { v, err := n.LookupByIndex(i) if err != nil { return err } if err := Marshal(v, sink, options); err != nil { return err } } // Emit list close. tk.Type = tok.TArrClose _, err := sink.Step(&tk) return err case datamodel.Kind_Bool: v, err := n.AsBool() if err != nil { return err } tk.Type = tok.TBool tk.Bool = v _, err = sink.Step(&tk) return err case datamodel.Kind_Int: v, err := n.AsInt() if err != nil { return err } tk.Type = tok.TInt tk.Int = int64(v) _, err = sink.Step(&tk) return err case datamodel.Kind_Float: v, err := n.AsFloat() if err != nil { return err } tk.Type = tok.TFloat64 tk.Float64 = v _, err = sink.Step(&tk) return err case datamodel.Kind_String: v, err := n.AsString() if err != nil { return err } tk.Type = tok.TString tk.Str = v _, err = sink.Step(&tk) return err case datamodel.Kind_Bytes: if !options.EncodeBytes { return fmt.Errorf("cannot marshal IPLD bytes to this codec") } v, err := n.AsBytes() if err != nil { return err } // Precisely seven tokens to emit: tk.Type = tok.TMapOpen tk.Length = 1 if _, err = sink.Step(&tk); err != nil { return err } tk.Type = tok.TString tk.Str = "/" if _, err = sink.Step(&tk); err != nil { return err } tk.Type = tok.TMapOpen tk.Length = 1 if _, err = sink.Step(&tk); err != nil { return err } tk.Type = tok.TString tk.Str = "bytes" if _, err = sink.Step(&tk); err != nil { return err } tk.Str = base64.RawStdEncoding.EncodeToString(v) if _, err = sink.Step(&tk); err != nil { return err } tk.Type = tok.TMapClose if _, err = sink.Step(&tk); err != nil { return err } tk.Type = tok.TMapClose if _, err = sink.Step(&tk); err != nil { return err } return nil case datamodel.Kind_Link: if !options.EncodeLinks { return fmt.Errorf("cannot marshal IPLD links to this codec") } v, err := n.AsLink() if err != nil { return err } switch lnk := v.(type) { case cidlink.Link: if !lnk.Cid.Defined() { return fmt.Errorf("encoding undefined CIDs are not supported by this codec") } // Precisely four tokens to emit: tk.Type = tok.TMapOpen tk.Length = 1 if _, err = sink.Step(&tk); err != nil { return err } tk.Type = tok.TString tk.Str = "/" if _, err = sink.Step(&tk); err != nil { return err } tk.Str = lnk.Cid.String() if _, err = sink.Step(&tk); err != nil { return err } tk.Type = tok.TMapClose if _, err = sink.Step(&tk); err != nil { return err } return nil default: return fmt.Errorf("schemafree link emission only supported by this codec for CID type links; got type %T", lnk) } default: panic("unreachable") } } ================================================ FILE: codec/dagjson/marshal_test.go ================================================ package dagjson import ( "testing" qt "github.com/frankban/quicktest" cid "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/fluent/qp" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipld/go-ipld-prime/node/basicnode" ) var link = cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi") func TestMarshalUndefCid(t *testing.T) { node, err := qp.BuildMap(basicnode.Prototype.Any, -1, func(ma datamodel.MapAssembler) { qp.MapEntry(ma, "UndefCid", qp.Link(cidlink.Link{Cid: cid.Undef})) qp.MapEntry(ma, "DefCid", qp.Link(cidlink.Link{Cid: link})) }) qt.Assert(t, err, qt.IsNil) _, err = ipld.Encode(node, Encode) qt.Assert(t, err, qt.ErrorMatches, "encoding undefined CIDs are not supported by this codec") } // mirrored in json but with errors func TestMarshalLinks(t *testing.T) { linkNode := basicnode.NewLink(cidlink.Link{Cid: link}) mapNode, err := qp.BuildMap(basicnode.Prototype.Any, -1, func(ma datamodel.MapAssembler) { qp.MapEntry(ma, "Lnk", qp.Node(linkNode)) }) qt.Assert(t, err, qt.IsNil) t.Run("link dag-json", func(t *testing.T) { byts, err := ipld.Encode(linkNode, Encode) qt.Assert(t, err, qt.IsNil) qt.Assert(t, string(byts), qt.Equals, `{"/":"bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi"}`) }) t.Run("nested link dag-json", func(t *testing.T) { byts, err := ipld.Encode(mapNode, Encode) qt.Assert(t, err, qt.IsNil) qt.Assert(t, string(byts), qt.Equals, `{"Lnk":{"/":"bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi"}}`) }) } // mirrored in json but with errors func TestMarshalBytes(t *testing.T) { bytsNode := basicnode.NewBytes([]byte("byte me")) mapNode, err := qp.BuildMap(basicnode.Prototype.Any, -1, func(ma datamodel.MapAssembler) { qp.MapEntry(ma, "Byts", qp.Node(bytsNode)) }) qt.Assert(t, err, qt.IsNil) t.Run("bytes dag-json", func(t *testing.T) { byts, err := ipld.Encode(bytsNode, Encode) qt.Assert(t, err, qt.IsNil) qt.Assert(t, string(byts), qt.Equals, `{"/":{"bytes":"Ynl0ZSBtZQ"}}`) }) t.Run("nested bytes dag-json", func(t *testing.T) { byts, err := ipld.Encode(mapNode, Encode) qt.Assert(t, err, qt.IsNil) qt.Assert(t, string(byts), qt.Equals, `{"Byts":{"/":{"bytes":"Ynl0ZSBtZQ"}}}`) }) } ================================================ FILE: codec/dagjson/multicodec.go ================================================ package dagjson import ( "io" "github.com/ipld/go-ipld-prime/codec" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/multicodec" ) var ( _ codec.Decoder = Decode _ codec.Encoder = Encode ) func init() { multicodec.RegisterEncoder(0x0129, Encode) multicodec.RegisterDecoder(0x0129, Decode) } // Decode deserializes data from the given io.Reader and feeds it into the given datamodel.NodeAssembler. // Decode fits the codec.Decoder function interface. // // A similar function is available on DecodeOptions type if you would like to customize any of the decoding details. // This function uses the defaults for the dag-json codec // (meaning: links are decoded, and bytes are decoded). // // This is the function that will be registered in the default multicodec registry during package init time. func Decode(na datamodel.NodeAssembler, r io.Reader) error { return DecodeOptions{ ParseLinks: true, ParseBytes: true, }.Decode(na, r) } // Encode walks the given datamodel.Node and serializes it to the given io.Writer. // Encode fits the codec.Encoder function interface. // // A similar function is available on EncodeOptions type if you would like to customize any of the encoding details. // This function uses the defaults for the dag-json codec // (meaning: links are encoded, bytes are encoded, and map keys are sorted during encode). // // This is the function that will be registered in the default multicodec registry during package init time. func Encode(n datamodel.Node, w io.Writer) error { return EncodeOptions{ EncodeLinks: true, EncodeBytes: true, MapSortMode: codec.MapSortMode_Lexical, }.Encode(n, w) } ================================================ FILE: codec/dagjson/nongreedy_test.go ================================================ package dagjson import ( "bytes" "testing" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/node/basicnode" ) func TestNonGreedy(t *testing.T) { buf := bytes.NewBufferString(`{"a": 1}{"b": 2}`) opts := DecodeOptions{ ParseLinks: false, ParseBytes: false, DontParseBeyondEnd: true, } nb1 := basicnode.Prototype.Map.NewBuilder() err := opts.Decode(nb1, buf) if err != nil { t.Fatalf("first decode (%v)", err) } n1 := nb1.Build() if n1.Kind() != datamodel.Kind_Map { t.Errorf("expecting a map") } if _, err := n1.LookupByString("a"); err != nil { t.Fatalf("missing fist key") } nb2 := basicnode.Prototype.Map.NewBuilder() err = opts.Decode(nb2, buf) if err != nil { t.Fatalf("second decode (%v)", err) } n2 := nb2.Build() if n2.Kind() != datamodel.Kind_Map { t.Errorf("expecting a map") } if _, err := n2.LookupByString("b"); err != nil { t.Fatalf("missing second key") } } ================================================ FILE: codec/dagjson/options_test.go ================================================ package dagjson import ( "bytes" "strings" "testing" qt "github.com/frankban/quicktest" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/node/basicnode" ) // TestDecodeOptions_MaxDepth verifies that the configurable nesting-depth // limit is respected, both with defaults and custom values. func TestDecodeOptions_MaxDepth(t *testing.T) { nested := func(depth int) []byte { return []byte(strings.Repeat("[", depth) + "null" + strings.Repeat("]", depth)) } t.Run("default depth rejects deeply nested structure", func(t *testing.T) { nb := basicnode.Prototype.Any.NewBuilder() err := Decode(nb, bytes.NewReader(nested(2000))) qt.Assert(t, err, qt.Equals, ErrDecodeDepthExceeded) }) t.Run("structure at default depth decodes", func(t *testing.T) { nb := basicnode.Prototype.Any.NewBuilder() err := Decode(nb, bytes.NewReader(nested(1024))) qt.Assert(t, err, qt.IsNil) qt.Assert(t, nb.Build().Kind(), qt.Equals, datamodel.Kind_List) }) t.Run("custom depth rejects when exceeded", func(t *testing.T) { nb := basicnode.Prototype.Any.NewBuilder() err := DecodeOptions{MaxDepth: 5}.Decode(nb, bytes.NewReader(nested(10))) qt.Assert(t, err, qt.Equals, ErrDecodeDepthExceeded) }) t.Run("custom depth accepts within limit", func(t *testing.T) { nb := basicnode.Prototype.Any.NewBuilder() err := DecodeOptions{MaxDepth: 10}.Decode(nb, bytes.NewReader(nested(5))) qt.Assert(t, err, qt.IsNil) qt.Assert(t, nb.Build().Kind(), qt.Equals, datamodel.Kind_List) }) t.Run("nested maps also limited", func(t *testing.T) { const depth = 2000 buf := strings.Repeat(`{"x":`, depth) + "null" + strings.Repeat("}", depth) nb := basicnode.Prototype.Any.NewBuilder() err := Decode(nb, bytes.NewReader([]byte(buf))) qt.Assert(t, err, qt.Equals, ErrDecodeDepthExceeded) }) t.Run("zero MaxDepth resolves to default", func(t *testing.T) { nb := basicnode.Prototype.Any.NewBuilder() err := DecodeOptions{MaxDepth: 0}.Decode(nb, bytes.NewReader(nested(2000))) qt.Assert(t, err, qt.Equals, ErrDecodeDepthExceeded) }) t.Run("ParseLinks lookahead does not bypass depth", func(t *testing.T) { // A valid DAG-JSON link ({"/":"..."}) wrapped in deep list nesting. // The lookahead path for ParseLinks must still honour the depth limit. const depth = 2000 buf := strings.Repeat("[", depth) + `{"/":"bafkqaaa"}` + strings.Repeat("]", depth) nb := basicnode.Prototype.Any.NewBuilder() err := DecodeOptions{ParseLinks: true}.Decode(nb, bytes.NewReader([]byte(buf))) qt.Assert(t, err, qt.Equals, ErrDecodeDepthExceeded) }) t.Run("ParseBytes lookahead does not bypass depth", func(t *testing.T) { // A valid DAG-JSON bytes object wrapped in deep list nesting. const depth = 2000 buf := strings.Repeat("[", depth) + `{"/":{"bytes":"aGVsbG8"}}` + strings.Repeat("]", depth) nb := basicnode.Prototype.Any.NewBuilder() err := DecodeOptions{ParseBytes: true}.Decode(nb, bytes.NewReader([]byte(buf))) qt.Assert(t, err, qt.Equals, ErrDecodeDepthExceeded) }) t.Run("ParseLinks within limit resolves link correctly", func(t *testing.T) { // Depth 5 well within the default limit; ensure the lookahead path // still yields a Link node when not overflowing. buf := strings.Repeat("[", 5) + `{"/":"bafkqaaa"}` + strings.Repeat("]", 5) nb := basicnode.Prototype.Any.NewBuilder() err := DecodeOptions{ParseLinks: true}.Decode(nb, bytes.NewReader([]byte(buf))) qt.Assert(t, err, qt.IsNil) n := nb.Build() for i := 0; i < 5; i++ { qt.Assert(t, n.Kind(), qt.Equals, datamodel.Kind_List) n, err = n.LookupByIndex(0) qt.Assert(t, err, qt.IsNil) } qt.Assert(t, n.Kind(), qt.Equals, datamodel.Kind_Link) }) } ================================================ FILE: codec/dagjson/roundtripBytes_test.go ================================================ package dagjson_test import ( "bytes" "strings" "testing" qt "github.com/frankban/quicktest" "github.com/ipld/go-ipld-prime/codec/dagjson" "github.com/ipld/go-ipld-prime/fluent" "github.com/ipld/go-ipld-prime/node/basicnode" nodetests "github.com/ipld/go-ipld-prime/node/tests" ) var byteNode = fluent.MustBuildMap(basicnode.Prototype.Map, 4, func(na fluent.MapAssembler) { na.AssembleEntry("plain").AssignString("olde string") na.AssembleEntry("bytes").AssignBytes([]byte("deadbeef")) }) var byteNodeSorted = fluent.MustBuildMap(basicnode.Prototype.Map, 4, func(na fluent.MapAssembler) { na.AssembleEntry("bytes").AssignBytes([]byte("deadbeef")) na.AssembleEntry("plain").AssignString("olde string") }) var byteSerial = `{"bytes":{"/":{"bytes":"ZGVhZGJlZWY"}},"plain":"olde string"}` func TestRoundtripBytes(t *testing.T) { t.Run("encoding", func(t *testing.T) { var buf bytes.Buffer err := dagjson.Encode(byteNode, &buf) qt.Assert(t, err, qt.IsNil) qt.Check(t, buf.String(), qt.Equals, byteSerial) }) t.Run("decoding", func(t *testing.T) { buf := strings.NewReader(byteSerial) nb := basicnode.Prototype.Map.NewBuilder() err := dagjson.Decode(nb, buf) qt.Assert(t, err, qt.IsNil) qt.Check(t, nb.Build(), nodetests.NodeContentEquals, byteNodeSorted) }) } var encapsulatedNode = fluent.MustBuildMap(basicnode.Prototype.Map, 1, func(na fluent.MapAssembler) { na.AssembleEntry("/").CreateMap(1, func(sa fluent.MapAssembler) { sa.AssembleEntry("bytes").AssignBytes([]byte("deadbeef")) }) }) var encapsulatedSerial = `{"/":{"bytes":{"/":{"bytes":"ZGVhZGJlZWY"}}}}` func TestEncapsulatedBytes(t *testing.T) { t.Run("encoding", func(t *testing.T) { var buf bytes.Buffer err := dagjson.Encode(encapsulatedNode, &buf) qt.Assert(t, err, qt.IsNil) qt.Check(t, buf.String(), qt.Equals, encapsulatedSerial) }) t.Run("decoding", func(t *testing.T) { buf := strings.NewReader(encapsulatedSerial) nb := basicnode.Prototype.Map.NewBuilder() err := dagjson.Decode(nb, buf) qt.Assert(t, err, qt.IsNil) qt.Check(t, nb.Build(), nodetests.NodeContentEquals, encapsulatedNode) }) } var withPadding = `{"/": {"bytes": "Bxrk96XO8cwr3hrcL4VeWtVdYudzHv47BbBl7CesWvmjRrRPOLZp9Ukg6sivn5Nqg4V5X2w43mk4Ppuzr+M+DA=="}}` func TestPaddedBytes(t *testing.T) { t.Run("decoding", func(t *testing.T) { buf := strings.NewReader(withPadding) nb := basicnode.Prototype.Bytes.NewBuilder() err := dagjson.Decode(nb, buf) qt.Assert(t, err, qt.IsNil) }) } ================================================ FILE: codec/dagjson/roundtripCidlink_test.go ================================================ package dagjson_test import ( "bytes" "io" "strings" "testing" qt "github.com/frankban/quicktest" cid "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime/codec/dagjson" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/linking" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipld/go-ipld-prime/node/basicnode" nodetests "github.com/ipld/go-ipld-prime/node/tests" ) func TestRoundtripCidlink(t *testing.T) { lp := cidlink.LinkPrototype{Prefix: cid.Prefix{ Version: 1, Codec: 0x0129, MhType: 0x13, MhLength: 4, }} lsys := cidlink.DefaultLinkSystem() buf := bytes.Buffer{} lsys.StorageWriteOpener = func(lnkCtx linking.LinkContext) (io.Writer, linking.BlockWriteCommitter, error) { return &buf, func(lnk datamodel.Link) error { return nil }, nil } lsys.StorageReadOpener = func(lnkCtx linking.LinkContext, lnk datamodel.Link) (io.Reader, error) { return bytes.NewReader(buf.Bytes()), nil } lnk, err := lsys.Store(linking.LinkContext{}, lp, n) qt.Assert(t, err, qt.IsNil) n2, err := lsys.Load(linking.LinkContext{}, lnk, basicnode.Prototype.Any) qt.Assert(t, err, qt.IsNil) qt.Check(t, n2, nodetests.NodeContentEquals, nSorted) } // Make sure that a map that *almost* looks like a link is handled safely. // // This is aiming very specifically at the corner case where a minimal number of // tokens have to be reprocessed before a recursion that find a real link appears. func TestUnmarshalTrickyMapContainingLink(t *testing.T) { // Create a link; don't particularly care about its contents. lnk := cidlink.LinkPrototype{Prefix: cid.Prefix{ Version: 1, Codec: 0x71, MhType: 0x13, MhLength: 4, }}.BuildLink([]byte{1, 2, 3, 4}) // dummy value, content does not matter to this test. // Compose the tricky corpus. (lnk.String "happens" to work here, although this isn't recommended or correct in general.) tricky := `{"/":{"/":"` + lnk.String() + `"}}` // Unmarshal. Hopefully we get a map with a link in it. nb := basicnode.Prototype.Any.NewBuilder() err := dagjson.Decode(nb, strings.NewReader(tricky)) qt.Assert(t, err, qt.IsNil) n := nb.Build() qt.Check(t, n.Kind(), qt.Equals, datamodel.Kind_Map) n2, err := n.LookupByString("/") qt.Assert(t, err, qt.IsNil) qt.Check(t, n2.Kind(), qt.Equals, datamodel.Kind_Link) } ================================================ FILE: codec/dagjson/roundtrip_test.go ================================================ package dagjson_test import ( "bytes" "math/rand" "strings" "testing" "time" qt "github.com/frankban/quicktest" "github.com/ipld/go-ipld-prime/codec/dagjson" "github.com/ipld/go-ipld-prime/fluent" "github.com/ipld/go-ipld-prime/node/basicnode" nodetests "github.com/ipld/go-ipld-prime/node/tests" "github.com/ipld/go-ipld-prime/testutil/garbage" ) var n = fluent.MustBuildMap(basicnode.Prototype.Map, 4, func(na fluent.MapAssembler) { na.AssembleEntry("plain").AssignString("olde string") na.AssembleEntry("map").CreateMap(2, func(na fluent.MapAssembler) { na.AssembleEntry("one").AssignInt(1) na.AssembleEntry("two").AssignInt(2) }) na.AssembleEntry("list").CreateList(2, func(na fluent.ListAssembler) { na.AssembleValue().AssignString("three") na.AssembleValue().AssignString("four") }) na.AssembleEntry("nested").CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry("deeper").CreateList(1, func(na fluent.ListAssembler) { na.AssembleValue().AssignString("things") }) }) }) var nSorted = fluent.MustBuildMap(basicnode.Prototype.Map, 4, func(na fluent.MapAssembler) { na.AssembleEntry("list").CreateList(2, func(na fluent.ListAssembler) { na.AssembleValue().AssignString("three") na.AssembleValue().AssignString("four") }) na.AssembleEntry("map").CreateMap(2, func(na fluent.MapAssembler) { na.AssembleEntry("one").AssignInt(1) na.AssembleEntry("two").AssignInt(2) }) na.AssembleEntry("nested").CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry("deeper").CreateList(1, func(na fluent.ListAssembler) { na.AssembleValue().AssignString("things") }) }) na.AssembleEntry("plain").AssignString("olde string") }) var serial = `{"list":["three","four"],"map":{"one":1,"two":2},"nested":{"deeper":["things"]},"plain":"olde string"}` func TestRoundtrip(t *testing.T) { t.Run("encoding", func(t *testing.T) { var buf bytes.Buffer err := dagjson.Encode(n, &buf) qt.Assert(t, err, qt.IsNil) qt.Check(t, buf.String(), qt.Equals, serial) }) t.Run("decoding", func(t *testing.T) { buf := strings.NewReader(serial) nb := basicnode.Prototype.Map.NewBuilder() err := dagjson.Decode(nb, buf) qt.Assert(t, err, qt.IsNil) qt.Check(t, nb.Build(), nodetests.NodeContentEquals, nSorted) }) } func TestRoundtripScalar(t *testing.T) { nb := basicnode.Prototype__String{}.NewBuilder() nb.AssignString("applesauce") simple := nb.Build() t.Run("encoding", func(t *testing.T) { var buf bytes.Buffer err := dagjson.Encode(simple, &buf) qt.Assert(t, err, qt.IsNil) qt.Check(t, buf.String(), qt.Equals, `"applesauce"`) }) t.Run("decoding", func(t *testing.T) { buf := strings.NewReader(`"applesauce"`) nb := basicnode.Prototype__String{}.NewBuilder() err := dagjson.Decode(nb, buf) qt.Assert(t, err, qt.IsNil) qt.Check(t, nb.Build(), nodetests.NodeContentEquals, simple) }) } func TestGarbage(t *testing.T) { t.Run("small garbage", func(t *testing.T) { seed := time.Now().Unix() t.Logf("randomness seed: %v\n", seed) rnd := rand.New(rand.NewSource(seed)) for i := 0; i < 1000; i++ { gbg := garbage.Generate(rnd, garbage.TargetBlockSize(1<<6)) var buf bytes.Buffer err := dagjson.Encode(gbg, &buf) qt.Assert(t, err, qt.IsNil) nb := basicnode.Prototype.Any.NewBuilder() err = dagjson.Decode(nb, bytes.NewReader(buf.Bytes())) qt.Assert(t, err, qt.IsNil) qt.Check(t, nb.Build(), nodetests.DeepNodeContentsEquals, gbg) } }) t.Run("medium garbage", func(t *testing.T) { seed := time.Now().Unix() t.Logf("randomness seed: %v\n", seed) rnd := rand.New(rand.NewSource(seed)) for i := 0; i < 100; i++ { gbg := garbage.Generate(rnd, garbage.TargetBlockSize(1<<16)) var buf bytes.Buffer err := dagjson.Encode(gbg, &buf) qt.Assert(t, err, qt.IsNil) nb := basicnode.Prototype.Any.NewBuilder() err = dagjson.Decode(nb, bytes.NewReader(buf.Bytes())) qt.Assert(t, err, qt.IsNil) qt.Check(t, nb.Build(), nodetests.DeepNodeContentsEquals, gbg) } }) t.Run("large garbage", func(t *testing.T) { seed := time.Now().Unix() t.Logf("randomness seed: %v\n", seed) rnd := rand.New(rand.NewSource(seed)) for i := 0; i < 10; i++ { gbg := garbage.Generate(rnd, garbage.TargetBlockSize(1<<20)) var buf bytes.Buffer err := dagjson.Encode(gbg, &buf) qt.Assert(t, err, qt.IsNil) nb := basicnode.Prototype.Any.NewBuilder() err = dagjson.Decode(nb, bytes.NewReader(buf.Bytes())) qt.Assert(t, err, qt.IsNil) qt.Check(t, nb.Build(), nodetests.DeepNodeContentsEquals, gbg) } }) } ================================================ FILE: codec/dagjson/unmarshal.go ================================================ package dagjson import ( "encoding/base64" "errors" "fmt" "io" cid "github.com/ipfs/go-cid" "github.com/polydawn/refmt/json" "github.com/polydawn/refmt/shared" "github.com/polydawn/refmt/tok" "github.com/ipld/go-ipld-prime/datamodel" cidlink "github.com/ipld/go-ipld-prime/linking/cid" ) // ErrDecodeDepthExceeded is returned when a decoded structure nests deeper // than the configured MaxDepth. var ErrDecodeDepthExceeded = errors.New("message structure exceeded maximum nesting depth") const defaultMaxDepth int64 = 1024 // This drifts pretty far from the general unmarshal in the parent package: // - we know JSON never has length hints, so we ignore that field in tokens; // - we know JSON never has tags, so we ignore that field as well; // - we have dag-json's special sauce for detecting schemafree links // (and this unfortunately turns out to *significantly* convolute the first // several steps of handling maps, because it necessitates peeking several // tokens before deciding what kind of value to create). // DecodeOptions can be used to customize the behavior of a decoding function. // The Decode method on this struct fits the codec.Decoder function interface. type DecodeOptions struct { // If true, parse DAG-JSON `{"/":"cid string"}` as a Link kind node rather // than a plain map ParseLinks bool // If true, parse DAG-JSON `{"/":{"bytes":"base64 bytes..."}}` as a Bytes kind // node rather than nested plain maps ParseBytes bool // If true, the decoder stops reading from the stream at the end of the JSON structure. // i.e. it does not slurp remaining whitespaces and EOF. // As per standard IPLD behavior, the parser considers the entire block to be // part of the JSON structure and will error if there is extraneous // non-whitespace data. DontParseBeyondEnd bool // MaxDepth sets the maximum nesting depth for decoded structures. If the // decoder encounters a map or list nested beyond this depth, it returns // ErrDecodeDepthExceeded. // // When zero, a default of 1024 is used. MaxDepth int64 } func (cfg DecodeOptions) maxDepth() int64 { if cfg.MaxDepth > 0 { return cfg.MaxDepth } return defaultMaxDepth } // Decode deserializes data from the given io.Reader and feeds it into the given datamodel.NodeAssembler. // Decode fits the codec.Decoder function interface. // // The behavior of the decoder can be customized by setting fields in the DecodeOptions struct before calling this method. func (cfg DecodeOptions) Decode(na datamodel.NodeAssembler, r io.Reader) error { err := Unmarshal(na, json.NewDecoder(r), cfg) if err != nil { return err } if cfg.DontParseBeyondEnd { return nil } // Slurp any remaining whitespace. // This behavior may be due for review. // (This is relevant if our reader is tee'ing bytes to a hasher, and // the json contained any trailing whitespace.) // (We can't actually support multiple objects per reader from here; // we can't unpeek if we find a non-whitespace token, so our only // option is to error if this reader seems to contain more content.) var buf [1]byte for { _, err := r.Read(buf[:]) switch buf[0] { case ' ', 0x0, '\t', '\r', '\n': // continue default: return fmt.Errorf("unexpected content after end of json object") } if err == nil { continue } else if err == io.EOF { return nil } else { return err } } } // Future work: we would like to remove the Unmarshal function, // and in particular, stop seeing types from refmt (like shared.TokenSource) be visible. // Right now, some kinds of configuration (e.g. for whitespace and prettyprint) are only available through interacting with the refmt types; // we should improve our API so that this can be done with only our own types in this package. // Unmarshal is a deprecated function. // Please consider switching to DecodeOptions.Decode instead. func Unmarshal(na datamodel.NodeAssembler, tokSrc shared.TokenSource, options DecodeOptions) error { var st unmarshalState st.options = options done, err := tokSrc.Step(&st.tk[0]) if err == io.EOF { return io.ErrUnexpectedEOF } if err != nil { return err } if done && !st.tk[0].Type.IsValue() && st.tk[0].Type != tok.TNull { return fmt.Errorf("unexpected eof") } return st.unmarshal(na, tokSrc, 0) } type unmarshalState struct { tk [7]tok.Token // mostly, only 0'th is used... but [1:7] are used during lookahead for links. shift int // how many times to slide something out of tk[1:7] instead of getting a new token. options DecodeOptions } // step leaves a "new" token in tk[0], // taking account of an shift left by linkLookahead. // It's only necessary to use this when handling maps, // since the situations resulting in nonzero shift are otherwise unreachable. // // At most, 'step' will be shifting buffered tokens for: // - the first map key // - the first map value (which will be a string) // - the second map key // // and so (fortunately! whew!) we can do this in a fixed amount of memory, // since none of those states can reach a recursion. func (st *unmarshalState) step(tokSrc shared.TokenSource) error { switch st.shift { case 0: _, err := tokSrc.Step(&st.tk[0]) return err case 1: st.tk[0] = st.tk[1] st.shift-- return nil case 2: st.tk[0] = st.tk[1] st.tk[1] = st.tk[2] st.shift-- return nil case 3: st.tk[0] = st.tk[1] st.tk[1] = st.tk[2] st.tk[2] = st.tk[3] st.shift-- return nil case 4: st.tk[0] = st.tk[1] st.tk[1] = st.tk[2] st.tk[2] = st.tk[3] st.tk[3] = st.tk[4] st.shift-- return nil case 5: st.tk[0] = st.tk[1] st.tk[1] = st.tk[2] st.tk[2] = st.tk[3] st.tk[3] = st.tk[4] st.tk[4] = st.tk[5] st.shift-- return nil case 6: st.tk[0] = st.tk[1] st.tk[1] = st.tk[2] st.tk[2] = st.tk[3] st.tk[3] = st.tk[4] st.tk[4] = st.tk[5] st.tk[5] = st.tk[6] st.shift-- return nil default: panic("unreachable") } } // ensure checks that the token lookahead-ahead (tk[lookhead]) is loaded from the underlying source. func (st *unmarshalState) ensure(tokSrc shared.TokenSource, lookahead int) error { if st.shift < lookahead { if _, err := tokSrc.Step(&st.tk[lookahead]); err != nil { return err } st.shift = lookahead } return nil } // linkLookahead is called after receiving a TMapOpen token; // when it returns, we will have either created a link, OR // it's not a link, and the caller should proceed to start a map // and while using st.step to ensure the peeked tokens are handled, OR // in case of error, the error should just rise. // If the bool return is true, we got a link, and you should not // continue to attempt to build a map. func (st *unmarshalState) linkLookahead(na datamodel.NodeAssembler, tokSrc shared.TokenSource) (bool, error) { // Peek next token. If it's a "/" string, link is still a possibility if err := st.ensure(tokSrc, 1); err != nil { return false, err } if st.tk[1].Type != tok.TString { return false, nil } if st.tk[1].Str != "/" { return false, nil } // Peek next token. If it's a string, link is still a possibility. // We won't try to parse it as a CID until we're sure it's the only thing in the map, though. if err := st.ensure(tokSrc, 2); err != nil { return false, err } if st.tk[2].Type != tok.TString { return false, nil } // Peek next token. If it's map close, we've got a link! // (Otherwise it had better be a string, because another map key is the // only other valid transition here... but we'll leave that check to the caller. if err := st.ensure(tokSrc, 3); err != nil { return false, err } if st.tk[3].Type != tok.TMapClose { return false, nil } // Okay, we made it -- this looks like a link. Parse it. // If it *doesn't* parse as a CID, we treat this as an error. elCid, err := cid.Decode(st.tk[2].Str) if err != nil { return false, err } if err := na.AssignLink(cidlink.Link{Cid: elCid}); err != nil { return false, err } // consume the look-ahead tokens st.shift = 0 return true, nil } func (st *unmarshalState) bytesLookahead(na datamodel.NodeAssembler, tokSrc shared.TokenSource) (bool, error) { // Peek next token. If it's a "/" string, bytes is still a possibility if err := st.ensure(tokSrc, 1); err != nil { return false, err } if st.tk[1].Type != tok.TString { return false, nil } if st.tk[1].Str != "/" { return false, nil } // Peek next token. If it's a map, bytes is still a possibility. if err := st.ensure(tokSrc, 2); err != nil { return false, err } if st.tk[2].Type != tok.TMapOpen { return false, nil } // peek next token. If it's the string "bytes", we're on track. if err := st.ensure(tokSrc, 3); err != nil { return false, err } if st.tk[3].Type != tok.TString { return false, nil } if st.tk[3].Str != "bytes" { return false, nil } // peek next token. if it's a string, we're on track. if err := st.ensure(tokSrc, 4); err != nil { return false, err } if st.tk[4].Type != tok.TString { return false, nil } // peek next token. if it's the first map close we're on track. if err := st.ensure(tokSrc, 5); err != nil { return false, err } if st.tk[5].Type != tok.TMapClose { return false, nil } // Peek next token. If it's map close, we've got bytes! if err := st.ensure(tokSrc, 6); err != nil { return false, err } if st.tk[6].Type != tok.TMapClose { return false, nil } // Okay, we made it -- this looks like bytes. Parse it. elBytes, err := base64.RawStdEncoding.DecodeString(st.tk[4].Str) if err != nil { if _, isInput := err.(base64.CorruptInputError); isInput { elBytes, err = base64.StdEncoding.DecodeString(st.tk[4].Str) } if err != nil { return false, err } } if err := na.AssignBytes(elBytes); err != nil { return false, err } // consume the look-ahead tokens st.shift = 0 return true, nil } // starts with the first token already primed. Necessary to get recursion // // to flow right without a peek+unpeek system. func (st *unmarshalState) unmarshal(na datamodel.NodeAssembler, tokSrc shared.TokenSource, depth int64) error { // FUTURE: check for schema.TypedNodeBuilder that's going to parse a Link (they can slurp any token kind they want). switch st.tk[0].Type { case tok.TMapOpen: if depth >= st.options.maxDepth() { return ErrDecodeDepthExceeded } // dag-json has special needs: we pump a few tokens ahead to look for dag-json's "link" pattern. // We can't actually call BeginMap until we're sure it's not gonna turn out to be a link. if st.options.ParseLinks { gotLink, err := st.linkLookahead(na, tokSrc) if err != nil { // return in error if any token peeks failed or if structure looked like a link but failed to parse as CID. return err } if gotLink { return nil } } if st.options.ParseBytes { gotBytes, err := st.bytesLookahead(na, tokSrc) if err != nil { return err } if gotBytes { return nil } } // Okay, now back to regularly scheduled map logic. ma, err := na.BeginMap(-1) if err != nil { return err } for { err := st.step(tokSrc) // shift next token into slot 0. if err != nil { // return in error if next token unreadable return err } switch st.tk[0].Type { case tok.TMapClose: return ma.Finish() case tok.TString: // continue default: return fmt.Errorf("unexpected %s token while expecting map key", st.tk[0].Type) } mva, err := ma.AssembleEntry(st.tk[0].Str) if err != nil { // return in error if the key was rejected return err } // Do another shift so the next token is primed before we recurse. err = st.step(tokSrc) if err != nil { // return in error if next token unreadable return err } err = st.unmarshal(mva, tokSrc, depth+1) if err != nil { // return in error if some part of the recursion errored return err } } case tok.TMapClose: return fmt.Errorf("unexpected mapClose token") case tok.TArrOpen: if depth >= st.options.maxDepth() { return ErrDecodeDepthExceeded } la, err := na.BeginList(-1) if err != nil { return err } for { _, err := tokSrc.Step(&st.tk[0]) if err != nil { return err } switch st.tk[0].Type { case tok.TArrClose: return la.Finish() default: err := st.unmarshal(la.AssembleValue(), tokSrc, depth+1) if err != nil { // return in error if some part of the recursion errored return err } } } case tok.TArrClose: return fmt.Errorf("unexpected arrClose token") case tok.TNull: return na.AssignNull() case tok.TString: return na.AssignString(st.tk[0].Str) case tok.TBytes: return na.AssignBytes(st.tk[0].Bytes) case tok.TBool: return na.AssignBool(st.tk[0].Bool) case tok.TInt: return na.AssignInt(st.tk[0].Int) case tok.TUint: return na.AssignInt(int64(st.tk[0].Uint)) // FIXME overflow check case tok.TFloat64: return na.AssignFloat(st.tk[0].Float64) default: panic("unreachable") } } ================================================ FILE: codec/decode_test.go ================================================ package codec_test import ( "errors" "io" "strings" "testing" _ "github.com/ipld/go-ipld-prime/codec/cbor" _ "github.com/ipld/go-ipld-prime/codec/dagcbor" _ "github.com/ipld/go-ipld-prime/codec/dagjson" _ "github.com/ipld/go-ipld-prime/codec/json" mcregistry "github.com/ipld/go-ipld-prime/multicodec" basicnode "github.com/ipld/go-ipld-prime/node/basic" "github.com/multiformats/go-multicodec" ) func TestDecodeZero(t *testing.T) { for _, code := range []multicodec.Code{ multicodec.Cbor, multicodec.DagCbor, multicodec.Json, multicodec.DagJson, } { t.Run(code.String(), func(t *testing.T) { nb := basicnode.Prototype.Any.NewBuilder() decode, err := mcregistry.LookupDecoder(uint64(code)) if err != nil { t.Fatal(err) } err = decode(nb, strings.NewReader("")) if !errors.Is(err, io.ErrUnexpectedEOF) { t.Fatalf("unexpected error: %v", err) } }) } } ================================================ FILE: codec/json/marshal_test.go ================================================ package json import ( "testing" qt "github.com/frankban/quicktest" "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/fluent/qp" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipld/go-ipld-prime/node/basicnode" ) var link = cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi") // mirrored in dag-json but without errors func TestMarshalLinks(t *testing.T) { linkNode := basicnode.NewLink(cidlink.Link{Cid: link}) mapNode, err := qp.BuildMap(basicnode.Prototype.Any, -1, func(ma datamodel.MapAssembler) { qp.MapEntry(ma, "Lnk", qp.Node(linkNode)) }) qt.Assert(t, err, qt.IsNil) t.Run("link json", func(t *testing.T) { _, err := ipld.Encode(linkNode, Encode) qt.Assert(t, err, qt.ErrorMatches, "cannot marshal IPLD links to this codec") }) t.Run("nested link json", func(t *testing.T) { _, err := ipld.Encode(mapNode, Encode) qt.Assert(t, err, qt.ErrorMatches, "cannot marshal IPLD links to this codec") }) } // mirrored in dag-json but without errors func TestMarshalBytes(t *testing.T) { bytsNode := basicnode.NewBytes([]byte("byte me")) mapNode, err := qp.BuildMap(basicnode.Prototype.Any, -1, func(ma datamodel.MapAssembler) { qp.MapEntry(ma, "Byts", qp.Node(bytsNode)) }) qt.Assert(t, err, qt.IsNil) t.Run("bytes json", func(t *testing.T) { _, err := ipld.Encode(bytsNode, Encode) qt.Assert(t, err, qt.ErrorMatches, "cannot marshal IPLD bytes to this codec") }) t.Run("nested bytes json", func(t *testing.T) { _, err := ipld.Encode(mapNode, Encode) qt.Assert(t, err, qt.ErrorMatches, "cannot marshal IPLD bytes to this codec") }) } ================================================ FILE: codec/json/multicodec.go ================================================ package json import ( "io" rfmtjson "github.com/polydawn/refmt/json" "github.com/ipld/go-ipld-prime/codec" "github.com/ipld/go-ipld-prime/codec/dagjson" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/multicodec" ) var ( _ codec.Decoder = Decode _ codec.Encoder = Encode ) func init() { multicodec.RegisterEncoder(0x0200, Encode) multicodec.RegisterDecoder(0x0200, Decode) } // Decode deserializes data from the given io.Reader and feeds it into the given datamodel.NodeAssembler. // Decode fits the codec.Decoder function interface. // // This is the function that will be registered in the default multicodec registry during package init time. func Decode(na datamodel.NodeAssembler, r io.Reader) error { return dagjson.DecodeOptions{ ParseLinks: false, ParseBytes: false, }.Decode(na, r) } // Encode walks the given datamodel.Node and serializes it to the given io.Writer. // Encode fits the codec.Encoder function interface. // // This is the function that will be registered in the default multicodec registry during package init time. func Encode(n datamodel.Node, w io.Writer) error { // Shell out directly to generic inspection path. // (There's not really any fastpaths of note for json.) // Write another function if you need to tune encoding options about whitespace. return dagjson.Marshal(n, rfmtjson.NewEncoder(w, rfmtjson.EncodeOptions{ Line: []byte{'\n'}, Indent: []byte{'\t'}, }), dagjson.EncodeOptions{ EncodeLinks: false, EncodeBytes: false, MapSortMode: codec.MapSortMode_None, }) } ================================================ FILE: codec/raw/codec.go ================================================ // Package raw implements IPLD's raw codec, which simply writes and reads a Node // which can be represented as bytes. // // The codec can be used with any node which supports AsBytes and AssignBytes. // In general, it only makes sense to use this codec on a plain "bytes" node // such as github.com/ipld/go-ipld-prime/node/basicnode.Prototype.Bytes. package raw import ( "fmt" "io" "github.com/ipld/go-ipld-prime/codec" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/multicodec" ) // TODO(mvdan): make go-ipld use go-multicodec soon const rawMulticodec = 0x55 var ( _ codec.Decoder = Decode _ codec.Encoder = Encode ) func init() { multicodec.RegisterEncoder(rawMulticodec, Encode) multicodec.RegisterDecoder(rawMulticodec, Decode) } // Decode implements decoding of a node with the raw codec. // // Note that if r has a Bytes method, such as is the case with *bytes.Buffer, we // will use those bytes directly to save having to allocate and copy them. The // Node interface is defined as immutable, so it is assumed that its bytes won't // be modified in-place. Similarly, we assume that the incoming buffer's bytes // won't get modified in-place later. // // To disable the shortcut above, hide the Bytes method by wrapping the buffer // with an io.Reader: // // Decode([...], struct{io.Reader}{buf}) func Decode(am datamodel.NodeAssembler, r io.Reader) error { var data []byte if buf, ok := r.(interface{ Bytes() []byte }); ok { data = buf.Bytes() } else { var err error data, err = io.ReadAll(r) if err != nil { return fmt.Errorf("could not decode raw node: %v", err) } } return am.AssignBytes(data) } // Encode implements encoding of a node with the raw codec. // // Note that Encode won't copy the node's bytes as returned by AsBytes, but the // call to Write will typically have to copy the bytes anyway. func Encode(node datamodel.Node, w io.Writer) error { data, err := node.AsBytes() if err != nil { return err } _, err = w.Write(data) return err } ================================================ FILE: codec/raw/codec_test.go ================================================ package raw import ( "bytes" "fmt" "io" "testing" qt "github.com/frankban/quicktest" "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/linking" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipld/go-ipld-prime/node/basicnode" nodetests "github.com/ipld/go-ipld-prime/node/tests" ) var tests = []struct { name string data []byte }{ {"Empty", nil}, {"Plaintext", []byte("hello there")}, {"JSON", []byte(`{"foo": "bar"}`)}, {"NullBytes", []byte("\x00\x00")}, } func TestRoundtrip(t *testing.T) { t.Parallel() for _, test := range tests { t.Run(test.name, func(t *testing.T) { nb := basicnode.Prototype.Bytes.NewBuilder() r := bytes.NewBuffer(test.data) err := Decode(nb, r) qt.Assert(t, err, qt.IsNil) node := nb.Build() buf := new(bytes.Buffer) err = Encode(node, buf) qt.Assert(t, err, qt.IsNil) qt.Assert(t, buf.Bytes(), qt.DeepEquals, test.data) }) } } func TestRoundtripCidlink(t *testing.T) { t.Parallel() lp := cidlink.LinkPrototype{Prefix: cid.Prefix{ Version: 1, Codec: rawMulticodec, MhType: 0x13, MhLength: 4, }} node := basicnode.NewBytes([]byte("hello there")) lsys := cidlink.DefaultLinkSystem() buf := bytes.Buffer{} lsys.StorageWriteOpener = func(lnkCtx linking.LinkContext) (io.Writer, linking.BlockWriteCommitter, error) { return &buf, func(lnk datamodel.Link) error { return nil }, nil } lsys.StorageReadOpener = func(lnkCtx linking.LinkContext, lnk datamodel.Link) (io.Reader, error) { return bytes.NewReader(buf.Bytes()), nil } lnk, err := lsys.Store(linking.LinkContext{}, lp, node) qt.Assert(t, err, qt.IsNil) newNode, err := lsys.Load(linking.LinkContext{}, lnk, basicnode.Prototype.Any) qt.Assert(t, err, qt.IsNil) qt.Assert(t, newNode, nodetests.NodeContentEquals, node) } // mustOnlyUseRead only exposes Read, hiding Bytes. type mustOnlyUseRead struct { buf *bytes.Buffer } func (r mustOnlyUseRead) Read(p []byte) (int, error) { return r.buf.Read(p) } // mustNotUseRead exposes Bytes and makes Read always error. type mustNotUseRead struct { buf *bytes.Buffer } func (r mustNotUseRead) Read(p []byte) (int, error) { return 0, fmt.Errorf("must not call Read") } func (r mustNotUseRead) Bytes() []byte { return r.buf.Bytes() } func TestDecodeBuffer(t *testing.T) { t.Parallel() var err error buf := bytes.NewBuffer([]byte("hello there")) err = Decode( basicnode.Prototype.Bytes.NewBuilder(), mustOnlyUseRead{buf}, ) qt.Assert(t, err, qt.IsNil) err = Decode( basicnode.Prototype.Bytes.NewBuilder(), mustNotUseRead{buf}, ) qt.Assert(t, err, qt.IsNil) } ================================================ FILE: codec.go ================================================ package ipld import ( "github.com/ipld/go-ipld-prime/codec" ) type ( Encoder = codec.Encoder Decoder = codec.Decoder ) ================================================ FILE: codecHelpers.go ================================================ package ipld import ( "bytes" "io" "reflect" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/node/bindnode" "github.com/ipld/go-ipld-prime/schema" ) // Encode serializes the given Node using the given Encoder function, // returning the serialized data or an error. // // The exact result data will depend the node content and on the encoder function, // but for example, using a json codec on a node with kind map will produce // a result starting in `{`, etc. // // Encode will automatically switch to encoding the representation form of the Node, // if it discovers the Node matches the schema.TypedNode interface. // This is probably what you want, in most cases; // if this is not desired, you can use the underlaying functions directly // (just look at the source of this function for an example of how!). // // If you would like this operation, but applied directly to a golang type instead of a Node, // look to the Marshal function. func Encode(n Node, encFn Encoder) ([]byte, error) { var buf bytes.Buffer err := EncodeStreaming(&buf, n, encFn) return buf.Bytes(), err } // EncodeStreaming is like Encode, but emits output to an io.Writer. func EncodeStreaming(wr io.Writer, n Node, encFn Encoder) error { if tn, ok := n.(schema.TypedNode); ok { n = tn.Representation() } return encFn(n, wr) } // Decode parses the given bytes into a Node using the given Decoder function, // returning a new Node or an error. // // The new Node that is returned will be the implementation from the node/basicnode package. // This implementation of Node will work for storing any kind of data, // but note that because it is general, it is also not necessarily optimized. // If you want more control over what kind of Node implementation (and thus memory layout) is used, // or want to use features like IPLD Schemas (which can be engaged by using a schema.TypedPrototype), // then look to the DecodeUsingPrototype family of functions, // which accept more parameters in order to give you that kind of control. // // If you would like this operation, but applied directly to a golang type instead of a Node, // look to the Unmarshal function. func Decode(b []byte, decFn Decoder) (Node, error) { return DecodeUsingPrototype(b, decFn, basicnode.Prototype.Any) } // DecodeStreaming is like Decode, but works on an io.Reader for input. func DecodeStreaming(r io.Reader, decFn Decoder) (Node, error) { return DecodeStreamingUsingPrototype(r, decFn, basicnode.Prototype.Any) } // DecodeUsingPrototype is like Decode, but with a NodePrototype parameter, // which gives you control over the Node type you'll receive, // and thus control over the memory layout, and ability to use advanced features like schemas. // (Decode is simply this function, but hardcoded to use basicnode.Prototype.Any.) // // DecodeUsingPrototype internally creates a NodeBuilder, and throws it away when done. // If building a high performance system, and creating data of the same shape repeatedly, // you may wish to use NodeBuilder directly, so that you can control and avoid these allocations. // // For symmetry with the behavior of Encode, DecodeUsingPrototype will automatically // switch to using the representation form of the node for decoding // if it discovers the NodePrototype matches the schema.TypedPrototype interface. // This is probably what you want, in most cases; // if this is not desired, you can use the underlaying functions directly // (just look at the source of this function for an example of how!). func DecodeUsingPrototype(b []byte, decFn Decoder, np NodePrototype) (Node, error) { return DecodeStreamingUsingPrototype(bytes.NewReader(b), decFn, np) } // DecodeStreamingUsingPrototype is like DecodeUsingPrototype, but works on an io.Reader for input. func DecodeStreamingUsingPrototype(r io.Reader, decFn Decoder, np NodePrototype) (Node, error) { if tnp, ok := np.(schema.TypedPrototype); ok { np = tnp.Representation() } nb := np.NewBuilder() if err := decFn(nb, r); err != nil { return nil, err } return nb.Build(), nil } // Marshal accepts a pointer to a Go value and an IPLD schema type, // and encodes the representation form of that data (which may be configured with the schema!) // using the given Encoder function. // // Marshal uses the node/bindnode subsystem. // See the documentation in that package for more details about its workings. // Please note that this subsystem is relatively experimental at this time. // // The schema.Type parameter is optional, and can be nil. // If given, it controls what kind of schema.Type (and what kind of representation strategy!) // to use when processing the data. // If absent, a default schema.Type will be inferred based on the golang type // (so, a struct in go will be inferred to have a schema with a similar struct, and the default representation strategy (e.g. map), etc). // Note that not all features of IPLD Schemas can be inferred from golang types alone. // For example, to use union types, the schema parameter will be required. // Similarly, to use most kinds of non-default representation strategy, the schema parameter is needed in order to convey that intention. func Marshal(encFn Encoder, bind interface{}, typ schema.Type, opts ...bindnode.Option) ([]byte, error) { n := bindnode.Wrap(bind, typ, opts...) return Encode(n.Representation(), encFn) } // MarshalStreaming is like Marshal, but emits output to an io.Writer. func MarshalStreaming(wr io.Writer, encFn Encoder, bind interface{}, typ schema.Type, opts ...bindnode.Option) error { n := bindnode.Wrap(bind, typ, opts...) return EncodeStreaming(wr, n.Representation(), encFn) } // Unmarshal accepts a pointer to a Go value and an IPLD schema type, // and fills the value with data by decoding into it with the given Decoder function. // // Unmarshal uses the node/bindnode subsystem. // See the documentation in that package for more details about its workings. // Please note that this subsystem is relatively experimental at this time. // // The schema.Type parameter is optional, and can be nil. // If given, it controls what kind of schema.Type (and what kind of representation strategy!) // to use when processing the data. // If absent, a default schema.Type will be inferred based on the golang type // (so, a struct in go will be inferred to have a schema with a similar struct, and the default representation strategy (e.g. map), etc). // Note that not all features of IPLD Schemas can be inferred from golang types alone. // For example, to use union types, the schema parameter will be required. // Similarly, to use most kinds of non-default representation strategy, the schema parameter is needed in order to convey that intention. // // In contrast to some other unmarshal conventions common in golang, // notice that we also return a Node value. // This Node points to the same data as the value you handed in as the bind parameter, // while making it available to read and iterate and handle as a ipld datamodel.Node. // If you don't need that interface, or intend to re-bind it later, you can discard that value. // // The 'bind' parameter may be nil. // In that case, the type of the nil is still used to infer what kind of value to return, // and a Node will still be returned based on that type. // bindnode.Unwrap can be used on that Node and will still return something // of the same golang type as the typed nil that was given as the 'bind' parameter. func Unmarshal(b []byte, decFn Decoder, bind interface{}, typ schema.Type, opts ...bindnode.Option) (Node, error) { return UnmarshalStreaming(bytes.NewReader(b), decFn, bind, typ, opts...) } // UnmarshalStreaming is like Unmarshal, but works on an io.Reader for input. func UnmarshalStreaming(r io.Reader, decFn Decoder, bind interface{}, typ schema.Type, opts ...bindnode.Option) (Node, error) { // Decode is fairly straightforward. np := bindnode.Prototype(bind, typ, opts...) n, err := DecodeStreamingUsingPrototype(r, decFn, np.Representation()) if err != nil { return nil, err } // ... but our approach above allocated new memory, and we have to copy it back out. // In the future, the bindnode API could be improved to make this easier. if !reflect.ValueOf(bind).IsNil() { reflect.ValueOf(bind).Elem().Set(reflect.ValueOf(bindnode.Unwrap(n)).Elem()) } // ... and we also have to re-bind a new node to the 'bind' value, // because probably the user will be surprised if mutating 'bind' doesn't affect the Node later. n = bindnode.Wrap(bind, typ, opts...) return n, err } ================================================ FILE: codecHelpers_test.go ================================================ package ipld_test import ( "fmt" "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/codec/json" "github.com/ipld/go-ipld-prime/must" "github.com/ipld/go-ipld-prime/schema" ) func Example_marshal() { type Foobar struct { Foo string Bar string } encoded, err := ipld.Marshal(json.Encode, &Foobar{"wow", "whee"}, nil) fmt.Printf("error: %v\n", err) fmt.Printf("data: %s\n", string(encoded)) // Output: // error: // data: { // "Foo": "wow", // "Bar": "whee" // } } // TODO: Example_Unmarshal, which uses nil and infers a typesystem. However, to match Example_Unmarshal_withSchema, that appears to need more features in bindnode. func Example_unmarshal_withSchema() { typesys := schema.MustTypeSystem( schema.SpawnStruct("Foobar", []schema.StructField{ schema.SpawnStructField("foo", "String", false, false), schema.SpawnStructField("bar", "String", false, false), }, schema.SpawnStructRepresentationMap(nil), ), schema.SpawnString("String"), ) type Foobar struct { Foo string Bar string } serial := []byte(`{"foo":"wow","bar":"whee"}`) foobar := Foobar{} n, err := ipld.Unmarshal(serial, json.Decode, &foobar, typesys.TypeByName("Foobar")) fmt.Printf("error: %v\n", err) fmt.Printf("go struct: %v\n", foobar) fmt.Printf("node kind and length: %s, %d\n", n.Kind(), n.Length()) fmt.Printf("node lookup 'foo': %q\n", must.String(must.Node(n.LookupByString("foo")))) // Output: // error: // go struct: {wow whee} // node kind and length: map, 2 // node lookup 'foo': "wow" } ================================================ FILE: datamodel/copy.go ================================================ package datamodel import ( "errors" "fmt" ) // Copy does an explicit shallow copy of a Node's data into a NodeAssembler. // // This can be used to flip data from one memory layout to another // (for example, from basicnode to using using bindnode, // or to codegenerated node implementations, // or to or from ADL nodes, etc). // // The copy is implemented by ranging over the contents if it's a recursive kind, // and for each of them, using `AssignNode` on the child values; // for scalars, it's just calling the appropriate `Assign*` method. // // Many NodeAssembler implementations use this as a fallback behavior in their // `AssignNode` method (that is, they call to this function after all other special // faster shortcuts they might prefer to employ, such as direct struct copying // if they share internal memory layouts, etc, have been tried already). func Copy(n Node, na NodeAssembler) error { if n == nil { return errors.New("cannot copy a nil node") } switch n.Kind() { case Kind_Null: if n.IsAbsent() { return errors.New("copying an absent node makes no sense") } return na.AssignNull() case Kind_Bool: v, err := n.AsBool() if err != nil { return fmt.Errorf("node violated contract: promised to be %v kind, but AsBool method returned %w", n.Kind(), err) } return na.AssignBool(v) case Kind_Int: v, err := n.AsInt() if err != nil { return fmt.Errorf("node violated contract: promised to be %v kind, but AsInt method returned %w", n.Kind(), err) } return na.AssignInt(v) case Kind_Float: v, err := n.AsFloat() if err != nil { return fmt.Errorf("node violated contract: promised to be %v kind, but AsFloat method returned %w", n.Kind(), err) } return na.AssignFloat(v) case Kind_String: v, err := n.AsString() if err != nil { return fmt.Errorf("node violated contract: promised to be %v kind, but AsString method returned %w", n.Kind(), err) } return na.AssignString(v) case Kind_Bytes: v, err := n.AsBytes() if err != nil { return fmt.Errorf("node violated contract: promised to be %v kind, but AsBytes method returned %w", n.Kind(), err) } return na.AssignBytes(v) case Kind_Link: v, err := n.AsLink() if err != nil { return fmt.Errorf("node violated contract: promised to be %v kind, but AsLink method returned %w", n.Kind(), err) } return na.AssignLink(v) case Kind_Map: ma, err := na.BeginMap(n.Length()) if err != nil { return err } itr := n.MapIterator() for !itr.Done() { k, v, err := itr.Next() if err != nil { return err } if v.IsAbsent() { continue } if err := ma.AssembleKey().AssignNode(k); err != nil { return err } if err := ma.AssembleValue().AssignNode(v); err != nil { return err } } return ma.Finish() case Kind_List: la, err := na.BeginList(n.Length()) if err != nil { return err } itr := n.ListIterator() for !itr.Done() { _, v, err := itr.Next() if err != nil { return err } if v.IsAbsent() { continue } if err := la.AssembleValue().AssignNode(v); err != nil { return err } } return la.Finish() default: return fmt.Errorf("node has invalid kind %v", n.Kind()) } } ================================================ FILE: datamodel/copy_test.go ================================================ package datamodel_test import ( "testing" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/fluent/qp" basic "github.com/ipld/go-ipld-prime/node/basicnode" ) var copyTests = []struct { name string na datamodel.NodeBuilder n datamodel.Node err string }{ {name: "Null / Any", na: basic.Prototype.Any.NewBuilder(), n: datamodel.Null}, {name: "Int / Any", na: basic.Prototype.Any.NewBuilder(), n: basic.NewInt(100)}, {name: "Int / Int", na: basic.Prototype.Int.NewBuilder(), n: basic.NewInt(1000)}, {name: "Bool / Any", na: basic.Prototype.Any.NewBuilder(), n: basic.NewBool(true)}, {name: "Bool / Bool", na: basic.Prototype.Bool.NewBuilder(), n: basic.NewBool(false)}, {name: "Float / Any", na: basic.Prototype.Any.NewBuilder(), n: basic.NewFloat(1.1)}, {name: "Float / Float", na: basic.Prototype.Float.NewBuilder(), n: basic.NewFloat(1.2)}, {name: "String / Any", na: basic.Prototype.Any.NewBuilder(), n: basic.NewString("mary had")}, {name: "String / String", na: basic.Prototype.String.NewBuilder(), n: basic.NewString("a little lamb")}, {name: "Bytes / Any", na: basic.Prototype.Any.NewBuilder(), n: basic.NewBytes([]byte("mary had"))}, {name: "Bytes / Bytes", na: basic.Prototype.Bytes.NewBuilder(), n: basic.NewBytes([]byte("a little lamb"))}, {name: "Link / Any", na: basic.Prototype.Any.NewBuilder(), n: basic.NewLink(globalLink)}, {name: "Link / Link", na: basic.Prototype.Link.NewBuilder(), n: basic.NewLink(globalLink2)}, { name: "List / Any", na: basic.Prototype.Any.NewBuilder(), n: qpMust(qp.BuildList(basic.Prototype.Any, -1, func(am datamodel.ListAssembler) { qp.ListEntry(am, qp.Int(7)) qp.ListEntry(am, qp.Int(8)) })), }, { name: "List / List", na: basic.Prototype.List.NewBuilder(), n: qpMust(qp.BuildList(basic.Prototype.List, -1, func(am datamodel.ListAssembler) { qp.ListEntry(am, qp.String("yep")) qp.ListEntry(am, qp.Int(8)) qp.ListEntry(am, qp.String("nope")) })), }, { name: "Map / Any", na: basic.Prototype.Any.NewBuilder(), n: qpMust(qp.BuildMap(basic.Prototype.Any, -1, func(am datamodel.MapAssembler) { qp.MapEntry(am, "foo", qp.Int(7)) qp.MapEntry(am, "bar", qp.Int(8)) })), }, { name: "Map / Map", na: basic.Prototype.Map.NewBuilder(), n: qpMust(qp.BuildMap(basic.Prototype.Map, -1, func(am datamodel.MapAssembler) { qp.MapEntry(am, "foo", qp.Int(7)) qp.MapEntry(am, "bar", qp.Int(8)) qp.MapEntry(am, "bang", qp.Link(globalLink)) })), }, {name: "nil", na: basic.Prototype.Any.NewBuilder(), n: nil, err: "cannot copy a nil node"}, {name: "absent", na: basic.Prototype.Any.NewBuilder(), n: datamodel.Absent, err: "copying an absent node makes no sense"}, } func TestCopy(t *testing.T) { for _, tt := range copyTests { tt := tt t.Run(tt.name, func(t *testing.T) { t.Parallel() err := datamodel.Copy(tt.n, tt.na) if err != nil { if tt.err != "" { if err.Error() != tt.err { t.Fatalf("expected error %q, got %q", tt.err, err.Error()) } } else { t.Fatal(err) } return } else if tt.err != "" { t.Fatalf("expected error %q, got nil", tt.err) return } out := tt.na.Build() if !datamodel.DeepEqual(tt.n, out) { t.Fatalf("deep equal failed") } }) } } ================================================ FILE: datamodel/doc.go ================================================ // The datamodel package defines the most essential interfaces for describing IPLD Data -- // such as Node, NodePrototype, NodeBuilder, Link, and Path. // // Note that since interfaces in this package are the core of the library, // choices made here maximize correctness and performance -- these choices // are *not* always the choices that would maximize ergonomics. // (Ergonomics can come on top; performance generally can't.) // You'll want to check out other packages for functions with more ergonomics; // for example, 'fluent' and its subpackages provide lots of ways to work with data; // 'traversal' provides some ergonomic features for walking around data graphs; // any use of schemas will provide a bunch of useful data validation options; // or you can make your own function decorators that do what *you* need. package datamodel ================================================ FILE: datamodel/equal.go ================================================ package datamodel // DeepEqual reports whether x and y are "deeply equal" as IPLD nodes. // This is similar to reflect.DeepEqual, but based around the Node interface. // // Two nodes must have the same kind to be deeply equal. // If either node has the invalid kind, the nodes are not deeply equal. // // Two nodes of scalar kinds (null, bool, int, float, string, bytes, link) // are deeply equal if their Go values, as returned by AsKind methods, are equal as // per Go's == comparison operator. // // Note that Links are compared in a shallow way, without being followed. // This will generally be enough, as it's rare to have two different links to the // same IPLD data by using a different codec or multihash type. // // Two nodes of recursive kinds (map, list) // must have the same length to be deeply equal. // Their elements, as reported by iterators, must be deeply equal. // The elements are compared in the iterator's order, // meaning two maps sorting the same keys differently might not be equal. // // Note that this function panics if either Node returns an error. // We only call valid methods for each Kind, // so an error should only happen if a Node implementation breaks that contract. // It is generally not recommended to call DeepEqual on ADL nodes. func DeepEqual(x, y Node) bool { if x == nil || y == nil { return x == y } xk, yk := x.Kind(), y.Kind() if xk != yk { return false } switch xk { // Scalar kinds. case Kind_Null: return x.IsNull() == y.IsNull() case Kind_Bool: xv, err := x.AsBool() if err != nil { panic(err) } yv, err := y.AsBool() if err != nil { panic(err) } return xv == yv case Kind_Int: xv, err := x.AsInt() if err != nil { panic(err) } yv, err := y.AsInt() if err != nil { panic(err) } return xv == yv case Kind_Float: xv, err := x.AsFloat() if err != nil { panic(err) } yv, err := y.AsFloat() if err != nil { panic(err) } return xv == yv case Kind_String: xv, err := x.AsString() if err != nil { panic(err) } yv, err := y.AsString() if err != nil { panic(err) } return xv == yv case Kind_Bytes: xv, err := x.AsBytes() if err != nil { panic(err) } yv, err := y.AsBytes() if err != nil { panic(err) } return string(xv) == string(yv) case Kind_Link: xv, err := x.AsLink() if err != nil { panic(err) } yv, err := y.AsLink() if err != nil { panic(err) } // Links are just compared via ==. // This requires the types to exactly match, // and the values to be equal as per == too. // This will generally work, // as ipld-prime assumes link types to be consistent. return xv == yv // Recursive kinds. case Kind_Map: if x.Length() != y.Length() { return false } xitr := x.MapIterator() yitr := y.MapIterator() for !xitr.Done() && !yitr.Done() { xkey, xval, err := xitr.Next() if err != nil { panic(err) } ykey, yval, err := yitr.Next() if err != nil { panic(err) } if !DeepEqual(xkey, ykey) { return false } if !DeepEqual(xval, yval) { return false } } return true case Kind_List: if x.Length() != y.Length() { return false } xitr := x.ListIterator() yitr := y.ListIterator() for !xitr.Done() && !yitr.Done() { _, xval, err := xitr.Next() if err != nil { panic(err) } _, yval, err := yitr.Next() if err != nil { panic(err) } if !DeepEqual(xval, yval) { return false } } return true // As per the docs, other kinds such as Invalid are not deeply equal. default: return false } } ================================================ FILE: datamodel/equal_test.go ================================================ package datamodel_test import ( "testing" "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/fluent/qp" cidlink "github.com/ipld/go-ipld-prime/linking/cid" basic "github.com/ipld/go-ipld-prime/node/basicnode" // shorter name for the tests ) var ( globalNode = basic.NewString("global") globalLink = func() datamodel.Link { someCid, _ := cid.Cast([]byte{1, 85, 0, 5, 0, 1, 2, 3, 4}) return cidlink.Link{Cid: someCid} }() globalLink2 = func() datamodel.Link { someCid, _ := cid.Cast([]byte{1, 85, 0, 5, 0, 5, 6, 7, 8}) return cidlink.Link{Cid: someCid} }() ) func qpMust(node datamodel.Node, err error) datamodel.Node { if err != nil { panic(err) } return node } var deepEqualTests = []struct { name string left, right datamodel.Node want bool }{ {"MismatchingKinds", basic.NewBool(true), basic.NewInt(3), false}, {"SameNodeSamePointer", globalNode, globalNode, true}, // Repeated basicnode.New invocations might return different pointers. {"SameNodeDiffPointer", basic.NewString("same"), basic.NewString("same"), true}, {"NilVsNil", nil, nil, true}, {"NilVsNull", nil, datamodel.Null, false}, {"SameKindNull", datamodel.Null, datamodel.Null, true}, {"DiffKindNull", datamodel.Null, datamodel.Absent, false}, {"SameKindBool", basic.NewBool(true), basic.NewBool(true), true}, {"DiffKindBool", basic.NewBool(true), basic.NewBool(false), false}, {"SameKindInt", basic.NewInt(12), basic.NewInt(12), true}, {"DiffKindInt", basic.NewInt(12), basic.NewInt(15), false}, {"SameKindFloat", basic.NewFloat(1.25), basic.NewFloat(1.25), true}, {"DiffKindFloat", basic.NewFloat(1.25), basic.NewFloat(1.75), false}, {"SameKindString", basic.NewString("foobar"), basic.NewString("foobar"), true}, {"DiffKindString", basic.NewString("foobar"), basic.NewString("baz"), false}, {"SameKindBytes", basic.NewBytes([]byte{5, 2, 3}), basic.NewBytes([]byte{5, 2, 3}), true}, {"DiffKindBytes", basic.NewBytes([]byte{5, 2, 3}), basic.NewBytes([]byte{5, 8, 3}), false}, {"SameKindLink", basic.NewLink(globalLink), basic.NewLink(globalLink), true}, {"DiffKindLink", basic.NewLink(globalLink), basic.NewLink(globalLink2), false}, { "SameKindList", qpMust(qp.BuildList(basic.Prototype.Any, -1, func(am datamodel.ListAssembler) { qp.ListEntry(am, qp.Int(7)) qp.ListEntry(am, qp.Int(8)) })), qpMust(qp.BuildList(basic.Prototype.Any, -1, func(am datamodel.ListAssembler) { qp.ListEntry(am, qp.Int(7)) qp.ListEntry(am, qp.Int(8)) })), true, }, { "DiffKindList_length", qpMust(qp.BuildList(basic.Prototype.Any, -1, func(am datamodel.ListAssembler) { qp.ListEntry(am, qp.Int(7)) qp.ListEntry(am, qp.Int(8)) })), qpMust(qp.BuildList(basic.Prototype.Any, -1, func(am datamodel.ListAssembler) { qp.ListEntry(am, qp.Int(7)) })), false, }, { "DiffKindList_elems", qpMust(qp.BuildList(basic.Prototype.Any, -1, func(am datamodel.ListAssembler) { qp.ListEntry(am, qp.Int(7)) qp.ListEntry(am, qp.Int(8)) })), qpMust(qp.BuildList(basic.Prototype.Any, -1, func(am datamodel.ListAssembler) { qp.ListEntry(am, qp.Int(3)) qp.ListEntry(am, qp.Int(2)) })), false, }, { "SameKindMap", qpMust(qp.BuildMap(basic.Prototype.Any, -1, func(am datamodel.MapAssembler) { qp.MapEntry(am, "foo", qp.Int(7)) qp.MapEntry(am, "bar", qp.Int(8)) })), qpMust(qp.BuildMap(basic.Prototype.Any, -1, func(am datamodel.MapAssembler) { qp.MapEntry(am, "foo", qp.Int(7)) qp.MapEntry(am, "bar", qp.Int(8)) })), true, }, { "DiffKindMap_length", qpMust(qp.BuildMap(basic.Prototype.Any, -1, func(am datamodel.MapAssembler) { qp.MapEntry(am, "foo", qp.Int(7)) qp.MapEntry(am, "bar", qp.Int(8)) })), qpMust(qp.BuildMap(basic.Prototype.Any, -1, func(am datamodel.MapAssembler) { qp.MapEntry(am, "foo", qp.Int(7)) })), false, }, { "DiffKindMap_elems", qpMust(qp.BuildMap(basic.Prototype.Any, -1, func(am datamodel.MapAssembler) { qp.MapEntry(am, "foo", qp.Int(7)) qp.MapEntry(am, "bar", qp.Int(8)) })), qpMust(qp.BuildMap(basic.Prototype.Any, -1, func(am datamodel.MapAssembler) { qp.MapEntry(am, "foo", qp.Int(3)) qp.MapEntry(am, "baz", qp.Int(8)) })), false, }, // TODO: tests involving different implementations, once bindnode is ready } func TestDeepEqual(t *testing.T) { t.Parallel() for _, tc := range deepEqualTests { tc := tc // capture range variable t.Run(tc.name, func(t *testing.T) { t.Parallel() got := datamodel.DeepEqual(tc.left, tc.right) if got != tc.want { t.Fatalf("DeepEqual got %v, want %v", got, tc.want) } }) } } ================================================ FILE: datamodel/errors.go ================================================ package datamodel import ( "fmt" ) // ErrWrongKind may be returned from functions on the Node interface when // a method is invoked which doesn't make sense for the Kind that node // concretely contains. // // For example, calling AsString on a map will return ErrWrongKind. // Calling Lookup on an int will similarly return ErrWrongKind. type ErrWrongKind struct { // TypeName may optionally indicate the named type of a node the function // was called on (if the node was typed!), or, may be the empty string. TypeName string // MethodName is literally the string for the operation attempted, e.g. // "AsString". // // For methods on nodebuilders, we say e.g. "NodeBuilder.CreateMap". MethodName string // ApprorpriateKind describes which Kinds the erroring method would // make sense for. AppropriateKind KindSet // ActualKind describes the Kind of the node the method was called on. // // In the case of typed nodes, this will typically refer to the 'natural' // data-model kind for such a type (e.g., structs will say 'map' here). ActualKind Kind // TODO: it may be desirable for this error to be able to describe the schema typekind, too, if applicable. // Of course this presents certain package import graph problems. Solution to this that maximizes user usability is unclear. } func (e ErrWrongKind) Error() string { if e.TypeName == "" { return fmt.Sprintf("func called on wrong kind: %q called on a %s node, but only makes sense on %s", e.MethodName, e.ActualKind, e.AppropriateKind) } else { return fmt.Sprintf("func called on wrong kind: %q called on a %s node (kind: %s), but only makes sense on %s", e.MethodName, e.TypeName, e.ActualKind, e.AppropriateKind) } } // TODO: revisit the claim below about ErrNoSuchField. I think we moved back away from that, or want to. // ErrNotExists may be returned from the lookup functions of the Node interface // to indicate a missing value. // // Note that schema.ErrNoSuchField is another type of error which sometimes // occurs in similar places as ErrNotExists. ErrNoSuchField is preferred // when handling data with constraints provided by a schema that mean that // a field can *never* exist (as differentiated from a map key which is // simply absent in some data). type ErrNotExists struct { Segment PathSegment } func (e ErrNotExists) Error() string { return fmt.Sprintf("key not found: %q", e.Segment) } // ErrRepeatedMapKey is an error indicating that a key was inserted // into a map that already contains that key. // // This error may be returned by any methods that add data to a map -- // any of the methods on a NodeAssembler that was yielded by MapAssembler.AssignKey(), // or from the MapAssembler.AssignDirectly() method. type ErrRepeatedMapKey struct { Key Node } func (e ErrRepeatedMapKey) Error() string { return fmt.Sprintf("cannot repeat map key %q", e.Key) } // ErrInvalidSegmentForList is returned when using Node.LookupBySegment and the // given PathSegment can't be applied to a list because it's unparsable as a number. type ErrInvalidSegmentForList struct { // TypeName may indicate the named type of a node the function was called on, // or be empty string if working on untyped data. TypeName string // TroubleSegment is the segment we couldn't use. TroubleSegment PathSegment // Reason may explain more about why the PathSegment couldn't be used; // in practice, it's probably a 'strconv.NumError'. Reason error } func (e ErrInvalidSegmentForList) Error() string { v := "invalid segment for lookup on a list" if e.TypeName != "" { v += " of type " + e.TypeName } return v + fmt.Sprintf(": %q: %s", e.TroubleSegment.s, e.Reason) } // ErrIteratorOverread is returned when calling 'Next' on a MapIterator or // ListIterator when it is already done. type ErrIteratorOverread struct{} func (e ErrIteratorOverread) Error() string { return "iterator overread" } ================================================ FILE: datamodel/kind.go ================================================ package datamodel // Kind represents the primitive kind in the IPLD data model. // All of these kinds map directly onto serializable data. // // Note that Kind contains the concept of "map", but not "struct" // or "object" -- those are a concepts that could be introduced in a // type system layers, but are *not* present in the data model layer, // and therefore they aren't included in the Kind enum. type Kind uint8 const ( Kind_Invalid Kind = 0 Kind_Map Kind = '{' Kind_List Kind = '[' Kind_Null Kind = '0' Kind_Bool Kind = 'b' Kind_Int Kind = 'i' Kind_Float Kind = 'f' Kind_String Kind = 's' Kind_Bytes Kind = 'x' Kind_Link Kind = '/' ) func (k Kind) String() string { switch k { case Kind_Invalid: return "INVALID" case Kind_Map: return "map" case Kind_List: return "list" case Kind_Null: return "null" case Kind_Bool: return "bool" case Kind_Int: return "int" case Kind_Float: return "float" case Kind_String: return "string" case Kind_Bytes: return "bytes" case Kind_Link: return "link" default: panic("invalid enumeration value!") } } // KindSet is a type with a few enumerated consts that are commonly used // (mostly, in error messages). type KindSet []Kind var ( KindSet_Recursive = KindSet{Kind_Map, Kind_List} KindSet_Scalar = KindSet{Kind_Null, Kind_Bool, Kind_Int, Kind_Float, Kind_String, Kind_Bytes, Kind_Link} KindSet_JustMap = KindSet{Kind_Map} KindSet_JustList = KindSet{Kind_List} KindSet_JustNull = KindSet{Kind_Null} KindSet_JustBool = KindSet{Kind_Bool} KindSet_JustInt = KindSet{Kind_Int} KindSet_JustFloat = KindSet{Kind_Float} KindSet_JustString = KindSet{Kind_String} KindSet_JustBytes = KindSet{Kind_Bytes} KindSet_JustLink = KindSet{Kind_Link} ) func (x KindSet) String() string { if len(x) == 0 { return "" } s := "" for i := 0; i < len(x)-1; i++ { s += x[i].String() + " or " } s += x[len(x)-1].String() return s } func (x KindSet) Contains(e Kind) bool { for _, v := range x { if v == e { return true } } return false } ================================================ FILE: datamodel/kind_test.go ================================================ package datamodel_test import ( "testing" qt "github.com/frankban/quicktest" "github.com/ipld/go-ipld-prime/datamodel" ) func TestErrWrongKind_String(t *testing.T) { qt.Check(t, datamodel.KindSet{}.String(), qt.Equals, ``) qt.Check(t, datamodel.ErrWrongKind{}.Error(), qt.Equals, `func called on wrong kind: "" called on a INVALID node, but only makes sense on `) } ================================================ FILE: datamodel/link.go ================================================ package datamodel // Link is a special kind of value in IPLD which can be "loaded" to access more nodes. // // Nodes can be a Link: "link" is one of the kinds in the IPLD Data Model; // and accordingly there is an `ipld.Kind_Link` enum value, and Node has an `AsLink` method. // // Links are considered a scalar value in the IPLD Data Model, // but when "loaded", the result can be any other IPLD kind: // maps, lists, strings, etc. // // Link is an interface in the go-ipld implementation, // but the most common instantiation of it comes from the `linking/cid` package, // and represents CIDs (see https://github.com/multiformats/cid). // // The Link interface says very little by itself; it's generally necessary to // use type assertions to unpack more specific forms of data. // The only real contract is that the Link must be able to return a LinkPrototype, // which must be able to produce new Link values of a similar form. // (In practice: if you're familiar with CIDs: Link.Prototype is analogous to cid.Prefix.) // // The traversal package contains powerful features for walking through large graphs of Nodes // while automatically loading and traversing links as the walk goes. // // Note that the Link interface should typically be inhabited by a struct or string, as opposed to a pointer. // This is because Link is often desirable to be able to use as a golang map key, // and in that context, pointers would not result in the desired behavior. type Link interface { // Prototype should return a LinkPrototype which carries the information // to make more Link values similar to this one (but with different hashes). Prototype() LinkPrototype // String should return a reasonably human-readable debug-friendly representation the Link. // There is no contract that requires that the string be able to be parsed back into a Link value, // but the string should be unique (e.g. not elide any parts of the hash). String() string // Binary should return the densest possible encoding of the Link. // The value need not be printable or human-readable; // the golang string type is used for immutability and for ease of use as a map key. // As with the String method, the returned value may not elide any parts of the hash. // // Note that there is still no contract that the returned value should be parsable back into a Link value; // not even in the case of `lnk.Prototype().BuildLink(lnk.Binary()[:])`. // This is because the value returned by this method may contain data that the LinkPrototype would also restate. // (For a concrete example: if using CIDs, this method will return a binary string that includes // the cid version indicator, the multicodec and multihash indicators, etc, in addition to the hash itself -- // whereas the LinkPrototype.BuildLink function still expects to receive only the hash itself alone.) Binary() string } // LinkPrototype encapsulates any implementation details and parameters // necessary for creating a Link, expect for the hash result itself. // // LinkPrototype, like Link, is an interface in go-ipld, // but the most common instantiation of it comes from the `linking/cid` package, // and represents CIDs (see https://github.com/multiformats/cid). // If using CIDs as an implementation, LinkPrototype will encapsulate information // like multihashType, multicodecType, and cidVersion, for example. // (LinkPrototype is analogous to cid.Prefix.) type LinkPrototype interface { // BuildLink should return a new Link value based on the given hashsum. // The hashsum argument should typically be a value returned from a // https://golang.org/pkg/hash/#Hash.Sum call. // // The hashsum reference must not be retained (the caller is free to reuse it). BuildLink(hashsum []byte) Link } ================================================ FILE: datamodel/node.go ================================================ package datamodel import "io" // Node represents a value in IPLD. Any point in a tree of data is a node: // scalar values (like int64, string, etc) are nodes, and // so are recursive values (like map and list). // // Nodes and kinds are described in the IPLD specs at // https://github.com/ipld/specs/blob/master/data-model-layer/data-model.md . // // Methods on the Node interface cover the superset of all possible methods for // all possible kinds -- but some methods only make sense for particular kinds, // and thus will only make sense to call on values of the appropriate kind. // (For example, 'Length' on an integer doesn't make sense, // and 'AsInt' on a map certainly doesn't work either!) // Use the Kind method to find out the kind of value before // calling kind-specific methods. // Individual method documentation state which kinds the method is valid for. // (If you're familiar with the stdlib reflect package, you'll find // the design of the Node interface very comparable to 'reflect.Value'.) // // The Node interface is read-only. All of the methods on the interface are // for examining values, and implementations should be immutable. // The companion interface, NodeBuilder, provides the matching writable // methods, and should be use to create a (thence immutable) Node. // // Keeping Node immutable and separating mutation into NodeBuilder makes // it possible to perform caching (or rather, memoization, since there's no // such thing as cache invalidation for immutable systems) of computed // properties of Node; use copy-on-write algorithms for memory efficiency; // and to generally build pleasant APIs. // Many library functions will rely on the immutability of Node (e.g., // assuming that pointer-equal nodes do not change in value over time), // so any user-defined Node implementations should be careful to uphold // the immutability contract.) // // There are many different concrete types which implement Node. // The primary purpose of various node implementations is to organize // memory in the program in different ways -- some in-memory layouts may // be more optimal for some programs than others, and changing the Node // (and NodeBuilder) implementations lets the programmer choose. // // For concrete implementations of Node, check out the "./node/" folder, // and the packages within it. // "node/basicnode" should probably be your first start; the Node and NodeBuilder // implementations in that package work for any data. // Other packages are optimized for specific use-cases. // Codegen tools can also be used to produce concrete implementations of Node; // these may be specific to certain data, but still conform to the Node // interface for interoperability and to support higher-level functions. // // Nodes may also be *typed* -- see the 'schema' package and `schema.TypedNode` // interface, which extends the Node interface with additional methods. // Typed nodes have additional constraints and behaviors: // for example, they may be a "struct" and have a specific type/structure // to what data you can put inside them, but still behave as a regular Node // in all ways this interface specifies (so you can traverse typed nodes, etc, // without any additional special effort). type Node interface { // Kind returns a value from the Kind enum describing what the // essential serializable kind of this node is (map, list, integer, etc). // Most other handling of a node requires first switching upon the kind. Kind() Kind // LookupByString looks up a child object in this node and returns it. // The returned Node may be any of the Kind: // a primitive (string, int64, etc), a map, a list, or a link. // // If the Kind of this Node is not Kind_Map, a nil node and an error // will be returned. // // If the key does not exist, a nil node and an error will be returned. LookupByString(key string) (Node, error) // LookupByNode is the equivalent of LookupByString, but takes a reified Node // as a parameter instead of a plain string. // This mechanism is useful if working with typed maps (if the key types // have constraints, and you already have a reified `schema.TypedNode` value, // using that value can save parsing and validation costs); // and may simply be convenient if you already have a Node value in hand. // // (When writing generic functions over Node, a good rule of thumb is: // when handling a map, check for `schema.TypedNode`, and in this case prefer // the LookupByNode(Node) method; otherwise, favor LookupByString; typically // implementations will have their fastest paths thusly.) LookupByNode(key Node) (Node, error) // LookupByIndex is the equivalent of LookupByString but for indexing into a list. // As with LookupByString, the returned Node may be any of the Kind: // a primitive (string, int64, etc), a map, a list, or a link. // // If the Kind of this Node is not Kind_List, a nil node and an error // will be returned. // // If idx is out of range, a nil node and an error will be returned. LookupByIndex(idx int64) (Node, error) // LookupBySegment is will act as either LookupByString or LookupByIndex, // whichever is contextually appropriate. // // Using LookupBySegment may imply an "atoi" conversion if used on a list node, // or an "itoa" conversion if used on a map node. If an "itoa" conversion // takes place, it may error, and this method may return that error. LookupBySegment(seg PathSegment) (Node, error) // Note that when using codegenerated types, there may be a fifth variant // of lookup method on maps: `Get($GeneratedTypeKey) $GeneratedTypeValue`! // MapIterator returns an iterator which yields key-value pairs // traversing the node. // If the node kind is anything other than a map, nil will be returned. // // The iterator will yield every entry in the map; that is, it // can be expected that itr.Next will be called node.Length times // before itr.Done becomes true. MapIterator() MapIterator // ListIterator returns an iterator which traverses the node and yields indicies and list entries. // If the node kind is anything other than a list, nil will be returned. // // The iterator will yield every entry in the list; that is, it // can be expected that itr.Next will be called node.Length times // before itr.Done becomes true. // // List iteration is ordered, and indices yielded during iteration will range from 0 to Node.Length-1. // (The IPLD Data Model definition of lists only defines that it is an ordered list of elements; // the definition does not include a concept of sparseness, so the indices are always sequential.) ListIterator() ListIterator // Length returns the length of a list, or the number of entries in a map, // or -1 if the node is not of list nor map kind. Length() int64 // Absent nodes are returned when traversing a struct field that is // defined by a schema but unset in the data. (Absent nodes are not // possible otherwise; you'll only see them from `schema.TypedNode`.) // The absent flag is necessary so iterating over structs can // unambiguously make the distinction between values that are // present-and-null versus values that are absent. // // Absent nodes respond to `Kind()` as `ipld.Kind_Null`, // for lack of any better descriptive value; you should therefore // always check IsAbsent rather than just a switch on kind // when it may be important to handle absent values distinctly. IsAbsent() bool IsNull() bool AsBool() (bool, error) AsInt() (int64, error) AsFloat() (float64, error) AsString() (string, error) AsBytes() ([]byte, error) AsLink() (Link, error) // Prototype returns a NodePrototype which can describe some properties of this node's implementation, // and also be used to get a NodeBuilder, // which can be use to create new nodes with the same implementation as this one. // // For typed nodes, the NodePrototype will also implement schema.Type. // // For Advanced Data Layouts, the NodePrototype will encapsulate any additional // parameters and configuration of the ADL, and will also (usually) // implement NodePrototypeSupportingAmend. // // Calling this method should not cause an allocation. Prototype() NodePrototype } // UintNode is an optional interface that can be used to represent an Int node // that provides access to the full uint64 range. // // EXPERIMENTAL: this API is experimental and may be changed or removed in a // future use. A future iteration may replace this with a BigInt interface to // access a larger range of integers that may be enabled by alternative codecs. type UintNode interface { Node // AsUint returns a uint64 representing the underlying integer if possible. // This may return an error if the Node represents a negative integer that // cannot be represented as a uint64. AsUint() (uint64, error) } // LargeBytesNode is an optional interface extending a Bytes node that allows its // contents to be accessed through an io.ReadSeeker instead of a []byte slice. Use of // an io.Reader is encouraged, as it allows for streaming large byte slices // without allocating a large slice in memory. type LargeBytesNode interface { Node // AsLargeBytes returns an io.ReadSeeker that can be used to read the contents of the node. // Note that the presence of this method / interface does not imply that the node // can always return a valid io.ReadSeeker, and the error value must also be checked // for support. // It is not guaranteed that all implementations will implement the full semantics of // Seek, in particular, they may refuse to seek to the end of a large bytes node if // it is not possible to do so efficiently. // The io.ReadSeeker returned by AsLargeBytes must be a separate instance from subsequent // calls to AsLargeBytes. Calls to read or seek on one returned instance should NOT // affect the read position of other returned instances. AsLargeBytes() (io.ReadSeeker, error) } // NodePrototype describes a node implementation (all Node have a NodePrototype), // and a NodePrototype can always be used to get a NodeBuilder. // // A NodePrototype may also provide other information about implementation; // such information is specific to this library ("prototype" isn't a concept // you'll find in the IPLD Specifications), and is usually provided through // feature-detection interfaces (for example, see NodePrototypeSupportingAmend). // // Generic algorithms for working with IPLD Nodes make use of NodePrototype // to get builders for new nodes when creating data, and can also use the // feature-detection interfaces to help decide what kind of operations // will be optimal to use on a given node implementation. // // Note that NodePrototype is not the same as schema.Type. // NodePrototype is a (golang-specific!) way to reflect upon the implementation // and in-memory layout of some IPLD data. // schema.Type is information about how a group of nodes is related in a schema // (if they have one!) and the rules that the type mandates the node must follow. // (Every node must have a prototype; but schema types are an optional feature.) type NodePrototype interface { // NewBuilder returns a NodeBuilder that can be used to create a new Node. // // Note that calling NewBuilder often performs an allocation // (while in contrast, getting a NodePrototype typically does not!) -- // this may be consequential when writing high performance code. NewBuilder() NodeBuilder } // NodePrototypeSupportingAmend is a feature-detection interface that can be // used on a NodePrototype to see if it's possible to build new nodes of this style // while sharing some internal data in a copy-on-write way. // // For example, Nodes using an Advanced Data Layout will typically // support this behavior, and since ADLs are often used for handling large // volumes of data, detecting and using this feature can result in significant // performance savings. type NodePrototypeSupportingAmend interface { AmendingBuilder(base Node) NodeBuilder // FUTURE: probably also needs a `AmendingWithout(base Node, filter func(k,v) bool) NodeBuilder`, or similar. // ("deletion" based APIs are also possible but both more complicated in interfaces added, and prone to accidentally quadratic usage.) // FUTURE: there should be some stdlib `Copy` (?) methods that automatically look for this feature, and fallback if absent. // Might include a wide range of point `Transform`, etc, methods. // FUTURE: consider putting this (and others like it) in a `feature` package, if there begin to be enough of them and docs get crowded. } // MapIterator is an interface for traversing map nodes. // Sequential calls to Next() will yield key-value pairs; // Done() describes whether iteration should continue. // // Iteration order is defined to be stable: two separate MapIterator // created to iterate the same Node will yield the same key-value pairs // in the same order. // The order itself may be defined by the Node implementation: some // Nodes may retain insertion order, and some may return iterators which // always yield data in sorted order, for example. type MapIterator interface { // Next returns the next key-value pair. // // An error value can also be returned at any step: in the case of advanced // data structures with incremental loading, it's possible to encounter // cancellation or I/O errors at any point in iteration. // If an error will be returned by the next call to Next, // then the boolean returned by the Done method will be false // (meaning it's acceptable to check Done first and move on if it's true, // since that both means the iterator is complete and that there is no error). // If an error is returned, the key and value may be nil. Next() (key Node, value Node, err error) // Done returns false as long as there's at least one more entry to iterate. // When Done returns true, iteration can stop. // // Note when implementing iterators for advanced data layouts (e.g. more than // one chunk of backing data, which is loaded incrementally): if your // implementation does any I/O during the Done method, and it encounters // an error, it must return 'false', so that the following Next call // has an opportunity to return the error. Done() bool } // ListIterator is an interface for traversing list nodes. // Sequential calls to Next() will yield index-value pairs; // Done() describes whether iteration should continue. // // ListIterator's Next method returns an index for convenience, // but this number will always start at 0 and increment by 1 monotonically. // A loop which iterates from 0 to Node.Length while calling Node.LookupByIndex // is equivalent to using a ListIterator. type ListIterator interface { // Next returns the next index and value. // // An error value can also be returned at any step: in the case of advanced // data structures with incremental loading, it's possible to encounter // cancellation or I/O errors at any point in iteration. // If an error will be returned by the next call to Next, // then the boolean returned by the Done method will be false // (meaning it's acceptable to check Done first and move on if it's true, // since that both means the iterator is complete and that there is no error). // If an error is returned, the key and value may be nil. Next() (idx int64, value Node, err error) // Done returns false as long as there's at least one more entry to iterate. // When Done returns false, iteration can stop. // // Note when implementing iterators for advanced data layouts (e.g. more than // one chunk of backing data, which is loaded incrementally): if your // implementation does any I/O during the Done method, and it encounters // an error, it must return 'false', so that the following Next call // has an opportunity to return the error. Done() bool } // REVIEW: immediate-mode AsBytes() method (as opposed to e.g. returning // an io.Reader instance) might be problematic, esp. if we introduce // AdvancedLayouts which support large bytes natively. // // Probable solution is having both immediate and iterator return methods. // Returning a reader for bytes when you know you want a slice already // is going to be high friction without purpose in many common uses. // // Unclear what SetByteStream() would look like for advanced layouts. // One could try to encapsulate the chunking entirely within the advlay // node impl... but would it be graceful? Not sure. Maybe. Hopefully! // Yes? The advlay impl would still tend to use SetBytes for the raw // data model layer nodes its composing, so overall, it shakes out nicely. ================================================ FILE: datamodel/nodeBuilder.go ================================================ package datamodel // NodeAssembler is the interface that describes all the ways we can set values // in a node that's under construction. // // A NodeAssembler is about filling in data. // To create a new Node, you should start with a NodeBuilder (which contains a // superset of the NodeAssembler methods, and can return the finished Node // from its `Build` method). // While continuing to build a recursive structure from there, // you'll see NodeAssembler for all the child values. // // For filling scalar data, there's a `Assign{Kind}` method for each kind; // after calling one of these methods, the data is filled in, and the assembler is done. // For recursives, there are `BeginMap` and `BeginList` methods, // which return an object that needs further manipulation to fill in the contents. // // There is also one special method: `AssignNode`. // `AssignNode` takes another `Node` as a parameter, // and should should internally call one of the other `Assign*` or `Begin*` (and subsequent) functions // as appropriate for the kind of the `Node` it is given. // This is roughly equivalent to using the `Copy` function (and is often implemented using it!), but // `AssignNode` may also try to take faster shortcuts in some implementations, when it detects they're possible. // (For example, for typed nodes, if they're the same type, lots of checking can be skipped. // For nodes implemented with pointers, lots of copying can be skipped. // For nodes that can detect the argument has the same memory layout, faster copy mechanisms can be used; etc.) // // Why do both this and the NodeBuilder interface exist? // In short: NodeBuilder is when you want to cause an allocation; // NodeAssembler can be used to just "fill in" memory. // (In the internal gritty details: separate interfaces, one of which lacks a // `Build` method, helps us write efficient library internals: avoiding the // requirement to be able to return a Node at any random point in the process // relieves internals from needing to implement 'freeze' features. // This is useful in turn because implementing those 'freeze' features in a // language without first-class/compile-time support for them (as golang is) // would tend to push complexity and costs to execution time; we'd rather not.) type NodeAssembler interface { BeginMap(sizeHint int64) (MapAssembler, error) BeginList(sizeHint int64) (ListAssembler, error) AssignNull() error AssignBool(bool) error AssignInt(int64) error AssignFloat(float64) error AssignString(string) error AssignBytes([]byte) error AssignLink(Link) error AssignNode(Node) error // if you already have a completely constructed subtree, this method puts the whole thing in place at once. // Prototype returns a NodePrototype describing what kind of value we're assembling. // // You often don't need this (because you should be able to // just feed data and check errors), but it's here. // // Using `this.Prototype().NewBuilder()` to produce a new `Node`, // then giving that node to `this.AssignNode(n)` should always work. // (Note that this is not necessarily an _exclusive_ statement on what // sort of values will be accepted by `this.AssignNode(n)`.) Prototype() NodePrototype } // MapAssembler assembles a map node! (You guessed it.) // // Methods on MapAssembler must be called in a valid order: // assemble a key, then assemble a value, then loop as long as desired; // when finished, call 'Finish'. // // Incorrect order invocations will panic. // Calling AssembleKey twice in a row will panic; // calling AssembleValue before finishing using the NodeAssembler from AssembleKey will panic; // calling AssembleValue twice in a row will panic; // etc. // // Note that the NodeAssembler yielded from AssembleKey has additional behavior: // if the node assembled there matches a key already present in the map, // that assembler will emit the error! type MapAssembler interface { AssembleKey() NodeAssembler // must be followed by call to AssembleValue. AssembleValue() NodeAssembler // must be called immediately after AssembleKey. AssembleEntry(k string) (NodeAssembler, error) // shortcut combining AssembleKey and AssembleValue into one step; valid when the key is a string kind. Finish() error // KeyPrototype returns a NodePrototype that knows how to build keys of a type this map uses. // // You often don't need this (because you should be able to // just feed data and check errors), but it's here. // // For all Data Model maps, this will answer with a basic concept of "string". // For Schema typed maps, this may answer with a more complex type // (potentially even a struct type or union type -- anything that can have a string representation). KeyPrototype() NodePrototype // ValuePrototype returns a NodePrototype that knows how to build values this map can contain. // // You often don't need this (because you should be able to // just feed data and check errors), but it's here. // // ValuePrototype requires a parameter describing the key in order to say what // NodePrototype will be acceptable as a value for that key, because when using // struct types (or union types) from the Schemas system, they behave as maps // but have different acceptable types for each field (or member, for unions). // For plain maps (that is, not structs or unions masquerading as maps), // the empty string can be used as a parameter, and the returned NodePrototype // can be assumed applicable for all values. // Using an empty string for a struct or union will return nil, // as will using any string which isn't a field or member of those types. // // (Design note: a string is sufficient for the parameter here rather than // a full Node, because the only cases where the value types vary are also // cases where the keys may not be complex.) ValuePrototype(k string) NodePrototype } type ListAssembler interface { AssembleValue() NodeAssembler Finish() error // ValuePrototype returns a NodePrototype that knows how to build values this map can contain. // // You often don't need this (because you should be able to // just feed data and check errors), but it's here. // // ValuePrototype, much like the matching method on the MapAssembler interface, // requires a parameter specifying the index in the list in order to say // what NodePrototype will be acceptable as a value at that position. // For many lists (and *all* lists which operate exclusively at the Data Model level), // this will return the same NodePrototype regardless of the value of 'idx'; // the only time this value will vary is when operating with a Schema, // and handling the representation NodeAssembler for a struct type with // a representation of a list kind. // If you know you are operating in a situation that won't have varying // NodePrototypes, it is acceptable to call `ValuePrototype(0)` and use the // resulting NodePrototype for all reasoning. ValuePrototype(idx int64) NodePrototype } type NodeBuilder interface { NodeAssembler // Build returns the new value after all other assembly has been completed. // // A method on the NodeAssembler that finishes assembly of the data must // be called first (e.g., any of the "Assign*" methods, or "Finish" if // the assembly was for a map or a list); that finishing method still has // all responsibility for validating the assembled data and returning // any errors from that process. // (Correspondingly, there is no error return from this method.) // // Note that building via a representation-level NodePrototype or NodeBuilder // returns a node at the type level which implements schema.TypedNode. // To obtain the representation-level node, you can do: // // // builder is at the representation level, so it returns typed nodes // node := builder.Build().(schema.TypedNode) // reprNode := node.Representation() Build() Node // Resets the builder. It can hereafter be used again. // Reusing a NodeBuilder can reduce allocations and improve performance. // // Only call this if you're going to reuse the builder. // (Otherwise, it's unnecessary, and may cause an unwanted allocation). Reset() } ================================================ FILE: datamodel/path.go ================================================ package datamodel import ( "strings" ) // Path describes a series of steps across a tree or DAG of Node, // where each segment in the path is a map key or list index // (literaly, Path is a slice of PathSegment values). // Path is used in describing progress in a traversal; and // can also be used as an instruction for traversing from one Node to another. // Path values will also often be encountered as part of error messages. // // (Note that Paths are useful as an instruction for traversing from // *one* Node to *one* other Node; to do a walk from one Node and visit // *several* Nodes based on some sort of pattern, look to IPLD Selectors, // and the 'traversal/selector' package in this project.) // // Path values are always relative. // Observe how 'traversal.Focus' requires both a Node and a Path argument -- // where to start, and where to go, respectively. // Similarly, error values which include a Path will be speaking in reference // to the "starting Node" in whatever context they arose from. // // The canonical form of a Path is as a list of PathSegment. // Each PathSegment is a string; by convention, the string should be // in UTF-8 encoding and use NFC normalization, but all operations // will regard the string as its constituent eight-bit bytes. // // There are no illegal or magical characters in IPLD Paths // (in particular, do not mistake them for UNIX system paths). // IPLD Paths can only go down: that is, each segment must traverse one node. // There is no ".." which means "go up"; // and there is no "." which means "stay here". // IPLD Paths have no magic behavior around characters such as "~". // IPLD Paths do not have a concept of "globs" nor behave specially // for a path segment string of "*" (but you may wish to see 'Selectors' // for globbing-like features that traverse over IPLD data). // // An empty string is a valid PathSegment. // (This leads to some unfortunate complications when wishing to represent // paths in a simple string format; however, consider that maps do exist // in serialized data in the wild where an empty string is used as the key: // it is important we be able to correctly describe and address this!) // // A string containing "/" (or even being simply "/"!) is a valid PathSegment. // (As with empty strings, this is unfortunate (in particular, because it // very much doesn't match up well with expectations popularized by UNIX-like // filesystems); but, as with empty strings, maps which contain such a key // certainly exist, and it is important that we be able to regard them!) // // A string starting, ending, or otherwise containing the NUL (\x00) byte // is also a valid PathSegment. This follows from the rule of "a string is // regarded as its constituent eight-bit bytes": an all-zero byte is not exceptional. // In golang, this doesn't pose particular difficulty, but note this would be // of marked concern for languages which have "C-style nul-terminated strings". // // For an IPLD Path to be represented as a string, an encoding system // including escaping is necessary. At present, there is not a single // canonical specification for such an escaping; we expect to decide one // in the future, but this is not yet settled and done. // (This implementation has a 'String' method, but it contains caveats // and may be ambiguous for some content. This may be fixed in the future.) type Path struct { segments []PathSegment } // EmptyPath is the Path with no segments. var EmptyPath = Path{} // NewPath returns a Path composed of the given segments. // // This constructor function does a defensive copy, // in case your segments slice should mutate in the future. // (Use NewPathNocopy if this is a performance concern, // and you're sure you know what you're doing.) func NewPath(segments []PathSegment) Path { p := Path{make([]PathSegment, len(segments))} copy(p.segments, segments) return p } // NewPathNocopy is identical to NewPath but trusts that // the segments slice you provide will not be mutated. func NewPathNocopy(segments []PathSegment) Path { return Path{segments} } // ParsePath converts a string to an IPLD Path, doing a basic parsing of the // string using "/" as a delimiter to produce a segmented Path. // This is a handy, but not a general-purpose nor spec-compliant (!), // way to create a Path: it cannot represent all valid paths. // // Multiple subsequent "/" characters will be silently collapsed. // E.g., `"foo///bar"` will be treated equivalently to `"foo/bar"`. // Prefixed and suffixed extraneous "/" characters are also discarded. // This makes this constructor incapable of handling some possible Path values // (specifically: paths with empty segments cannot be created with this constructor). // // There is no escaping mechanism used by this function. // This makes this constructor incapable of handling some possible Path values // (specifically, a path segment containing "/" cannot be created, because it // will always be interpreted as a segment separator). // // No other "cleaning" of the path occurs. See the documentation of the Path struct; // in particular, note that ".." does not mean "go up", nor does "." mean "stay here" -- // correspondingly, there isn't anything to "clean" in the same sense as // 'filepath.Clean' from the standard library filesystem path packages would. // // If the provided string contains unprintable characters, or non-UTF-8 // or non-NFC-canonicalized bytes, no remark will be made about this, // and those bytes will remain part of the PathSegments in the resulting Path. func ParsePath(pth string) Path { // FUTURE: we should probably have some escaping mechanism which makes // it possible to encode a slash in a segment. Specification needed. ss := strings.FieldsFunc(pth, func(r rune) bool { return r == '/' }) ssl := len(ss) p := Path{make([]PathSegment, ssl)} for i := 0; i < ssl; i++ { p.segments[i] = PathSegmentOfString(ss[i]) } return p } // String representation of a Path is simply the join of each segment with '/'. // It does not include a leading nor trailing slash. // // This is a handy, but not a general-purpose nor spec-compliant (!), // way to reduce a Path to a string. // There is no escaping mechanism used by this function, // and as a result, not all possible valid Path values (such as those with // empty segments or with segments containing "/") can be encoded unambiguously. // For Path values containing these problematic segments, ParsePath applied // to the string returned from this function may return a nonequal Path value. // // No escaping for unprintable characters is provided. // No guarantee that the resulting string is UTF-8 nor NFC canonicalized // is provided unless all the constituent PathSegment had those properties. func (p Path) String() string { l := len(p.segments) if l == 0 { return "" } sb := strings.Builder{} for i := 0; i < l-1; i++ { sb.WriteString(p.segments[i].String()) sb.WriteByte('/') } sb.WriteString(p.segments[l-1].String()) return sb.String() } // Segments returns a slice of the path segment strings. // // It is not lawful to mutate nor append the returned slice. func (p Path) Segments() []PathSegment { return p.segments } // Len returns the number of segments in this path. // // Zero segments means the path refers to "the current node". // One segment means it refers to a child of the current node; etc. func (p Path) Len() int { return len(p.segments) } // Join creates a new path composed of the concatenation of this and the given path's segments. func (p Path) Join(p2 Path) Path { combinedSegments := make([]PathSegment, len(p.segments)+len(p2.segments)) copy(combinedSegments, p.segments) copy(combinedSegments[len(p.segments):], p2.segments) p.segments = combinedSegments return p } // AppendSegment is as per Join, but a shortcut when appending single segments. func (p Path) AppendSegment(ps PathSegment) Path { l := len(p.segments) combinedSegments := make([]PathSegment, l+1) copy(combinedSegments, p.segments) combinedSegments[l] = ps p.segments = combinedSegments return p } // AppendSegmentString is as per AppendSegment, but a shortcut when the segment is a string. func (p Path) AppendSegmentString(ps string) Path { return p.AppendSegment(PathSegmentOfString(ps)) } // AppendSegmentInt is as per AppendSegment, but a shortcut when the segment is an int. func (p Path) AppendSegmentInt(ps int64) Path { return p.AppendSegment(PathSegmentOfInt(ps)) } // Parent returns a path with the last of its segments popped off (or // the zero path if it's already empty). func (p Path) Parent() Path { if len(p.segments) == 0 { return EmptyPath } return Path{p.segments[0 : len(p.segments)-1]} } // Truncate returns a path with only as many segments remaining as requested. func (p Path) Truncate(i int) Path { return Path{p.segments[0:i]} } // Last returns the trailing segment of the path. Path length should be checked before making use of this value. // If the path is empty, EmptyPathSegment is returned which may not be useful. func (p Path) Last() PathSegment { if len(p.segments) < 1 { return EmptyPathSegment } return p.segments[len(p.segments)-1] } // Pop returns a path with all segments except the last. func (p Path) Pop() Path { if len(p.segments) < 1 { return EmptyPath } return Path{p.segments[0 : len(p.segments)-1]} } // Shift returns the first segment of the path together with the remaining path after that first segment. // If you intend to use the resulting PathSegment, you should check its length before doing so. // If applied to a zero-length path, it returns EmptyPathSegment and the same zero-length path. func (p Path) Shift() (PathSegment, Path) { if len(p.segments) < 1 { return EmptyPathSegment, EmptyPath } return p.segments[0], Path{p.segments[1:]} } ================================================ FILE: datamodel/pathSegment.go ================================================ package datamodel import ( "strconv" ) // PathSegment can describe either a key in a map, or an index in a list. // // Create a PathSegment via either ParsePathSegment, PathSegmentOfString, // or PathSegmentOfInt; or, via one of the constructors of Path, // which will implicitly create PathSegment internally. // Using PathSegment's natural zero value directly is discouraged // (it will act like ParsePathSegment("0"), which likely not what you'd expect). // // Path segments are "stringly typed" -- they may be interpreted as either strings or ints depending on context. // A path segment of "123" will be used as a string when traversing a node of map kind; // and it will be converted to an integer when traversing a node of list kind. // (If a path segment string cannot be parsed to an int when traversing a node of list kind, then traversal will error.) // It is not possible to ask which kind (string or integer) a PathSegment is, because that is not defined -- this is *only* interpreted contextually. // // Internally, PathSegment will store either a string or an integer, // depending on how it was constructed, // and will automatically convert to the other on request. // (This means if two pieces of code communicate using PathSegment, // one producing ints and the other expecting ints, // then they will work together efficiently.) // PathSegment in a Path produced by ParsePath generally have all strings internally, // because there is no distinction possible when parsing a Path string // (and attempting to pre-parse all strings into ints "just in case" would waste time in almost all cases). // // Be cautious of attempting to use PathSegment as a map key! // Due to the implementation detail of internal storage, it's possible for // PathSegment values which are "equal" per PathSegment.Equal's definition // to still be unequal in the eyes of golang's native maps. // You should probably use the string values of the PathSegment as map keys. // (This has the additional bonus of hitting a special fastpath that the golang // built-in maps have specifically for plain string keys.) type PathSegment struct { /* A quick implementation note about the Go compiler and "union" semantics: There are roughly two ways to do "union" semantics in Go. The first is to make a struct with each of the values. The second is to make an interface and use an unexported method to keep it closed. The second tactic provides somewhat nicer semantics to the programmer. (Namely, it's clearly impossible to have two inhabitants, which is... the point.) The downside is... putting things in interfaces generally incurs an allocation (grep your assembly output for "runtime.conv*"). The first tactic looks kludgier, and would seem to waste memory (the struct reserves space for each possible value, even though the semantic is that only one may be non-zero). However, in most cases, more *bytes* are cheaper than more *allocs* -- garbage collection costs are domininated by alloc count, not alloc size. Because PathSegment is something we expect to put in fairly "hot" paths, we're using the first tactic. (We also currently get away with having no extra discriminator bit because we use a signed int for indexes, and negative values aren't valid there, and thus we can use it as a sentinel value. (Fun note: Empty strings were originally used for this sentinel, but it turns out empty strings are valid PathSegment themselves, so!)) */ s string i int64 } // EmptyPathSegment is the PathSegment with no value. This is not the same as // the zero value of PathSegment (PathSegment{}), which will be interpreted to // mean "0", as a list index. EmptyPathSegment is equivalent to // ParsePathSegment("") or PathSegmentOfString(""). var EmptyPathSegment = PathSegment{i: -1} // ParsePathSegment parses a string into a PathSegment, // handling any escaping if present. // (Note: there is currently no escaping specified for PathSegments, // so this is currently functionally equivalent to PathSegmentOfString.) func ParsePathSegment(s string) PathSegment { return PathSegment{s: s, i: -1} } // PathSegmentOfString boxes a string into a PathSegment. // It does not attempt to parse any escaping; use ParsePathSegment for that. func PathSegmentOfString(s string) PathSegment { return PathSegment{s: s, i: -1} } // PathSegmentOfString boxes an int into a PathSegment. func PathSegmentOfInt(i int64) PathSegment { return PathSegment{i: i} } // containsString is unexported because we use it to see what our *storage* form is, // but this is considered an implementation detail that's non-semantic. // If it returns false, it implicitly means "containsInt", as these are the only options. func (ps PathSegment) containsString() bool { return ps.i < 0 } // String returns the PathSegment as a string. func (ps PathSegment) String() string { switch ps.containsString() { case true: return ps.s case false: return strconv.FormatInt(ps.i, 10) } panic("unreachable") } // Index returns the PathSegment as an integer, // or returns an error if the segment is a string that can't be parsed as an int. func (ps PathSegment) Index() (int64, error) { switch ps.containsString() { case true: return strconv.ParseInt(ps.s, 10, 64) case false: return ps.i, nil } panic("unreachable") } // Equals checks if two PathSegment values are equal. // // Because PathSegment is "stringly typed", this comparison does not // regard if one of the segments is stored as a string and one is stored as an int; // if string values of two segments are equal, they are "equal" overall. // In other words, `PathSegmentOfInt(2).Equals(PathSegmentOfString("2")) == true`! // (You should still typically prefer this method over converting two segments // to string and comparing those, because even though that may be functionally // correct, this method will be faster if they're both ints internally.) func (x PathSegment) Equals(o PathSegment) bool { if !x.containsString() && !o.containsString() { return x.i == o.i } return x.String() == o.String() } ================================================ FILE: datamodel/path_test.go ================================================ package datamodel import ( "reflect" "testing" qt "github.com/frankban/quicktest" "github.com/google/go-cmp/cmp" ) func TestParsePath(t *testing.T) { // Allow equality checker to check all unexported fields in PathSegment. pathSegmentEquals := qt.CmpEquals(cmp.Exporter(func(reflect.Type) bool { return true })) t.Run("parsing one segment", func(t *testing.T) { qt.Check(t, ParsePath("0").segments, pathSegmentEquals, []PathSegment{{s: "0", i: -1}}) }) t.Run("parsing three segments", func(t *testing.T) { qt.Check(t, ParsePath("0/foo/2").segments, pathSegmentEquals, []PathSegment{{s: "0", i: -1}, {s: "foo", i: -1}, {s: "2", i: -1}}) }) t.Run("eliding leading slashes", func(t *testing.T) { qt.Check(t, ParsePath("/0/2").segments, pathSegmentEquals, []PathSegment{{s: "0", i: -1}, {s: "2", i: -1}}) }) t.Run("eliding trailing", func(t *testing.T) { qt.Check(t, ParsePath("0/2/").segments, pathSegmentEquals, []PathSegment{{s: "0", i: -1}, {s: "2", i: -1}}) }) t.Run("eliding empty segments", func(t *testing.T) { // NOTE: a spec for string encoding might cause this to change in the future! qt.Check(t, ParsePath("0//2").segments, pathSegmentEquals, []PathSegment{{s: "0", i: -1}, {s: "2", i: -1}}) }) t.Run("escaping segments", func(t *testing.T) { // NOTE: a spec for string encoding might cause this to change in the future! qt.Check(t, ParsePath(`0/\//2`).segments, pathSegmentEquals, []PathSegment{{s: "0", i: -1}, {s: `\`, i: -1}, {s: "2", i: -1}}) }) } func TestPathSegmentZeroValue(t *testing.T) { qt.Check(t, PathSegment{}.String(), qt.Equals, "0") i, err := PathSegment{}.Index() qt.Check(t, err, qt.IsNil) qt.Check(t, i, qt.Equals, int64(0)) qt.Check(t, EmptyPathSegment.String(), qt.Equals, "") _, err = EmptyPathSegment.Index() qt.Check(t, err, qt.IsNotNil) } ================================================ FILE: datamodel/unit.go ================================================ package datamodel // Null is the one kind of node we can have a true singleton implementation of. // This is that value. // The Null Node has Kind() == Kind_Null, returns IsNull() == true, // returns ErrWrongKind to most other inquiries (as you'd expect), // and returns a NodePrototype with a NewBuilder method that simply panics // (because why would you ever try to build a new "null"?). var Null Node = nullNode{} type nullNode struct{} func (nullNode) Kind() Kind { return Kind_Null } func (nullNode) LookupByString(key string) (Node, error) { return nil, ErrWrongKind{TypeName: "null", MethodName: "LookupByString", AppropriateKind: KindSet_JustMap, ActualKind: Kind_Null} } func (nullNode) LookupByNode(key Node) (Node, error) { return nil, ErrWrongKind{TypeName: "null", MethodName: "LookupByNode", AppropriateKind: KindSet_JustMap, ActualKind: Kind_Null} } func (nullNode) LookupByIndex(idx int64) (Node, error) { return nil, ErrWrongKind{TypeName: "null", MethodName: "LookupByIndex", AppropriateKind: KindSet_JustList, ActualKind: Kind_Null} } func (nullNode) LookupBySegment(seg PathSegment) (Node, error) { return nil, ErrWrongKind{TypeName: "null", MethodName: "LookupBySegment", AppropriateKind: KindSet_Recursive, ActualKind: Kind_Null} } func (nullNode) MapIterator() MapIterator { return nil } func (nullNode) ListIterator() ListIterator { return nil } func (nullNode) Length() int64 { return -1 } func (nullNode) IsAbsent() bool { return false } func (nullNode) IsNull() bool { return true } func (nullNode) AsBool() (bool, error) { return false, ErrWrongKind{TypeName: "null", MethodName: "AsBool", AppropriateKind: KindSet_JustBool, ActualKind: Kind_Null} } func (nullNode) AsInt() (int64, error) { return 0, ErrWrongKind{TypeName: "null", MethodName: "AsInt", AppropriateKind: KindSet_JustInt, ActualKind: Kind_Null} } func (nullNode) AsFloat() (float64, error) { return 0, ErrWrongKind{TypeName: "null", MethodName: "AsFloat", AppropriateKind: KindSet_JustFloat, ActualKind: Kind_Null} } func (nullNode) AsString() (string, error) { return "", ErrWrongKind{TypeName: "null", MethodName: "AsString", AppropriateKind: KindSet_JustString, ActualKind: Kind_Null} } func (nullNode) AsBytes() ([]byte, error) { return nil, ErrWrongKind{TypeName: "null", MethodName: "AsBytes", AppropriateKind: KindSet_JustBytes, ActualKind: Kind_Null} } func (nullNode) AsLink() (Link, error) { return nil, ErrWrongKind{TypeName: "null", MethodName: "AsLink", AppropriateKind: KindSet_JustLink, ActualKind: Kind_Null} } func (nullNode) Prototype() NodePrototype { return nullPrototype{} } type nullPrototype struct{} func (nullPrototype) NewBuilder() NodeBuilder { panic("cannot build null nodes") } // Absent is the _other_ kind of node (besides Null) we can have a true singleton implementation of. // This is the singleton value for Absent. // The Absent Node has Kind() == Kind_Null, returns IsNull() == false (!), // returns IsAbsent() == true, // returns ErrWrongKind to most other inquiries (as you'd expect), // and returns a NodePrototype with a NewBuilder method that simply panics // (because why would you ever try to build a new "nothing"?). // // Despite its presence in the datamodel package, "absent" is not really a data model concept. // Absent should not really be seen in any datamodel Node implementations, for example. // Absent is seen used in the realm of schemas and typed data, because there, // there's a concept of data that has been described, and yet is not currently present; // it is this concept that "absent" is used to express. // Absent also sometimes shows up as a distinct case in codecs or other diagnostic printing, // and suchlike miscellaneous places; it is for these reasons that it's here in the datamodel package, // because it would end up imported somewhat universally for those diagnostic purposes anyway. // (This may be worth a design review at some point, but holds up well enough for now.) var Absent Node = absentNode{} type absentNode struct{} func (absentNode) Kind() Kind { return Kind_Null } func (absentNode) LookupByString(key string) (Node, error) { return nil, ErrWrongKind{TypeName: "absent", MethodName: "LookupByString", AppropriateKind: KindSet_JustMap, ActualKind: Kind_Null} } func (absentNode) LookupByNode(key Node) (Node, error) { return nil, ErrWrongKind{TypeName: "absent", MethodName: "LookupByNode", AppropriateKind: KindSet_JustMap, ActualKind: Kind_Null} } func (absentNode) LookupByIndex(idx int64) (Node, error) { return nil, ErrWrongKind{TypeName: "absent", MethodName: "LookupByIndex", AppropriateKind: KindSet_JustList, ActualKind: Kind_Null} } func (absentNode) LookupBySegment(seg PathSegment) (Node, error) { return nil, ErrWrongKind{TypeName: "absent", MethodName: "LookupBySegment", AppropriateKind: KindSet_Recursive, ActualKind: Kind_Null} } func (absentNode) MapIterator() MapIterator { return nil } func (absentNode) ListIterator() ListIterator { return nil } func (absentNode) Length() int64 { return -1 } func (absentNode) IsAbsent() bool { return true } func (absentNode) IsNull() bool { return false } func (absentNode) AsBool() (bool, error) { return false, ErrWrongKind{TypeName: "absent", MethodName: "AsBool", AppropriateKind: KindSet_JustBool, ActualKind: Kind_Null} } func (absentNode) AsInt() (int64, error) { return 0, ErrWrongKind{TypeName: "absent", MethodName: "AsInt", AppropriateKind: KindSet_JustInt, ActualKind: Kind_Null} } func (absentNode) AsFloat() (float64, error) { return 0, ErrWrongKind{TypeName: "absent", MethodName: "AsFloat", AppropriateKind: KindSet_JustFloat, ActualKind: Kind_Null} } func (absentNode) AsString() (string, error) { return "", ErrWrongKind{TypeName: "absent", MethodName: "AsString", AppropriateKind: KindSet_JustString, ActualKind: Kind_Null} } func (absentNode) AsBytes() ([]byte, error) { return nil, ErrWrongKind{TypeName: "absent", MethodName: "AsBytes", AppropriateKind: KindSet_JustBytes, ActualKind: Kind_Null} } func (absentNode) AsLink() (Link, error) { return nil, ErrWrongKind{TypeName: "absent", MethodName: "AsLink", AppropriateKind: KindSet_JustLink, ActualKind: Kind_Null} } func (absentNode) Prototype() NodePrototype { return absentPrototype{} } type absentPrototype struct{} func (absentPrototype) NewBuilder() NodeBuilder { panic("cannot build absent nodes") } ================================================ FILE: datamodel.go ================================================ package ipld import ( "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/linking" "github.com/ipld/go-ipld-prime/schema" ) type ( Kind = datamodel.Kind Node = datamodel.Node NodeAssembler = datamodel.NodeAssembler NodeBuilder = datamodel.NodeBuilder NodePrototype = datamodel.NodePrototype MapIterator = datamodel.MapIterator MapAssembler = datamodel.MapAssembler ListIterator = datamodel.ListIterator ListAssembler = datamodel.ListAssembler Link = datamodel.Link LinkPrototype = datamodel.LinkPrototype Path = datamodel.Path PathSegment = datamodel.PathSegment ) var ( Null = datamodel.Null Absent = datamodel.Absent ) const ( Kind_Invalid = datamodel.Kind_Invalid Kind_Map = datamodel.Kind_Map Kind_List = datamodel.Kind_List Kind_Null = datamodel.Kind_Null Kind_Bool = datamodel.Kind_Bool Kind_Int = datamodel.Kind_Int Kind_Float = datamodel.Kind_Float Kind_String = datamodel.Kind_String Kind_Bytes = datamodel.Kind_Bytes Kind_Link = datamodel.Kind_Link ) // Future: These aliases for the `KindSet_*` values may be dropped someday. // I don't think they're very important to have cluttering up namespace here. // They're included for a brief transitional period, largely for the sake of codegen things which have referred to them, but may disappear in the future. var ( KindSet_Recursive = datamodel.KindSet_Recursive KindSet_Scalar = datamodel.KindSet_Scalar KindSet_JustMap = datamodel.KindSet_JustMap KindSet_JustList = datamodel.KindSet_JustList KindSet_JustNull = datamodel.KindSet_JustNull KindSet_JustBool = datamodel.KindSet_JustBool KindSet_JustInt = datamodel.KindSet_JustInt KindSet_JustFloat = datamodel.KindSet_JustFloat KindSet_JustString = datamodel.KindSet_JustString KindSet_JustBytes = datamodel.KindSet_JustBytes KindSet_JustLink = datamodel.KindSet_JustLink ) // Future: These error type aliases may be dropped someday. // Being able to see them as having more than one package name is not helpful to clarity. // They are left here for now for a brief transitional period, because it was relatively easy to do so. type ( ErrWrongKind = datamodel.ErrWrongKind ErrNotExists = datamodel.ErrNotExists ErrRepeatedMapKey = datamodel.ErrRepeatedMapKey ErrInvalidSegmentForList = datamodel.ErrInvalidSegmentForList ErrIteratorOverread = datamodel.ErrIteratorOverread ErrInvalidKey = schema.ErrInvalidKey ErrMissingRequiredField = schema.ErrMissingRequiredField ErrHashMismatch = linking.ErrHashMismatch ) // Future: a bunch of these alias methods for path creation may be dropped someday. // They don't hurt anything, but I don't think they add much clarity either, vs the amount of namespace noise they create; // many of the high level convenience functions we add here in the root package will probably refer to datamodel.Path, and that should be sufficient to create clarity for new users for where to look for more on pathing. // They are here for now for a transitional period, but may eventually be revisited and perhaps removed. // NewPath is an alias for datamodel.NewPath. // // Pathing is a concept defined in the data model layer of IPLD. func NewPath(segments []PathSegment) Path { return datamodel.NewPath(segments) } // ParsePath is an alias for datamodel.ParsePath. // // Pathing is a concept defined in the data model layer of IPLD. func ParsePath(pth string) Path { return datamodel.ParsePath(pth) } // ParsePathSegment is an alias for datamodel.ParsePathSegment. // // Pathing is a concept defined in the data model layer of IPLD. func ParsePathSegment(s string) PathSegment { return datamodel.ParsePathSegment(s) } // PathSegmentOfString is an alias for datamodel.PathSegmentOfString. // // Pathing is a concept defined in the data model layer of IPLD. func PathSegmentOfString(s string) PathSegment { return datamodel.PathSegmentOfString(s) } // PathSegmentOfInt is an alias for datamodel.PathSegmentOfInt. // // Pathing is a concept defined in the data model layer of IPLD. func PathSegmentOfInt(i int64) PathSegment { return datamodel.PathSegmentOfInt(i) } ================================================ FILE: doc.go ================================================ // go-ipld-prime is a series of go interfaces for manipulating IPLD data. // // See https://ipld.io/ for more information about the basics // of "What is IPLD?". // // Here in the godoc, the first couple of types to look at should be: // // - Node // - NodeBuilder and NodeAssembler // - NodePrototype. // // These types provide a generic description of the data model. // // A Node is a piece of IPLD data which can be inspected. // A NodeAssembler is used to create Nodes. // (A NodeBuilder is just like a NodeAssembler, but allocates memory // (whereas a NodeAssembler just fills up memory; using these carefully // allows construction of very efficient code.) // // Different NodePrototypes can be used to describe Nodes which follow certain logical rules // (e.g., we use these as part of implementing Schemas), // and can also be used so that programs can use different memory layouts for different data // (which can be useful for constructing efficient programs when data has known shape for // which we can use specific or compacted memory layouts). // // If working with linked data (data which is split into multiple // trees of Nodes, loaded separately, and connected by some kind of // "link" reference), the next types you should look at are: // // - LinkSystem // - ... and its fields. // // The most typical use of LinkSystem is to use the linking/cid package // to get a LinkSystem that works with CIDs: // // lsys := cidlink.DefaultLinkSystem() // // ... and then assign the StorageWriteOpener and StorageReadOpener fields // in order to control where data is stored to and read from. // Methods on the LinkSystem then provide the functions typically used // to get data in and out of Nodes so you can work with it. // // This root package gathers some of the most important ease-of-use functions // all in one place, but is mostly aliases out to features originally found // in other more specific sub-packages. (If you're interested in keeping // your binary sizes small, and don't use some of the features of this library, // you'll probably want to look into using the relevant sub-packages directly.) // // Particularly interesting subpackages include: // // - datamodel -- the most essential interfaces for describing data live here, // describing Node, NodePrototype, NodeBuilder, Link, and Path. // - node/* -- various Node + NodeBuilder implementations. // - node/basicnode -- the first Node implementation you should try. // - codec/* -- functions for serializing and deserializing Nodes. // - linking -- the LinkSystem, which is a facade to all data loading and storing and hashing. // - linking/* -- ways to bind concrete Link implementations (namely, // the linking/cidlink package, which connects the go-cid library to our datamodel.Link interface). // - traversal -- functions for walking Node graphs (including automatic link loading) // and visiting them programmatically. // - traversal/selector -- functions for working with IPLD Selectors, // which are a language-agnostic declarative format for describing graph walks. // - fluent/* -- various options for making datamodel Node and NodeBuilder easier to work with. // - schema -- interfaces for working with IPLD Schemas, which can bring constraints // and validation systems to otherwise schemaless and unstructured IPLD data. // - adl/* -- examples of creating and using Advanced Data Layouts (in short, custom Node implementations) // to do complex data structures transparently within the IPLD Data Model. package ipld ================================================ FILE: examples_test.go ================================================ package ipld_test import ( "fmt" "os" "strings" "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/codec/dagjson" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/node/bindnode" "github.com/ipld/go-ipld-prime/schema" ) // Example_createDataAndMarshal shows how you can feed data into a NodeBuilder, // and also how to then hand that to an Encoder. // // Often you'll encoding implicitly through a LinkSystem.Store call instead, // but you can do it directly, too. func Example_createDataAndMarshal() { np := basicnode.Prototype.Any // Pick a prototype: this is how we decide what implementation will store the in-memory data. nb := np.NewBuilder() // Create a builder. ma, _ := nb.BeginMap(2) // Begin assembling a map. ma.AssembleKey().AssignString("hey") ma.AssembleValue().AssignString("it works!") ma.AssembleKey().AssignString("yes") ma.AssembleValue().AssignBool(true) ma.Finish() // Call 'Finish' on the map assembly to let it know no more data is coming. n := nb.Build() // Call 'Build' to get the resulting Node. (It's immutable!) dagjson.Encode(n, os.Stdout) // Output: // {"hey":"it works!","yes":true} } // Example_unmarshalData shows how you can use a Decoder // and a NodeBuilder (or NodePrototype) together to do unmarshalling. // // Often you'll do this implicitly through a LinkSystem.Load call instead, // but you can do it directly, too. func Example_unmarshalData() { serial := strings.NewReader(`{"hey":"it works!","yes": true}`) np := basicnode.Prototype.Any // Pick a style for the in-memory data. nb := np.NewBuilder() // Create a builder. dagjson.Decode(nb, serial) // Hand the builder to decoding -- decoding will fill it in! n := nb.Build() // Call 'Build' to get the resulting Node. (It's immutable!) fmt.Printf("the data decoded was a %s kind\n", n.Kind()) fmt.Printf("the length of the node is %d\n", n.Length()) // Output: // the data decoded was a map kind // the length of the node is 2 } func ExampleLoadSchema() { ts, err := ipld.LoadSchema("sample.ipldsch", strings.NewReader(` type Root struct { foo Int bar nullable String } `)) if err != nil { panic(err) } typeRoot := ts.TypeByName("Root").(*schema.TypeStruct) for _, field := range typeRoot.Fields() { fmt.Printf("field name=%q nullable=%t type=%v\n", field.Name(), field.IsNullable(), field.Type().Name()) } // Output: // field name="foo" nullable=false type=Int // field name="bar" nullable=true type=String } // Example_goValueWithSchema shows how to combine a Go value with an IPLD // schema, which can then be used as an IPLD node. // // For more examples and documentation, see the node/bindnode package. func Example_goValueWithSchema() { type Person struct { Name string Age int Friends []string } ts, err := ipld.LoadSchemaBytes([]byte(` type Person struct { name String age Int friends [String] } representation tuple `)) if err != nil { panic(err) } schemaType := ts.TypeByName("Person") person := &Person{Name: "Alice", Age: 34, Friends: []string{"Bob"}} node := bindnode.Wrap(person, schemaType) fmt.Printf("%#v\n", person) dagjson.Encode(node.Representation(), os.Stdout) // Output: // &ipld_test.Person{Name:"Alice", Age:34, Friends:[]string{"Bob"}} // ["Alice",34,["Bob"]] } ================================================ FILE: fluent/bench_test.go ================================================ package fluent_test import ( "strings" "testing" "github.com/ipld/go-ipld-prime/codec/dagjson" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/fluent" "github.com/ipld/go-ipld-prime/fluent/qp" "github.com/ipld/go-ipld-prime/node/basicnode" ) func BenchmarkQp(b *testing.B) { b.ReportAllocs() f2 := func(na datamodel.NodeAssembler, a string, b string, c string, d []string) { qp.Map(4, func(ma datamodel.MapAssembler) { qp.MapEntry(ma, "destination", qp.String(a)) qp.MapEntry(ma, "type", qp.String(b)) qp.MapEntry(ma, "source", qp.String(c)) qp.MapEntry(ma, "options", qp.List(int64(len(d)), func(la datamodel.ListAssembler) { for _, s := range d { qp.ListEntry(la, qp.String(s)) } })) })(na) } for i := 0; i < b.N; i++ { n, err := qp.BuildList(basicnode.Prototype.Any, -1, func(la datamodel.ListAssembler) { f2(la.AssembleValue(), // TODO: forgot to check error? "/", "overlay", "none", []string{ "lowerdir=" + "/", "upperdir=" + "/tmp/overlay-root/upper", "workdir=" + "/tmp/overlay-root/work", }, ) }) if err != nil { b.Fatal(err) } _ = n } } func BenchmarkUnmarshal(b *testing.B) { b.ReportAllocs() var n datamodel.Node var err error serial := `[{ "destination": "/", "type": "overlay", "source": "none", "options": [ "lowerdir=/", "upperdir=/tmp/overlay-root/upper", "workdir=/tmp/overlay-root/work" ] }]` r := strings.NewReader(serial) for i := 0; i < b.N; i++ { nb := basicnode.Prototype.Any.NewBuilder() err = dagjson.Decode(nb, r) n = nb.Build() r.Reset(serial) } _ = n if err != nil { b.Fatal(err) } } func BenchmarkFluent(b *testing.B) { b.ReportAllocs() var n datamodel.Node var err error for i := 0; i < b.N; i++ { n, err = fluent.BuildList(basicnode.Prototype.Any, -1, func(la fluent.ListAssembler) { la.AssembleValue().CreateMap(4, func(ma fluent.MapAssembler) { ma.AssembleEntry("destination").AssignString("/") ma.AssembleEntry("type").AssignString("overlay") ma.AssembleEntry("source").AssignString("none") ma.AssembleEntry("options").CreateList(-1, func(la fluent.ListAssembler) { la.AssembleValue().AssignString("lowerdir=" + "/") la.AssembleValue().AssignString("upperdir=" + "/tmp/overlay-root/upper") la.AssembleValue().AssignString("workdir=" + "/tmp/overlay-root/work") }) }) }) } _ = n if err != nil { b.Fatal(err) } } func BenchmarkReflect(b *testing.B) { b.ReportAllocs() var n datamodel.Node var err error val := []interface{}{ map[string]interface{}{ "destination": "/", "type": "overlay", "source": "none", "options": []string{ "lowerdir=/", "upperdir=/tmp/overlay-root/upper", "workdir=/tmp/overlay-root/work", }, }, } for i := 0; i < b.N; i++ { n, err = fluent.Reflect(basicnode.Prototype.Any, val) } _ = n if err != nil { b.Fatal(err) } } func BenchmarkReflectIncludingInitialization(b *testing.B) { b.ReportAllocs() var n datamodel.Node var err error for i := 0; i < b.N; i++ { n, err = fluent.Reflect(basicnode.Prototype.Any, []interface{}{ map[string]interface{}{ "destination": "/", "type": "overlay", "source": "none", "options": []string{ "lowerdir=/", "upperdir=/tmp/overlay-root/upper", "workdir=/tmp/overlay-root/work", }, }, }) } _ = n if err != nil { b.Fatal(err) } } func BenchmarkAgonizinglyBare(b *testing.B) { b.ReportAllocs() var n datamodel.Node var err error for i := 0; i < b.N; i++ { n, err = fab() } _ = n if err != nil { b.Fatal(err) } } func fab() (datamodel.Node, error) { nb := basicnode.Prototype.Any.NewBuilder() la1, err := nb.BeginList(-1) if err != nil { return nil, err } ma, err := la1.AssembleValue().BeginMap(4) if err != nil { return nil, err } va, err := ma.AssembleEntry("destination") if err != nil { return nil, err } err = va.AssignString("/") if err != nil { return nil, err } va, err = ma.AssembleEntry("type") if err != nil { return nil, err } err = va.AssignString("overlay") if err != nil { return nil, err } va, err = ma.AssembleEntry("source") if err != nil { return nil, err } err = va.AssignString("none") if err != nil { return nil, err } va, err = ma.AssembleEntry("options") if err != nil { return nil, err } la2, err := va.BeginList(-4) if err != nil { return nil, err } err = la2.AssembleValue().AssignString("lowerdir=" + "/") if err != nil { return nil, err } err = la2.AssembleValue().AssignString("upperdir=" + "/tmp/overlay-root/upper") if err != nil { return nil, err } err = la2.AssembleValue().AssignString("workdir=" + "/tmp/overlay-root/work") if err != nil { return nil, err } err = la2.Finish() if err != nil { return nil, err } err = ma.Finish() if err != nil { return nil, err } err = la1.Finish() if err != nil { return nil, err } return nb.Build(), nil } ================================================ FILE: fluent/doc.go ================================================ /* The fluent package offers helper utilities for using NodeAssembler more tersely by providing an interface that handles all errors for you, and allows use of closures for any recursive assembly so that creating trees of data results in indentation for legibility. Note that the fluent package creates wrapper objects in order to provide the API conveniences that it does, and this comes at some cost to performance. If you're optimizing for performance, using some of the features of the fluent package may be inadvisable (and some moreso than others). However, as with any performance questions, benchmark before making decisions; its entirely possible that your performance bottlenecks will be elsewhere and there's no reason to deny yourself syntactic sugar if the costs don't detectably affect the bottom line. Various feature of the package will also attempt to document how costly they are in relative terms (e.g. the fluent.Reflect helper methods are very costly; */ package fluent ================================================ FILE: fluent/fluentBuilder.go ================================================ package fluent import ( "github.com/ipld/go-ipld-prime/datamodel" ) func Build(np datamodel.NodePrototype, fn func(NodeAssembler)) (datamodel.Node, error) { nb := np.NewBuilder() fna := WrapAssembler(nb) err := Recover(func() { fn(fna) }) if err != nil { return nil, err } return nb.Build(), nil } func BuildMap(np datamodel.NodePrototype, sizeHint int64, fn func(MapAssembler)) (datamodel.Node, error) { return Build(np, func(fna NodeAssembler) { fna.CreateMap(sizeHint, fn) }) } func BuildList(np datamodel.NodePrototype, sizeHint int64, fn func(ListAssembler)) (datamodel.Node, error) { return Build(np, func(fna NodeAssembler) { fna.CreateList(sizeHint, fn) }) } func MustBuild(np datamodel.NodePrototype, fn func(NodeAssembler)) datamodel.Node { nb := np.NewBuilder() fn(WrapAssembler(nb)) return nb.Build() } func MustBuildMap(np datamodel.NodePrototype, sizeHint int64, fn func(MapAssembler)) datamodel.Node { return MustBuild(np, func(fna NodeAssembler) { fna.CreateMap(sizeHint, fn) }) } func MustBuildList(np datamodel.NodePrototype, sizeHint int64, fn func(ListAssembler)) datamodel.Node { return MustBuild(np, func(fna NodeAssembler) { fna.CreateList(sizeHint, fn) }) } func WrapAssembler(na datamodel.NodeAssembler) NodeAssembler { return &nodeAssembler{na} } // NodeAssembler is the same as the interface in the core package, except: // instead of returning errors, any error will cause panic // (and you can collect these with `fluent.Recover`); // and all recursive operations take a function as a parameter, // within which you will receive another {Map,List,}NodeAssembler. type NodeAssembler interface { CreateMap(sizeHint int64, fn func(MapAssembler)) CreateList(sizeHint int64, fn func(ListAssembler)) AssignNull() AssignBool(bool) AssignInt(int64) AssignFloat(float64) AssignString(string) AssignBytes([]byte) AssignLink(datamodel.Link) AssignNode(datamodel.Node) Prototype() datamodel.NodePrototype } // MapAssembler is the same as the interface in the core package, except: // instead of returning errors, any error will cause panic // (and you can collect these with `fluent.Recover`); // and all recursive operations take a function as a parameter, // within which you will receive another {Map,List,}NodeAssembler. type MapAssembler interface { AssembleKey() NodeAssembler AssembleValue() NodeAssembler AssembleEntry(k string) NodeAssembler KeyPrototype() datamodel.NodePrototype ValuePrototype(k string) datamodel.NodePrototype } // ListAssembler is the same as the interface in the core package, except: // instead of returning errors, any error will cause panic // (and you can collect these with `fluent.Recover`); // and all recursive operations take a function as a parameter, // within which you will receive another {Map,List,}NodeAssembler. type ListAssembler interface { AssembleValue() NodeAssembler ValuePrototype(idx int64) datamodel.NodePrototype } type nodeAssembler struct { na datamodel.NodeAssembler } func (fna *nodeAssembler) CreateMap(sizeHint int64, fn func(MapAssembler)) { if ma, err := fna.na.BeginMap(sizeHint); err != nil { panic(Error{err}) } else { fn(&mapNodeAssembler{ma}) if err := ma.Finish(); err != nil { panic(Error{err}) } } } func (fna *nodeAssembler) CreateList(sizeHint int64, fn func(ListAssembler)) { if la, err := fna.na.BeginList(sizeHint); err != nil { panic(Error{err}) } else { fn(&listNodeAssembler{la}) if err := la.Finish(); err != nil { panic(Error{err}) } } } func (fna *nodeAssembler) AssignNull() { if err := fna.na.AssignNull(); err != nil { panic(Error{err}) } } func (fna *nodeAssembler) AssignBool(v bool) { if err := fna.na.AssignBool(v); err != nil { panic(Error{err}) } } func (fna *nodeAssembler) AssignInt(v int64) { if err := fna.na.AssignInt(v); err != nil { panic(Error{err}) } } func (fna *nodeAssembler) AssignFloat(v float64) { if err := fna.na.AssignFloat(v); err != nil { panic(Error{err}) } } func (fna *nodeAssembler) AssignString(v string) { if err := fna.na.AssignString(v); err != nil { panic(Error{err}) } } func (fna *nodeAssembler) AssignBytes(v []byte) { if err := fna.na.AssignBytes(v); err != nil { panic(Error{err}) } } func (fna *nodeAssembler) AssignLink(v datamodel.Link) { if err := fna.na.AssignLink(v); err != nil { panic(Error{err}) } } func (fna *nodeAssembler) AssignNode(v datamodel.Node) { if err := fna.na.AssignNode(v); err != nil { panic(Error{err}) } } func (fna *nodeAssembler) Prototype() datamodel.NodePrototype { return fna.na.Prototype() } type mapNodeAssembler struct { ma datamodel.MapAssembler } func (fma *mapNodeAssembler) AssembleKey() NodeAssembler { return &nodeAssembler{fma.ma.AssembleKey()} } func (fma *mapNodeAssembler) AssembleValue() NodeAssembler { return &nodeAssembler{fma.ma.AssembleValue()} } func (fma *mapNodeAssembler) AssembleEntry(k string) NodeAssembler { va, err := fma.ma.AssembleEntry(k) if err != nil { panic(Error{err}) } return &nodeAssembler{va} } func (fma *mapNodeAssembler) KeyPrototype() datamodel.NodePrototype { return fma.ma.KeyPrototype() } func (fma *mapNodeAssembler) ValuePrototype(k string) datamodel.NodePrototype { return fma.ma.ValuePrototype(k) } type listNodeAssembler struct { la datamodel.ListAssembler } func (fla *listNodeAssembler) AssembleValue() NodeAssembler { return &nodeAssembler{fla.la.AssembleValue()} } func (fla *listNodeAssembler) ValuePrototype(idx int64) datamodel.NodePrototype { return fla.la.ValuePrototype(idx) } ================================================ FILE: fluent/fluentBuilder_test.go ================================================ package fluent_test import ( "testing" qt "github.com/frankban/quicktest" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/fluent" "github.com/ipld/go-ipld-prime/must" "github.com/ipld/go-ipld-prime/node/basicnode" ) func TestBuild(t *testing.T) { t.Run("scalar build should work", func(t *testing.T) { n := fluent.MustBuild(basicnode.Prototype__String{}, func(fna fluent.NodeAssembler) { fna.AssignString("fine") }) qt.Check(t, n.Kind(), qt.Equals, datamodel.Kind_String) v2, err := n.AsString() qt.Check(t, err, qt.IsNil) qt.Check(t, v2, qt.Equals, "fine") }) t.Run("map build should work", func(t *testing.T) { n := fluent.MustBuild(basicnode.Prototype.Map, func(fna fluent.NodeAssembler) { fna.CreateMap(3, func(fma fluent.MapAssembler) { fma.AssembleEntry("k1").AssignString("fine") fma.AssembleEntry("k2").AssignString("super") fma.AssembleEntry("k3").CreateMap(3, func(fma fluent.MapAssembler) { fma.AssembleEntry("k31").AssignString("thanks") fma.AssembleEntry("k32").AssignString("for") fma.AssembleEntry("k33").AssignString("asking") }) }) }) qt.Check(t, n.Kind(), qt.Equals, datamodel.Kind_Map) qt.Check(t, n.Length(), qt.Equals, int64(3)) qt.Check(t, must.String(must.Node(n.LookupByString("k1"))), qt.Equals, "fine") qt.Check(t, must.String(must.Node(n.LookupByString("k2"))), qt.Equals, "super") n = must.Node(n.LookupByString("k3")) qt.Check(t, n.Length(), qt.Equals, int64(3)) qt.Check(t, must.String(must.Node(n.LookupByString("k31"))), qt.Equals, "thanks") qt.Check(t, must.String(must.Node(n.LookupByString("k32"))), qt.Equals, "for") qt.Check(t, must.String(must.Node(n.LookupByString("k33"))), qt.Equals, "asking") }) t.Run("list build should work", func(t *testing.T) { n := fluent.MustBuild(basicnode.Prototype.List, func(fna fluent.NodeAssembler) { fna.CreateList(1, func(fla fluent.ListAssembler) { fla.AssembleValue().CreateList(1, func(fla fluent.ListAssembler) { fla.AssembleValue().CreateList(1, func(fla fluent.ListAssembler) { fla.AssembleValue().CreateList(1, func(fla fluent.ListAssembler) { fla.AssembleValue().AssignInt(2) }) }) }) }) }) qt.Check(t, n.Kind(), qt.Equals, datamodel.Kind_List) qt.Check(t, n.Length(), qt.Equals, int64(1)) n = must.Node(n.LookupByIndex(0)) qt.Check(t, n.Kind(), qt.Equals, datamodel.Kind_List) qt.Check(t, n.Length(), qt.Equals, int64(1)) n = must.Node(n.LookupByIndex(0)) qt.Check(t, n.Kind(), qt.Equals, datamodel.Kind_List) qt.Check(t, n.Length(), qt.Equals, int64(1)) n = must.Node(n.LookupByIndex(0)) qt.Check(t, n.Kind(), qt.Equals, datamodel.Kind_List) qt.Check(t, n.Length(), qt.Equals, int64(1)) n = must.Node(n.LookupByIndex(0)) qt.Check(t, n.Kind(), qt.Equals, datamodel.Kind_Int) qt.Check(t, must.Int(n), qt.Equals, int64(2)) }) } ================================================ FILE: fluent/fluentRecover.go ================================================ package fluent type Error struct { Err error } func (e Error) Error() string { return e.Err.Error() } // Recover invokes a function within a panic-recovering context, and returns // any raised fluent.Error values; any other values are re-panicked. // // This can be useful for writing large blocks of code using fluent nodes, // and handling any errors at once at the end. func Recover(fn func()) (err error) { defer func() { ei := recover() switch e2 := ei.(type) { case nil: return case Error: err = e2 default: panic(ei) } }() fn() return } ================================================ FILE: fluent/fluentRecover_test.go ================================================ package fluent_test import ( "testing" qt "github.com/frankban/quicktest" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/fluent" "github.com/ipld/go-ipld-prime/node/basicnode" ) func TestRecover(t *testing.T) { t.Run("simple build error should capture", func(t *testing.T) { qt.Check(t, fluent.Recover(func() { fluent.MustBuild(basicnode.Prototype__String{}, func(fna fluent.NodeAssembler) { fna.AssignInt(9) }) t.Fatal("should not be reached") }), qt.DeepEquals, fluent.Error{datamodel.ErrWrongKind{TypeName: "string", MethodName: "AssignInt", AppropriateKind: datamodel.KindSet_JustInt, ActualKind: datamodel.Kind_String}}, ) }) t.Run("correct build should return nil", func(t *testing.T) { qt.Check(t, fluent.Recover(func() { fluent.MustBuild(basicnode.Prototype__String{}, func(fna fluent.NodeAssembler) { fna.AssignString("fine") }) }), qt.IsNil, ) }) t.Run("other panics should continue to rise", func(t *testing.T) { qt.Check(t, func() (r interface{}) { defer func() { r = recover() }() fluent.Recover(func() { panic("fuqawds") }) return }(), qt.Equals, "fuqawds", ) }) } ================================================ FILE: fluent/qp/example_test.go ================================================ package qp_test import ( "os" "github.com/ipld/go-ipld-prime/codec/dagjson" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/fluent/qp" "github.com/ipld/go-ipld-prime/node/basicnode" ) // TODO: can we make ListEntry/MapEntry less verbose? func Example() { n, err := qp.BuildMap(basicnode.Prototype.Any, 4, func(ma datamodel.MapAssembler) { qp.MapEntry(ma, "some key", qp.String("some value")) qp.MapEntry(ma, "another key", qp.String("another value")) qp.MapEntry(ma, "nested map", qp.Map(2, func(ma datamodel.MapAssembler) { qp.MapEntry(ma, "deeper entries", qp.String("deeper values")) qp.MapEntry(ma, "more deeper entries", qp.String("more deeper values")) })) qp.MapEntry(ma, "nested list", qp.List(2, func(la datamodel.ListAssembler) { qp.ListEntry(la, qp.Int(1)) qp.ListEntry(la, qp.Int(2)) })) }) if err != nil { panic(err) } dagjson.Encode(n, os.Stdout) // Output: // {"another key":"another value","nested list":[1,2],"nested map":{"deeper entries":"deeper values","more deeper entries":"more deeper values"},"some key":"some value"} } ================================================ FILE: fluent/qp/qp.go ================================================ // qp helps to quickly build IPLD nodes. // // It contains top-level Build funcs, such as BuildMap and BuildList, which // return the final node as well as an error. // // Underneath, one can use a number of Assemble functions to construct basic // nodes, such as String or Int. // // Finally, functions like MapEntry and ListEntry allow inserting into maps and // lists. // // These all use the same IPLD datamodel interfaces such as NodePrototype and // NodeAssembler, but with some magic to reduce verbosity. package qp import ( "fmt" "github.com/ipld/go-ipld-prime/datamodel" ) type Assemble = func(datamodel.NodeAssembler) func BuildMap(np datamodel.NodePrototype, sizeHint int64, fn func(datamodel.MapAssembler)) (_ datamodel.Node, err error) { defer func() { if r := recover(); r != nil { if rerr, ok := r.(error); ok { err = rerr } else { // A reasonable fallback, for e.g. strings. err = fmt.Errorf("%v", r) } } }() nb := np.NewBuilder() Map(sizeHint, fn)(nb) return nb.Build(), nil } type mapParams struct { sizeHint int64 fn func(datamodel.MapAssembler) } func (mp mapParams) Assemble(na datamodel.NodeAssembler) { ma, err := na.BeginMap(mp.sizeHint) if err != nil { panic(err) } mp.fn(ma) if err := ma.Finish(); err != nil { panic(err) } } func Map(sizeHint int64, fn func(datamodel.MapAssembler)) Assemble { return mapParams{sizeHint, fn}.Assemble } func MapEntry(ma datamodel.MapAssembler, k string, fn Assemble) { na, err := ma.AssembleEntry(k) if err != nil { panic(err) } fn(na) } func BuildList(np datamodel.NodePrototype, sizeHint int64, fn func(datamodel.ListAssembler)) (_ datamodel.Node, err error) { defer func() { if r := recover(); r != nil { if rerr, ok := r.(error); ok { err = rerr } else { // A reasonable fallback, for e.g. strings. err = fmt.Errorf("%v", r) } } }() nb := np.NewBuilder() List(sizeHint, fn)(nb) return nb.Build(), nil } type listParams struct { sizeHint int64 fn func(datamodel.ListAssembler) } func (lp listParams) Assemble(na datamodel.NodeAssembler) { la, err := na.BeginList(lp.sizeHint) if err != nil { panic(err) } lp.fn(la) if err := la.Finish(); err != nil { panic(err) } } func List(sizeHint int64, fn func(datamodel.ListAssembler)) Assemble { return listParams{sizeHint, fn}.Assemble } func ListEntry(la datamodel.ListAssembler, fn Assemble) { fn(la.AssembleValue()) } type nullParam struct{} func (s nullParam) Assemble(na datamodel.NodeAssembler) { if err := na.AssignNull(); err != nil { panic(err) } } func Null() Assemble { return nullParam{}.Assemble } type boolParam bool func (s boolParam) Assemble(na datamodel.NodeAssembler) { if err := na.AssignBool(bool(s)); err != nil { panic(err) } } func Bool(b bool) Assemble { return boolParam(b).Assemble } type intParam int64 func (i intParam) Assemble(na datamodel.NodeAssembler) { if err := na.AssignInt(int64(i)); err != nil { panic(err) } } func Int(i int64) Assemble { return intParam(i).Assemble } type floatParam float64 func (f floatParam) Assemble(na datamodel.NodeAssembler) { if err := na.AssignFloat(float64(f)); err != nil { panic(err) } } func Float(f float64) Assemble { return floatParam(f).Assemble } type stringParam string func (s stringParam) Assemble(na datamodel.NodeAssembler) { if err := na.AssignString(string(s)); err != nil { panic(err) } } func String(s string) Assemble { return stringParam(s).Assemble } type bytesParam []byte func (p bytesParam) Assemble(na datamodel.NodeAssembler) { if err := na.AssignBytes([]byte(p)); err != nil { panic(err) } } func Bytes(p []byte) Assemble { return bytesParam(p).Assemble } type linkParam struct { x datamodel.Link } func (l linkParam) Assemble(na datamodel.NodeAssembler) { if err := na.AssignLink(l.x); err != nil { panic(err) } } func Link(l datamodel.Link) Assemble { return linkParam{l}.Assemble } type nodeParam struct { x datamodel.Node } func (n nodeParam) Assemble(na datamodel.NodeAssembler) { if err := na.AssignNode(n.x); err != nil { panic(err) } } func Node(n datamodel.Node) Assemble { return nodeParam{n}.Assemble } ================================================ FILE: fluent/reflect.go ================================================ package fluent import ( "fmt" "reflect" "sort" "github.com/ipld/go-ipld-prime/datamodel" ) // Reflect creates a new Node by looking at a golang value with reflection // and converting it into IPLD Data Model. // This is a quick-and-dirty way to get data into the IPLD Data Model; // it's useful for rapid prototyping and demos, // but note that this feature is not intended to be suitable for "production" use // due to low performance and lack of configurability. // // The concrete type of the returned Node is determined by // the NodePrototype argument provided by the caller. // // No type information from the golang value will be observable in the result. // // The reflection will walk over any golang value, but is not configurable. // Golang maps become IPLD maps; golang slices and arrays become IPLD lists; // and golang structs become IPLD maps too. // When converting golang structs to IPLD maps, the field names will become the map keys. // Pointers and interfaces will be traversed transparently and are not visible in the output. // // An error will be returned if the process of assembling the Node returns any errors // (for example, if the NodePrototype is for a schema-constrained Node, // any validation errors from the schema will cause errors to be returned). // // A panic will be raised if there is any difficulty examining the golang value via reflection // (for example, if the value is a struct with unexported fields, // or if a non-data type like a channel or function is encountered). // // Some configuration (in particular, what to do about map ordering) is available via the Reflector struct. // That structure has a method of the same name and signiture as this one on it. // (This function is a shortcut for calling that method on a Reflector struct with default configuration.) // // Performance remarks: performance of this function will generally be poor. // In general, creating data in golang types and then *flipping* it to IPLD form // involves handling the data at least twice, and so will always be slower // than just creating the same data in IPLD form programmatically directly. // In particular, reflection is generally not fast, and this feature has // not been optimized for either speed nor allocation avoidance. // Other features in the fluent package will typically out-perform this, // and using NodeAssemblers directly (without any fluent tools) will be much faster. // Only use this function if performance is not of consequence. func Reflect(np datamodel.NodePrototype, i interface{}) (datamodel.Node, error) { return defaultReflector.Reflect(np, i) } // MustReflect is a shortcut for Reflect but panics on any error. // It is useful if you need a single return value for function composition purposes. func MustReflect(np datamodel.NodePrototype, i interface{}) datamodel.Node { n, err := Reflect(np, i) if err != nil { panic(err) } return n } // ReflectIntoAssembler is similar to Reflect, but takes a NodeAssembler parameter // instead of a Node Prototype. // This may be useful if you need more direct control over allocations, // or want to fill in only part of a larger node assembly process using the reflect tool. // Data is accumulated by the NodeAssembler parameter, so no Node is returned. func ReflectIntoAssembler(na datamodel.NodeAssembler, i interface{}) error { return defaultReflector.ReflectIntoAssembler(na, i) } var defaultReflector = Reflector{ MapOrder: func(x, y string) bool { return x < y }, } // Reflector allows configuration of the Reflect family of functions // (`Reflect`, `ReflectIntoAssembler`, etc). type Reflector struct { // MapOrder is used to decide a deterministic order for inserting entries to maps. // (This is used when converting golang maps, since their iteration order is randomized; // it is not used when converting other types such as structs, since those have a stable order.) // MapOrder should return x < y in the same way as sort.Interface.Less. // // If using a default Reflector (e.g. via the package-scope functions), // this function is a simple natural golang string sort: it performs `x < y` on the strings. MapOrder func(x, y string) bool } // Reflect is as per the package-scope function of the same name and signature, // but using the configuration in the Reflector struct. // See the package-scope function for documentation. func (rcfg Reflector) Reflect(np datamodel.NodePrototype, i interface{}) (datamodel.Node, error) { nb := np.NewBuilder() if err := rcfg.ReflectIntoAssembler(nb, i); err != nil { return nil, err } return nb.Build(), nil } // ReflectIntoAssembler is as per the package-scope function of the same name and signature, // but using the configuration in the Reflector struct. // See the package-scope function for documentation. func (rcfg Reflector) ReflectIntoAssembler(na datamodel.NodeAssembler, i interface{}) error { // Cover the most common values with a type-switch, as it's faster than reflection. switch x := i.(type) { case map[string]string: keys := make([]string, 0, len(x)) for k := range x { keys = append(keys, k) } sort.Sort(sortableStrings{keys, rcfg.MapOrder}) ma, err := na.BeginMap(int64(len(x))) if err != nil { return err } for _, k := range keys { va, err := ma.AssembleEntry(k) if err != nil { return err } if err := va.AssignString(x[k]); err != nil { return err } } return ma.Finish() case map[string]interface{}: keys := make([]string, 0, len(x)) for k := range x { keys = append(keys, k) } sort.Sort(sortableStrings{keys, rcfg.MapOrder}) ma, err := na.BeginMap(int64(len(x))) if err != nil { return err } for _, k := range keys { va, err := ma.AssembleEntry(k) if err != nil { return err } if err := rcfg.ReflectIntoAssembler(va, x[k]); err != nil { return err } } return ma.Finish() case []string: la, err := na.BeginList(int64(len(x))) if err != nil { return err } for _, v := range x { if err := la.AssembleValue().AssignString(v); err != nil { return err } } return la.Finish() case []interface{}: la, err := na.BeginList(int64(len(x))) if err != nil { return err } for _, v := range x { if err := rcfg.ReflectIntoAssembler(la.AssembleValue(), v); err != nil { return err } } return la.Finish() case string: return na.AssignString(x) case []byte: return na.AssignBytes(x) case int64: return na.AssignInt(x) case nil: return na.AssignNull() } // That didn't fly? Reflection time. rv := reflect.ValueOf(i) switch rv.Kind() { case reflect.Bool: return na.AssignBool(rv.Bool()) case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: return na.AssignInt(rv.Int()) case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: return na.AssignInt(int64(rv.Uint())) // TODO: check overflow case reflect.Float32, reflect.Float64: return na.AssignFloat(rv.Float()) case reflect.String: return na.AssignString(rv.String()) case reflect.Slice, reflect.Array: if rv.Type().Elem().Kind() == reflect.Uint8 { // byte slices are a special case return na.AssignBytes(rv.Bytes()) } l := rv.Len() la, err := na.BeginList(int64(l)) if err != nil { return err } for i := 0; i < l; i++ { if err := rcfg.ReflectIntoAssembler(la.AssembleValue(), rv.Index(i).Interface()); err != nil { return err } } return la.Finish() case reflect.Map: // the keys slice for sorting keeps things in reflect.Value form, because unboxing is cheap, // but re-boxing is not cheap, and the MapIndex method requires reflect.Value again later. keys := make([]reflect.Value, 0, rv.Len()) itr := rv.MapRange() for itr.Next() { k := itr.Key() if k.Kind() != reflect.String { return fmt.Errorf("cannot convert a map with non-string keys (%T)", i) } keys = append(keys, k) } sort.Sort(sortableReflectStrings{keys, rcfg.MapOrder}) ma, err := na.BeginMap(int64(rv.Len())) if err != nil { return err } for _, k := range keys { va, err := ma.AssembleEntry(k.String()) if err != nil { return err } if err := rcfg.ReflectIntoAssembler(va, rv.MapIndex(k).Interface()); err != nil { return err } } return ma.Finish() case reflect.Struct: l := rv.NumField() ma, err := na.BeginMap(int64(l)) if err != nil { return err } for i := 0; i < l; i++ { fn := rv.Type().Field(i).Name fv := rv.Field(i) va, err := ma.AssembleEntry(fn) if err != nil { return err } if err := rcfg.ReflectIntoAssembler(va, fv.Interface()); err != nil { return err } } return ma.Finish() case reflect.Ptr: if rv.IsNil() { return na.AssignNull() } return rcfg.ReflectIntoAssembler(na, rv.Elem()) case reflect.Interface: return rcfg.ReflectIntoAssembler(na, rv.Elem()) } // Some kints of values -- like Uintptr, Complex64/128, Channels, etc -- are not supported by this function. return fmt.Errorf("fluent.Reflect: unsure how to handle type %T (kind: %v)", i, rv.Kind()) } type sortableStrings struct { a []string less func(x, y string) bool } func (a sortableStrings) Len() int { return len(a.a) } func (a sortableStrings) Swap(i, j int) { a.a[i], a.a[j] = a.a[j], a.a[i] } func (a sortableStrings) Less(i, j int) bool { return a.less(a.a[i], a.a[j]) } type sortableReflectStrings struct { a []reflect.Value less func(x, y string) bool } func (a sortableReflectStrings) Len() int { return len(a.a) } func (a sortableReflectStrings) Swap(i, j int) { a.a[i], a.a[j] = a.a[j], a.a[i] } func (a sortableReflectStrings) Less(i, j int) bool { return a.less(a.a[i].String(), a.a[j].String()) } ================================================ FILE: fluent/reflect_test.go ================================================ package fluent_test import ( "testing" qt "github.com/frankban/quicktest" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/fluent" "github.com/ipld/go-ipld-prime/must" "github.com/ipld/go-ipld-prime/node/basicnode" ) func TestReflect(t *testing.T) { t.Run("Map", func(t *testing.T) { n, err := fluent.Reflect(basicnode.Prototype.Any, map[string]interface{}{ "k1": "fine", "k2": "super", "k3": map[string]string{ "k31": "thanks", "k32": "for", "k33": "asking", }, }) qt.Check(t, err, qt.IsNil) qt.Check(t, n.Kind(), qt.Equals, datamodel.Kind_Map) t.Run("CorrectContents", func(t *testing.T) { qt.Check(t, n.Length(), qt.Equals, int64(3)) qt.Check(t, must.String(must.Node(n.LookupByString("k1"))), qt.Equals, "fine") qt.Check(t, must.String(must.Node(n.LookupByString("k2"))), qt.Equals, "super") n := must.Node(n.LookupByString("k3")) qt.Check(t, n.Length(), qt.Equals, int64(3)) qt.Check(t, must.String(must.Node(n.LookupByString("k31"))), qt.Equals, "thanks") qt.Check(t, must.String(must.Node(n.LookupByString("k32"))), qt.Equals, "for") qt.Check(t, must.String(must.Node(n.LookupByString("k33"))), qt.Equals, "asking") }) t.Run("CorrectOrder", func(t *testing.T) { itr := n.MapIterator() k, _, _ := itr.Next() qt.Check(t, must.String(k), qt.Equals, "k1") k, _, _ = itr.Next() qt.Check(t, must.String(k), qt.Equals, "k2") k, v, _ := itr.Next() qt.Check(t, must.String(k), qt.Equals, "k3") itr = v.MapIterator() k, _, _ = itr.Next() qt.Check(t, must.String(k), qt.Equals, "k31") k, _, _ = itr.Next() qt.Check(t, must.String(k), qt.Equals, "k32") k, _, _ = itr.Next() qt.Check(t, must.String(k), qt.Equals, "k33") }) }) t.Run("Struct", func(t *testing.T) { type Woo struct { A string B string } type Whee struct { X string Z string M Woo } n, err := fluent.Reflect(basicnode.Prototype.Any, Whee{ X: "fine", Z: "super", M: Woo{"thanks", "really"}, }) qt.Check(t, err, qt.IsNil) qt.Check(t, n.Kind(), qt.Equals, datamodel.Kind_Map) t.Run("CorrectContents", func(t *testing.T) { qt.Check(t, n.Length(), qt.Equals, int64(3)) qt.Check(t, must.String(must.Node(n.LookupByString("X"))), qt.Equals, "fine") qt.Check(t, must.String(must.Node(n.LookupByString("Z"))), qt.Equals, "super") n := must.Node(n.LookupByString("M")) qt.Check(t, n.Length(), qt.Equals, int64(2)) qt.Check(t, must.String(must.Node(n.LookupByString("A"))), qt.Equals, "thanks") qt.Check(t, must.String(must.Node(n.LookupByString("B"))), qt.Equals, "really") }) t.Run("CorrectOrder", func(t *testing.T) { itr := n.MapIterator() k, _, _ := itr.Next() qt.Check(t, must.String(k), qt.Equals, "X") k, _, _ = itr.Next() qt.Check(t, must.String(k), qt.Equals, "Z") k, v, _ := itr.Next() qt.Check(t, must.String(k), qt.Equals, "M") itr = v.MapIterator() k, _, _ = itr.Next() qt.Check(t, must.String(k), qt.Equals, "A") k, _, _ = itr.Next() qt.Check(t, must.String(k), qt.Equals, "B") }) }) t.Run("NamedString", func(t *testing.T) { type Foo string type Bar struct { Z Foo } n, err := fluent.Reflect(basicnode.Prototype.Any, Bar{"foo"}) qt.Check(t, err, qt.IsNil) qt.Check(t, n.Kind(), qt.Equals, datamodel.Kind_Map) qt.Check(t, must.String(must.Node(n.LookupByString("Z"))), qt.Equals, "foo") }) t.Run("Interface", func(t *testing.T) { type Zaz struct { Z interface{} } n, err := fluent.Reflect(basicnode.Prototype.Any, Zaz{map[string]interface{}{"wow": "wee"}}) qt.Check(t, err, qt.IsNil) qt.Check(t, n.Kind(), qt.Equals, datamodel.Kind_Map) n, err = n.LookupByString("Z") qt.Check(t, err, qt.IsNil) qt.Check(t, n.Kind(), qt.Equals, datamodel.Kind_Map) qt.Check(t, must.String(must.Node(n.LookupByString("wow"))), qt.Equals, "wee") }) t.Run("Bytes", func(t *testing.T) { n, err := fluent.Reflect(basicnode.Prototype.Any, []byte{0x1, 0x2, 0x3}) qt.Check(t, err, qt.IsNil) qt.Check(t, n.Kind(), qt.Equals, datamodel.Kind_Bytes) b, err := n.AsBytes() qt.Check(t, err, qt.IsNil) qt.Check(t, b, qt.DeepEquals, []byte{0x1, 0x2, 0x3}) }) t.Run("NamedBytes", func(t *testing.T) { type Foo []byte type Bar struct { Z Foo } n, err := fluent.Reflect(basicnode.Prototype.Any, Bar{[]byte{0x1, 0x2, 0x3}}) qt.Check(t, err, qt.IsNil) qt.Check(t, n.Kind(), qt.Equals, datamodel.Kind_Map) n, err = n.LookupByString("Z") qt.Check(t, err, qt.IsNil) qt.Check(t, n.Kind(), qt.Equals, datamodel.Kind_Bytes) b, err := n.AsBytes() qt.Check(t, err, qt.IsNil) qt.Check(t, b, qt.DeepEquals, []byte{0x1, 0x2, 0x3}) }) t.Run("InterfaceContainingBytes", func(t *testing.T) { type Zaz struct { Z interface{} } n, err := fluent.Reflect(basicnode.Prototype.Any, Zaz{[]byte{0x1, 0x2, 0x3}}) qt.Check(t, err, qt.IsNil) qt.Check(t, n.Kind(), qt.Equals, datamodel.Kind_Map) n, err = n.LookupByString("Z") qt.Check(t, err, qt.IsNil) qt.Check(t, n.Kind(), qt.Equals, datamodel.Kind_Bytes) b, err := n.AsBytes() qt.Check(t, err, qt.IsNil) qt.Check(t, b, qt.DeepEquals, []byte{0x1, 0x2, 0x3}) }) } ================================================ FILE: fluent/toInterfaceValue.go ================================================ package fluent import ( "errors" "github.com/ipld/go-ipld-prime/datamodel" ) var errInvalidKind = errors.New("invalid kind") var errUnknownKind = errors.New("unknown kind") // ToInterface converts an IPLD node to its simplest equivalent Go value. // // Booleans, integers, floats, strings, bytes, and links are returned as themselves, // as per the node's AsT method. Note that nulls are returned as untyped nils. // // Lists and maps are returned as []interface{} and map[string]interface{}, respectively. func ToInterface(node datamodel.Node) (interface{}, error) { switch k := node.Kind(); k { case datamodel.Kind_Invalid: return nil, errInvalidKind case datamodel.Kind_Null: return nil, nil case datamodel.Kind_Bool: return node.AsBool() case datamodel.Kind_Int: return node.AsInt() case datamodel.Kind_Float: return node.AsFloat() case datamodel.Kind_String: return node.AsString() case datamodel.Kind_Bytes: return node.AsBytes() case datamodel.Kind_Link: return node.AsLink() case datamodel.Kind_Map: outMap := make(map[string]interface{}, node.Length()) for mi := node.MapIterator(); !mi.Done(); { k, v, err := mi.Next() if err != nil { return nil, err } kVal, err := k.AsString() if err != nil { return nil, err } vVal, err := ToInterface(v) if err != nil { return nil, err } outMap[kVal] = vVal } return outMap, nil case datamodel.Kind_List: outList := make([]interface{}, 0, node.Length()) for li := node.ListIterator(); !li.Done(); { _, v, err := li.Next() if err != nil { return nil, err } vVal, err := ToInterface(v) if err != nil { return nil, err } outList = append(outList, vVal) } return outList, nil default: return nil, errUnknownKind } } ================================================ FILE: fluent/toInterfaceValue_test.go ================================================ package fluent_test import ( "testing" qt "github.com/frankban/quicktest" "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime/fluent" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipld/go-ipld-prime/node/basicnode" ) var roundTripTestCases = []struct { name string value interface{} }{ {name: "Number", value: int64(100)}, {name: "String", value: "hi"}, {name: "Bool", value: true}, {name: "Bytes", value: []byte("hi")}, {name: "Map", value: map[string]interface{}{"a": "1", "b": int64(2), "c": 3.14, "d": true}}, {name: "Array", value: []interface{}{"a", "b", "c"}}, {name: "Nil", value: nil}, } func TestRoundTrip(t *testing.T) { for _, testCase := range roundTripTestCases { t.Run(testCase.name, func(t *testing.T) { c := qt.New(t) n, err := fluent.Reflect(basicnode.Prototype.Any, testCase.value) c.Assert(err, qt.IsNil) out, err := fluent.ToInterface(n) c.Assert(err, qt.IsNil) c.Check(out, qt.DeepEquals, testCase.value) }) } } func TestLink(t *testing.T) { c := qt.New(t) someCid, err := cid.Parse("bafybeihrqe2hmfauph5yfbd6ucv7njqpiy4tvbewlvhzjl4bhnyiu6h7pm") c.Assert(err, qt.IsNil) link := cidlink.Link{Cid: someCid} v, err := fluent.ToInterface(basicnode.NewLink(link)) c.Assert(err, qt.IsNil) c.Assert(v.(cidlink.Link), qt.Equals, link) } ================================================ FILE: go.mod ================================================ module github.com/ipld/go-ipld-prime go 1.25.7 require ( github.com/frankban/quicktest v1.14.6 github.com/google/go-cmp v0.7.0 github.com/ipfs/go-cid v0.6.1 github.com/multiformats/go-multicodec v0.10.0 github.com/multiformats/go-multihash v0.2.3 github.com/polydawn/refmt v0.89.1-0.20231129105047-37766d95467a github.com/warpfork/go-testmark v0.12.1 gopkg.in/yaml.v2 v2.4.0 ) require ( github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/minio/sha256-simd v1.0.0 // indirect github.com/mr-tron/base58 v1.3.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect github.com/multiformats/go-multibase v0.3.0 // indirect github.com/multiformats/go-varint v0.1.0 // indirect github.com/rogpeppe/go-internal v1.9.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect golang.org/x/crypto v0.50.0 // indirect golang.org/x/sys v0.43.0 // indirect lukechampine.com/blake3 v1.1.6 // indirect ) ================================================ FILE: go.sum ================================================ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/go-cid v0.6.1 h1:T5TnNb08+ueovG76Z5gx1L4Y7QOaGTXHg1F6raWFxIc= github.com/ipfs/go-cid v0.6.1/go.mod h1:zrY0SwOhjrrIdfPQ/kf+k1sXyJ0QE7cMxfCployLBs0= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mr-tron/base58 v1.3.0 h1:K6Y13R2h+dku0wOqKtecgRnBUBPrZzLZy5aIj8lCcJI= github.com/mr-tron/base58 v1.3.0/go.mod h1:2BuubE67DCSWwVfx37JWNG8emOC0sHEU4/HpcYgCLX8= github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= github.com/multiformats/go-multibase v0.3.0 h1:8helZD2+4Db7NNWFiktk2NePbF0boolBe6bDQvM4r68= github.com/multiformats/go-multibase v0.3.0/go.mod h1:MoBLQPCkRTOL3eveIPO81860j2AQY8JwcnNlRkGRUfI= github.com/multiformats/go-multicodec v0.10.0 h1:UpP223cig/Cx8J76jWt91njpK3GTAO1w02sdcjZDSuc= github.com/multiformats/go-multicodec v0.10.0/go.mod h1:wg88pM+s2kZJEQfRCKBNU+g32F5aWBEjyFHXvZLTcLI= github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= github.com/multiformats/go-varint v0.1.0 h1:i2wqFp4sdl3IcIxfAonHQV9qU5OsZ4Ts9IOoETFs5dI= github.com/multiformats/go-varint v0.1.0/go.mod h1:5KVAVXegtfmNQQm/lCY+ATvDzvJJhSkUlGQV9wgObdI= github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/sourcemap v0.0.0-20200213170602-2833bce08e4c/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/polydawn/refmt v0.89.1-0.20231129105047-37766d95467a h1:cgqrm0F3zwf9IPzca7xN4w+Zy6MC9ZkPvAC8QEWa/iQ= github.com/polydawn/refmt v0.89.1-0.20231129105047-37766d95467a/go.mod h1:ocZfO/tLSHqfScRDNTJbAJR1by4D1lewauX9OwTaPuY= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shurcooL/go v0.0.0-20200502201357-93f07166e636/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY= github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/warpfork/go-fsx v0.3.0/go.mod h1:oTACCMj+Zle+vgVa5SAhGAh7WksYpLgGUCKEAVc+xPg= github.com/warpfork/go-testmark v0.12.1 h1:rMgCpJfwy1sJ50x0M0NgyphxYYPMOODIJHhsXyEHU0s= github.com/warpfork/go-testmark v0.12.1/go.mod h1:kHwy7wfvGSPh1rQJYKayD4AbtNaeyZdcGi9tNJTaa5Y= github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ= github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= lukechampine.com/blake3 v1.1.6 h1:H3cROdztr7RCfoaTpGZFQsrqvweFLrqS73j7L7cmR5c= lukechampine.com/blake3 v1.1.6/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= ================================================ FILE: linking/cid/HACKME.md ================================================ Why does this package exist? ---------------------------- The `linking/cid` package bends the `github.com/ipfs/go-cid` package into conforming to the `ipld.Link` interface. The `linking/cid` package also contains factory functions for `ipld.LinkSystem`. These LinkSystem will be constructed with `EncoderChooser`, `DecoderChooser`, and `HasherChooser` funcs which will use multicodec registries and multihash registries respectively. ### Why not use go-cid directly? We need a "Link" interface in the root `ipld` package or things just aren't definable. But we don't want the root `ipld.Link` concept to directly map to `go-cid.Cid` for several reasons: 1. We might want to revisit the go-cid library. Possibly in the "significantly breaking changes" sense. - It's also not clear when we might do this -- and if we do, the transition period will be *long* because it's a highly-depended-upon library. - See below for some links to a gist that discusses why. 2. We might want to extend the concept of linking to more than just plain CIDs. - This is hypothetical at present -- but an often-discussed example is "what if CID+Path was also a Link?" 3. We might sometimes want to use IPLD libraries without using any CID implementation at all. - e.g. it's totally believable to want to use IPLD libraries for handling JSON and CBOR, even if you don't want IPLD linking. - if the CID packages were cheap enough, maybe this concern would fade -- but right now, they're **definitely** not; the transitive dependency tree of go-cid is *huge*. #### If go-cid is revisited, what might that look like? No idea. (At least, not in a committal way.) https://gist.github.com/warpfork/e871b7fee83cb814fb1f043089983bb3#existing-implementations gathers some reflections on the problems that would be nice to solve, though. https://gist.github.com/warpfork/e871b7fee83cb814fb1f043089983bb3#file-cid-go contains a draft outline of what a revisited API could look like, but note that at the time of writing, it is not strongly ratified nor in any way committed to. At any rate, though, the operative question for this package is: if we do revisit go-cid, how are we going to make the transition manageable? It seems unlikely we'd be able to make the transition manageable without some interface, somewhere. So we might as well draw that line at `ipld.Link`. (I hypothesize that a transition story might involve two CID packages, which could grow towards a shared interface, doing so in a way that's purely additive in the established `go-cid` package. We'd need two separate go modules to do this, since the aim is reducing dependency bloat for those that use the new one. The shared interface in this story could have more info than `ipld.Link` does now, but would nonetheless still certainly be an interface in order to support the separation of modules.) ### Why are LinkSystem factory functions here, instead of in the main IPLD package? Same reason as why we don't use go-cid directly. If we put these LinkSystem defaults in the root `ipld` package, we'd bring on all the transitive dependencies of `go-cid` onto an user of `ipld` unconditionally... and we don't want to do that. You know that Weird Al song "It's all about the pentiums"? Retune that in your mind to "It's all about dependencies". ================================================ FILE: linking/cid/cidLink.go ================================================ package cidlink import ( "fmt" cid "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime/datamodel" multihash "github.com/multiformats/go-multihash" ) var ( _ datamodel.Link = Link{} _ datamodel.LinkPrototype = LinkPrototype{} ) // Link implements the datamodel.Link interface using a CID. // See https://github.com/ipfs/go-cid for more information about CIDs. // // When using this value, typically you'll use it as `Link`, and not `*Link`. // This includes when handling the value as an `datamodel.Link` interface -- the non-pointer form is typically preferable. // This is because the datamodel.Link interface is often desirable to be able to use as a golang map key, // and in that context, pointers would not result in the desired behavior. type Link struct { cid.Cid } func (lnk Link) Prototype() datamodel.LinkPrototype { return LinkPrototype{lnk.Cid.Prefix()} } func (lnk Link) String() string { return lnk.Cid.String() } func (lnk Link) Binary() string { return lnk.Cid.KeyString() } type LinkPrototype struct { cid.Prefix } func (lp LinkPrototype) BuildLink(hashsum []byte) datamodel.Link { // Does this method body look surprisingly complex? I agree. // We actually have to do all this work. The go-cid package doesn't expose a constructor that just lets us directly set the bytes and the prefix numbers next to each other. // No, `cid.Prefix.Sum` is not the method you are looking for: that expects the whole data body. // Most of the logic here is the same as the body of `cid.Prefix.Sum`; we just couldn't get at the relevant parts without copypasta. // There is also some logic that's sort of folded in from the go-multihash module. This is really a mess. // The go-cid package needs review. So does go-multihash. Their responsibilies are not well compartmentalized and they don't play well with other stdlib golang interfaces. p := lp.Prefix length := p.MhLength if p.MhType == multihash.IDENTITY { length = -1 } if p.Version == 0 && (p.MhType != multihash.SHA2_256 || (p.MhLength != 32 && p.MhLength != -1)) { panic(fmt.Errorf("invalid cid v0 prefix")) } if length != -1 { hashsum = hashsum[:p.MhLength] } mh, err := multihash.Encode(hashsum, p.MhType) if err != nil { panic(err) // No longer possible, but multihash still returns an error for legacy reasons. } switch lp.Prefix.Version { case 0: return Link{cid.NewCidV0(mh)} case 1: return Link{cid.NewCidV1(p.Codec, mh)} default: panic(fmt.Errorf("invalid cid version")) } } ================================================ FILE: linking/cid/linksystem.go ================================================ package cidlink import ( "fmt" "hash" multihash "github.com/multiformats/go-multihash/core" "github.com/ipld/go-ipld-prime/codec" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/linking" "github.com/ipld/go-ipld-prime/multicodec" ) // DefaultLinkSystem returns a linking.LinkSystem which uses cidlink.Link for datamodel.Link. // During selection of encoders, decoders, and hashers, it examines the multicodec indicator numbers and multihash indicator numbers from the CID, // and uses the default global multicodec registry (see the go-ipld-prime/multicodec package) for resolving codec implementations, // and the default global multihash registry (see the go-multihash/core package) for resolving multihash implementations. // // No storage functions are present in the returned LinkSystem. // The caller can assign those themselves as desired. func DefaultLinkSystem() linking.LinkSystem { return LinkSystemUsingMulticodecRegistry(multicodec.DefaultRegistry) } // LinkSystemUsingMulticodecRegistry is similar to DefaultLinkSystem, but accepts a multicodec.Registry as a parameter. // // This can help create a LinkSystem which uses different multicodec implementations than the global registry. // (Sometimes this can be desired if you want some parts of a program to support a more limited suite of codecs than other parts of the program, // or needed to use a different multicodec registry than the global one for synchronization purposes, or etc.) func LinkSystemUsingMulticodecRegistry(mcReg multicodec.Registry) linking.LinkSystem { return linking.LinkSystem{ EncoderChooser: func(lp datamodel.LinkPrototype) (codec.Encoder, error) { switch lp2 := lp.(type) { case LinkPrototype: fn, err := mcReg.LookupEncoder(lp2.GetCodec()) if err != nil { return nil, err } return fn, nil default: return nil, fmt.Errorf("this encoderChooser can only handle cidlink.LinkPrototype; got %T", lp) } }, DecoderChooser: func(lnk datamodel.Link) (codec.Decoder, error) { lp := lnk.Prototype() switch lp2 := lp.(type) { case LinkPrototype: fn, err := mcReg.LookupDecoder(lp2.GetCodec()) if err != nil { return nil, err } return fn, nil default: return nil, fmt.Errorf("this decoderChooser can only handle cidlink.LinkPrototype; got %T", lp) } }, HasherChooser: func(lp datamodel.LinkPrototype) (hash.Hash, error) { switch lp2 := lp.(type) { case LinkPrototype: h, err := multihash.GetHasher(lp2.MhType) if err != nil { return nil, fmt.Errorf("no hasher registered for multihash indicator 0x%x: %w", lp2.MhType, err) } return h, nil default: return nil, fmt.Errorf("this hasherChooser can only handle cidlink.LinkPrototype; got %T", lp) } }, } } ================================================ FILE: linking/cid/memorystorage.go ================================================ package cidlink import ( "bytes" "fmt" "io" "os" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/linking" ) // Memory is a simple in-memory storage for cidlinks. It's the same as `storage.Memory` // but uses typical multihash semantics used when reading/writing cidlinks. // // Using multihash as the storage key rather than the whole CID will remove the // distinction between CIDv0 and their CIDv1 counterpart. It also removes the // distinction between CIDs where the multihash is the same but the codec is // different, e.g. `dag-cbor` and a `raw` version of the same data. type Memory struct { Bag map[string][]byte } func (store *Memory) beInitialized() { if store.Bag != nil { return } store.Bag = make(map[string][]byte) } func (store *Memory) OpenRead(lnkCtx linking.LinkContext, lnk datamodel.Link) (io.Reader, error) { store.beInitialized() cl, ok := lnk.(Link) if !ok { return nil, fmt.Errorf("incompatible link type: %T", lnk) } data, exists := store.Bag[string(cl.Hash())] if !exists { return nil, os.ErrNotExist } return bytes.NewReader(data), nil } func (store *Memory) OpenWrite(lnkCtx linking.LinkContext) (io.Writer, linking.BlockWriteCommitter, error) { store.beInitialized() buf := bytes.Buffer{} return &buf, func(lnk datamodel.Link) error { cl, ok := lnk.(Link) if !ok { return fmt.Errorf("incompatible link type: %T", lnk) } store.Bag[string(cl.Hash())] = buf.Bytes() return nil }, nil } ================================================ FILE: linking/errors.go ================================================ package linking import ( "fmt" "github.com/ipld/go-ipld-prime/datamodel" ) // ErrLinkingSetup is returned by methods on LinkSystem when some part of the system is not set up correctly, // or when one of the components refuses to handle a Link or LinkPrototype given. // (It is not yielded for errors from the storage nor codec systems once they've started; those errors rise without interference.) type ErrLinkingSetup struct { Detail string // Perhaps an enum here as well, which states which internal function was to blame? Cause error } func (e ErrLinkingSetup) Error() string { return fmt.Sprintf("%s: %v", e.Detail, e.Cause) } func (e ErrLinkingSetup) Unwrap() error { return e.Cause } // ErrHashMismatch is the error returned when loading data and verifying its hash // and finding that the loaded data doesn't re-hash to the expected value. // It is typically seen returned by functions like LinkSystem.Load or LinkSystem.Fill. type ErrHashMismatch struct { Actual datamodel.Link Expected datamodel.Link } func (e ErrHashMismatch) Error() string { return fmt.Sprintf("hash mismatch! %v (actual) != %v (expected)", e.Actual, e.Expected) } ================================================ FILE: linking/functions.go ================================================ package linking import ( "bytes" "context" "io" "github.com/ipld/go-ipld-prime/datamodel" ) // This file contains all the functions on LinkSystem. // These are the helpful, user-facing functions we expect folks to use "most of the time" when loading and storing data. // Variations: // - Load vs Store vs ComputeLink // - Load vs LoadPlusRaw // - With or without LinkContext? // - Brevity would be nice but I can't think of what to name the functions, so: everything takes LinkContext. Zero value is fine though. // - [for load direction only]: Prototype (and return Node|error) or Assembler (and just return error)? // - naming: Load vs Fill. // - 'Must' variants. // Can we get as far as a `QuickLoad(lnk Link) (Node, error)` function, which doesn't even ask you for a NodePrototype? // No, not quite. (Alas.) If we tried to do so, and make it use `basicnode.Prototype`, we'd have import cycles; ded. // Load looks up some data identified by a Link, and does everything necessary to turn it into usable data. // In detail, that means it: // brings that data into memory, // verifies the hash, // parses it into the Data Model using a codec, // and returns an IPLD Node. // // Where the data will be loaded from is determined by the configuration of the LinkSystem // (namely, the StorageReadOpener callback, which can either be set directly, // or configured via the SetReadStorage function). // // The in-memory form used for the returned Node is determined by the given NodePrototype parameter. // A new builder and a new node will be allocated, via NodePrototype.NewBuilder. // (If you'd like more control over memory allocation, you may wish to see the Fill function instead.) // // A schema may also be used, and apply additional data validation during loading, // by using a schema.TypedNodePrototype as the NodePrototype argument. // // The LinkContext parameter may be used to pass contextual information down to the loading layer. // // Which hashing function is used to validate the loaded data is determined by LinkSystem.HasherChooser. // Which codec is used to parse the loaded data into the Data Model is determined by LinkSystem.DecoderChooser. // // The LinkSystem.NodeReifier callback is also applied before returning the Node, // and so Load may also thereby return an ADL. func (lsys *LinkSystem) Load(lnkCtx LinkContext, lnk datamodel.Link, np datamodel.NodePrototype) (datamodel.Node, error) { nb := np.NewBuilder() if err := lsys.Fill(lnkCtx, lnk, nb); err != nil { return nil, err } nd := nb.Build() if lsys.NodeReifier == nil { return nd, nil } return lsys.NodeReifier(lnkCtx, nd, lsys) } // MustLoad is identical to Load, but panics in the case of errors. // // This function is meant for convenience of use in test and demo code, but should otherwise probably be avoided. func (lsys *LinkSystem) MustLoad(lnkCtx LinkContext, lnk datamodel.Link, np datamodel.NodePrototype) datamodel.Node { if n, err := lsys.Load(lnkCtx, lnk, np); err != nil { panic(err) } else { return n } } // LoadPlusRaw is similar to Load, but additionally retains and returns the byte slice of the raw data parsed. // // Be wary of using this with large data, since it will hold all data in memory at once. // For more control over streaming, you may want to construct a LinkSystem where you wrap the storage opener callbacks, // and thus can access the streams (and tee them, or whatever you need to do) as they're opened. // This function is meant for convenience when data sizes are small enough that fitting them into memory at once is not a problem. func (lsys *LinkSystem) LoadPlusRaw(lnkCtx LinkContext, lnk datamodel.Link, np datamodel.NodePrototype) (datamodel.Node, []byte, error) { // Choose all the parts. decoder, err := lsys.DecoderChooser(lnk) if err != nil { return nil, nil, ErrLinkingSetup{"could not choose a decoder", err} } // Use LoadRaw to get the data. // If we're going to have everything in memory at once, we might as well do that first, and then give the codec and the hasher the whole thing at once. block, err := lsys.LoadRaw(lnkCtx, lnk) if err != nil { return nil, block, err } // Create a NodeBuilder. // Deploy the codec. // Build the node. nb := np.NewBuilder() if err := decoder(nb, bytes.NewBuffer(block)); err != nil { return nil, block, err } nd := nb.Build() // Consider applying NodeReifier, if applicable. if lsys.NodeReifier == nil { return nd, block, nil } nd, err = lsys.NodeReifier(lnkCtx, nd, lsys) return nd, block, err } // LoadRaw looks up some data identified by a Link, brings that data into memory, // verifies the hash, and returns it directly as a byte slice. // // LoadRaw does not return a data model view of the data, // nor does it verify that a codec can parse the data at all! // Use this function at your own risk; it does not provide the same guarantees as the Load or Fill functions do. func (lsys *LinkSystem) LoadRaw(lnkCtx LinkContext, lnk datamodel.Link) ([]byte, error) { if lnkCtx.Ctx == nil { lnkCtx.Ctx = context.Background() } // Choose all the parts. hasher, err := lsys.HasherChooser(lnk.Prototype()) if err != nil { return nil, ErrLinkingSetup{"could not choose a hasher", err} } if lsys.StorageReadOpener == nil { return nil, ErrLinkingSetup{"no storage configured for reading", io.ErrClosedPipe} // REVIEW: better cause? } // Open storage: get the data. // FUTURE: this could probably use storage.ReadableStorage.Get instead of streaming and a buffer, if we refactored LinkSystem to carry that interface through. reader, err := lsys.StorageReadOpener(lnkCtx, lnk) if err != nil { return nil, err } if closer, ok := reader.(io.Closer); ok { defer closer.Close() } var buf bytes.Buffer if _, err := io.Copy(&buf, reader); err != nil { return nil, err } // Compute the hash. // (Then do a bit of a jig to build a link out of it -- because that's what we do the actual hash equality check on.) hasher.Write(buf.Bytes()) hash := hasher.Sum(nil) lnk2 := lnk.Prototype().BuildLink(hash) if lnk2.Binary() != lnk.Binary() { return nil, ErrHashMismatch{Actual: lnk2, Expected: lnk} } // No codec to deploy; this is the raw load function. // So we're done. return buf.Bytes(), nil } // Fill is similar to Load, but allows more control over memory allocations. // Instead of taking a NodePrototype parameter, Fill takes a NodeAssembler parameter: // this allows you to use your own NodeBuilder (and reset it, etc, thus controlling allocations), // or, to fill in some part of a larger structure. // // Note that Fill does not regard NodeReifier, even if one has been configured. // (This is in contrast to Load, which does regard a NodeReifier if one is configured, and thus may return an ADL node). func (lsys *LinkSystem) Fill(lnkCtx LinkContext, lnk datamodel.Link, na datamodel.NodeAssembler) error { if lnkCtx.Ctx == nil { lnkCtx.Ctx = context.Background() } // Choose all the parts. decoder, err := lsys.DecoderChooser(lnk) if err != nil { return ErrLinkingSetup{"could not choose a decoder", err} } hasher, err := lsys.HasherChooser(lnk.Prototype()) if err != nil { return ErrLinkingSetup{"could not choose a hasher", err} } if lsys.StorageReadOpener == nil { return ErrLinkingSetup{"no storage configured for reading", io.ErrClosedPipe} // REVIEW: better cause? } // Open storage; get a reader stream. reader, err := lsys.StorageReadOpener(lnkCtx, lnk) if err != nil { return err } if closer, ok := reader.(io.Closer); ok { defer closer.Close() } // TrustedStorage indicates the data coming out of this reader has already been hashed and verified earlier. // As a result, we can skip rehashing it if lsys.TrustedStorage { return decoder(na, reader) } // Tee the stream so that the hasher is fed as the unmarshal progresses through the stream. tee := io.TeeReader(reader, hasher) // The actual read is then dragged forward by the codec. decodeErr := decoder(na, tee) if decodeErr != nil { // It is important to security to check the hash before returning any other observation about the content, // so, if the decode process returns any error, we have several steps to take before potentially returning it. // First, we try to copy any data remaining that wasn't already pulled through the TeeReader by the decoder, // so that the hasher can reach the end of the stream. // If _that_ errors, return the I/O level error. // We hang onto decodeErr for a while: we can't return that until all the way after we check the hash equality. _, err := io.Copy(hasher, reader) if err != nil { return err } } // Compute the hash. // (Then do a bit of a jig to build a link out of it -- because that's what we do the actual hash equality check on.) hash := hasher.Sum(nil) lnk2 := lnk.Prototype().BuildLink(hash) if lnk2.Binary() != lnk.Binary() { return ErrHashMismatch{Actual: lnk2, Expected: lnk} } // If we got all the way through IO and through the hash check: // now, finally, if we did get an error from the codec, we can admit to that. if decodeErr != nil { return decodeErr } return nil } // MustFill is identical to Fill, but panics in the case of errors. // // This function is meant for convenience of use in test and demo code, but should otherwise probably be avoided. func (lsys *LinkSystem) MustFill(lnkCtx LinkContext, lnk datamodel.Link, na datamodel.NodeAssembler) { if err := lsys.Fill(lnkCtx, lnk, na); err != nil { panic(err) } } func (lsys *LinkSystem) Store(lnkCtx LinkContext, lp datamodel.LinkPrototype, n datamodel.Node) (datamodel.Link, error) { if lnkCtx.Ctx == nil { lnkCtx.Ctx = context.Background() } // Choose all the parts. encoder, err := lsys.EncoderChooser(lp) if err != nil { return nil, ErrLinkingSetup{"could not choose an encoder", err} } hasher, err := lsys.HasherChooser(lp) if err != nil { return nil, ErrLinkingSetup{"could not choose a hasher", err} } if lsys.StorageWriteOpener == nil { return nil, ErrLinkingSetup{"no storage configured for writing", io.ErrClosedPipe} // REVIEW: better cause? } // Open storage write stream, feed serial data to the storage and the hasher, and funnel the codec output into both. writer, commitFn, err := lsys.StorageWriteOpener(lnkCtx) if err != nil { return nil, err } tee := io.MultiWriter(writer, hasher) err = encoder(n, tee) if err != nil { return nil, err } lnk := lp.BuildLink(hasher.Sum(nil)) return lnk, commitFn(lnk) } func (lsys *LinkSystem) MustStore(lnkCtx LinkContext, lp datamodel.LinkPrototype, n datamodel.Node) datamodel.Link { if lnk, err := lsys.Store(lnkCtx, lp, n); err != nil { panic(err) } else { return lnk } } // ComputeLink returns a Link for the given data, but doesn't do anything else // (e.g. it doesn't try to store any of the serial-form data anywhere else). func (lsys *LinkSystem) ComputeLink(lp datamodel.LinkPrototype, n datamodel.Node) (datamodel.Link, error) { encoder, err := lsys.EncoderChooser(lp) if err != nil { return nil, ErrLinkingSetup{"could not choose an encoder", err} } hasher, err := lsys.HasherChooser(lp) if err != nil { return nil, ErrLinkingSetup{"could not choose a hasher", err} } err = encoder(n, hasher) if err != nil { return nil, err } return lp.BuildLink(hasher.Sum(nil)), nil } func (lsys *LinkSystem) MustComputeLink(lp datamodel.LinkPrototype, n datamodel.Node) datamodel.Link { if lnk, err := lsys.ComputeLink(lp, n); err != nil { panic(err) } else { return lnk } } ================================================ FILE: linking/functions_test.go ================================================ package linking_test import ( "bytes" "context" "testing" qt "github.com/frankban/quicktest" "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/codec/dagcbor" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/fluent" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/storage/memstore" "github.com/multiformats/go-multicodec" ) func TestLinkSystem_LoadHashMismatch(t *testing.T) { subject := cidlink.DefaultLinkSystem() storage := &memstore.Store{} subject.SetReadStorage(storage) subject.SetWriteStorage(storage) // Construct some test IPLD node. wantNode := fluent.MustBuildMap(basicnode.Prototype.Map, 1, func(na fluent.MapAssembler) { na.AssembleEntry("fish").AssignString("barreleye") }) // Encode as raw value to be used for testing LoadRaw var buf bytes.Buffer qt.Check(t, dagcbor.Encode(wantNode, &buf), qt.IsNil) wantNodeRaw := buf.Bytes() // Store the test IPLD node and get link back. lctx := ipld.LinkContext{Ctx: context.TODO()} gotLink, err := subject.Store(lctx, cidlink.LinkPrototype{ Prefix: cid.Prefix{ Version: 1, Codec: uint64(multicodec.DagCbor), MhType: uint64(multicodec.Sha2_256), MhLength: -1, }, }, wantNode) qt.Check(t, err, qt.IsNil) gotCidlink := gotLink.(cidlink.Link) // Assert all load variations return expected values for different link representations. for _, test := range []struct { name string link datamodel.Link }{ {"datamodel.Link", gotLink}, {"cidlink.Link", gotCidlink}, {"&cidlink.Link", &gotCidlink}, } { t.Run(test.name, func(t *testing.T) { gotNode, err := subject.Load(lctx, test.link, basicnode.Prototype.Any) qt.Check(t, err, qt.IsNil) qt.Check(t, ipld.DeepEqual(wantNode, gotNode), qt.IsTrue) gotNodeRaw, err := subject.LoadRaw(lctx, test.link) qt.Check(t, err, qt.IsNil) qt.Check(t, bytes.Equal(wantNodeRaw, gotNodeRaw), qt.IsTrue) gotNode, gotNodeRaw, err = subject.LoadPlusRaw(lctx, test.link, basicnode.Prototype.Any) qt.Check(t, err, qt.IsNil) qt.Check(t, ipld.DeepEqual(wantNode, gotNode), qt.IsTrue) qt.Check(t, bytes.Equal(wantNodeRaw, gotNodeRaw), qt.IsTrue) }) } } ================================================ FILE: linking/linkingExamples_test.go ================================================ package linking_test import ( "fmt" "github.com/ipfs/go-cid" _ "github.com/ipld/go-ipld-prime/codec/dagcbor" "github.com/ipld/go-ipld-prime/fluent" "github.com/ipld/go-ipld-prime/linking" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/storage/memstore" ) // storage is a map where we'll store serialized IPLD data. // // ExampleLinkSystem_Store will put data into this; // ExampleLinkSystem_Load will read out from it. // // In a real program, you'll probably make functions to load and store from disk, // or some network storage, or... whatever you want, really :) var store = memstore.Store{} // TODO: These examples are really heavy on CIDs and the multicodec and multihash magic tables. // It would be good to have examples that create and use less magical LinkSystem constructions, too. func ExampleLinkSystem_Store() { // Creating a Link is done by choosing a concrete link implementation (typically, CID), // getting a LinkSystem that knows how to work with that, and then using the LinkSystem methods. // Let's get a LinkSystem. We're going to be working with CID links, // so let's get the default LinkSystem that's ready to work with those. lsys := cidlink.DefaultLinkSystem() // We want to store the serialized data somewhere. // We'll use an in-memory store for this. (It's a package scoped variable.) // You can use any kind of storage system here; // or if you need even more control, you could also write a function that conforms to the linking.BlockWriteOpener interface. lsys.SetWriteStorage(&store) // To create any links, first we need a LinkPrototype. // This gathers together any parameters that might be needed when making a link. // (For CIDs, the version, the codec, and the multihash type are all parameters we'll need.) // Often, you can probably make this a constant for your whole application. lp := cidlink.LinkPrototype{Prefix: cid.Prefix{ Version: 1, // Usually '1'. Codec: 0x71, // 0x71 means "dag-cbor" -- See the multicodecs table: https://github.com/multiformats/multicodec/ MhType: 0x13, // 0x20 means "sha2-512" -- See the multicodecs table: https://github.com/multiformats/multicodec/ MhLength: 64, // sha2-512 hash has a 64-byte sum. }} // And we need some data to link to! Here's a quick piece of example data: n := fluent.MustBuildMap(basicnode.Prototype.Map, 1, func(na fluent.MapAssembler) { na.AssembleEntry("hello").AssignString("world") }) // Before we use the LinkService, NOTE: // There's a side-effecting import at the top of the file. It's for the dag-cbor codec. // The CID LinkSystem defaults use a global registry called the multicodec table; // and the multicodec table is populated in part by the dag-cbor package when it's first imported. // You'll need that side-effecting import, too, to copy this example. // It can happen anywhere in your program; once, in any package, is enough. // If you don't have this import, the codec will not be registered in the multicodec registry, // and when you use the LinkSystem we got from the cidlink package, it will return an error of type ErrLinkingSetup. // If you initialize a custom LinkSystem, you can control this more directly; // these registry systems are only here as defaults. // Now: time to apply the LinkSystem, and do the actual store operation! lnk, err := lsys.Store( linking.LinkContext{}, // The zero value is fine. Configure it it you want cancellability or other features. lp, // The LinkPrototype says what codec and hashing to use. n, // And here's our data. ) if err != nil { panic(err) } // That's it! We got a link. fmt.Printf("link: %s\n", lnk) fmt.Printf("concrete type: `%T`\n", lnk) // Remember: the serialized data was also stored to the 'store' variable as a side-effect. // (We set this up back when we customized the LinkSystem.) // We'll pick this data back up again in the example for loading. // Output: // link: bafyrgqhai26anf3i7pips7q22coa4sz2fr4gk4q4sqdtymvvjyginfzaqewveaeqdh524nsktaq43j65v22xxrybrtertmcfxufdam3da3hbk // concrete type: `cidlink.Link` } func ExampleLinkSystem_Load() { // Let's say we want to load this link (it's the same one we created in ExampleLinkSystem_Store). cid, _ := cid.Decode("bafyrgqhai26anf3i7pips7q22coa4sz2fr4gk4q4sqdtymvvjyginfzaqewveaeqdh524nsktaq43j65v22xxrybrtertmcfxufdam3da3hbk") lnk := cidlink.Link{Cid: cid} // Let's get a LinkSystem. We're going to be working with CID links, // so let's get the default LinkSystem that's ready to work with those. // (This is the same as we did in ExampleLinkSystem_Store.) lsys := cidlink.DefaultLinkSystem() // We need somewhere to go looking for any of the data we might want to load! // We'll use an in-memory store for this. (It's a package scoped variable.) // (This particular memory store was filled with the data we'll load earlier, during ExampleLinkSystem_Store.) // You can use any kind of storage system here; // or if you need even more control, you could also write a function that conforms to the linking.BlockReadOpener interface. lsys.SetReadStorage(&store) // We'll need to decide what in-memory implementation of datamodel.Node we want to use. // Here, we'll use the "basicnode" implementation. This is a good getting-started choice. // But you could also use other implementations, or even a code-generated type with special features! np := basicnode.Prototype.Any // Before we use the LinkService, NOTE: // There's a side-effecting import at the top of the file. It's for the dag-cbor codec. // See the comments in ExampleLinkSystem_Store for more discussion of this and why it's important. // Apply the LinkSystem, and ask it to load our link! n, err := lsys.Load( linking.LinkContext{}, // The zero value is fine. Configure it it you want cancellability or other features. lnk, // The Link we want to load! np, // The NodePrototype says what kind of Node we want as a result. ) if err != nil { panic(err) } // Tada! We have the data as node that we can traverse and use as desired. fmt.Printf("we loaded a %s with %d entries\n", n.Kind(), n.Length()) // Output: // we loaded a map with 1 entries } ================================================ FILE: linking/preload/preload.go ================================================ package preload import ( "context" "github.com/ipld/go-ipld-prime/datamodel" ) // Loader is a function that will be called with a link discovered in a preload // pass of a traversal. A preload pass can be used to collect all links in each // block prior to traversal of that block, allowing for parallel (background) // loading of blocks in anticipation of eventual actual load during traversal. type Loader func(PreloadContext, Link) // PreloadContext carries information about the current state of a traversal // where a set of links that may be preloaded were encountered. type PreloadContext struct { // Ctx is the familiar golang Context pattern. // Use this for cancellation, or attaching additional info // (for example, perhaps to pass auth tokens through to the storage functions). Ctx context.Context // Path where the link was encountered. May be zero. // // Functions in the traversal package will set this automatically. BasePath datamodel.Path // Parent of the LinkNode. May be zero. // // Functions in the traversal package will set this automatically. ParentNode datamodel.Node } // Link provides the link encountered during a preload pass, the node it was // encountered on, and the segment of the path that led to the link. type Link struct { Segment datamodel.PathSegment LinkNode datamodel.Node Link datamodel.Link } ================================================ FILE: linking/setup.go ================================================ package linking import ( "io" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/storage" ) // SetReadStorage configures how the LinkSystem will look for information to load, // setting it to look at the given storage.ReadableStorage. // // This will overwrite the LinkSystem.StorageReadOpener field. // // This mechanism only supports setting exactly one ReadableStorage. // If you would like to make a more complex configuration // (for example, perhaps using information from a LinkContext to decide which storage area to use?) // then you should set LinkSystem.StorageReadOpener to a custom callback of your own creation instead. func (lsys *LinkSystem) SetReadStorage(store storage.ReadableStorage) { lsys.StorageReadOpener = func(lctx LinkContext, lnk datamodel.Link) (io.Reader, error) { return storage.GetStream(lctx.Ctx, store, lnk.Binary()) } } // SetWriteStorage configures how the LinkSystem will store information, // setting it to write into the given storage.WritableStorage. // // This will overwrite the LinkSystem.StorageWriteOpener field. // // This mechanism only supports setting exactly one WritableStorage. // If you would like to make a more complex configuration // (for example, perhaps using information from a LinkContext to decide which storage area to use?) // then you should set LinkSystem.StorageWriteOpener to a custom callback of your own creation instead. func (lsys *LinkSystem) SetWriteStorage(store storage.WritableStorage) { lsys.StorageWriteOpener = func(lctx LinkContext) (io.Writer, BlockWriteCommitter, error) { wr, wrcommit, err := storage.PutStream(lctx.Ctx, store) return wr, func(lnk datamodel.Link) error { return wrcommit(lnk.Binary()) }, err } } ================================================ FILE: linking/types.go ================================================ package linking import ( "context" "hash" "io" "github.com/ipld/go-ipld-prime/codec" "github.com/ipld/go-ipld-prime/datamodel" ) // LinkSystem is a struct that composes all the individual functions // needed to load and store content addressed data using IPLD -- // encoding functions, hashing functions, and storage connections -- // and then offers the operations a user wants -- Store and Load -- as methods. // // Typically, the functions which are fields of LinkSystem are not used // directly by users (except to set them, when creating the LinkSystem), // and it's the higher level operations such as Store and Load that user code then calls. // // The most typical way to get a LinkSystem is from the linking/cid package, // which has a factory function called DefaultLinkSystem. // The LinkSystem returned by that function will be based on CIDs, // and use the multicodec registry and multihash registry to select encodings and hashing mechanisms. // The BlockWriteOpener and BlockReadOpener must still be provided by the user; // otherwise, only the ComputeLink method will work. // // Some implementations of BlockWriteOpener and BlockReadOpener may be // found in the storage package. Applications are also free to write their own. // Custom wrapping of BlockWriteOpener and BlockReadOpener are also common, // and may be reasonable if one wants to build application features that are block-aware. type LinkSystem struct { EncoderChooser func(datamodel.LinkPrototype) (codec.Encoder, error) DecoderChooser func(datamodel.Link) (codec.Decoder, error) HasherChooser func(datamodel.LinkPrototype) (hash.Hash, error) StorageWriteOpener BlockWriteOpener StorageReadOpener BlockReadOpener TrustedStorage bool NodeReifier NodeReifier KnownReifiers map[string]NodeReifier } // The following three types are the key functionality we need from a "blockstore". // // Some libraries might provide a "blockstore" object that has these as methods; // it may also have more methods (like enumeration features, GC features, etc), // but IPLD doesn't generally concern itself with those. // We just need these key things, so we can "put" and "get". // // The functions are a tad more complicated than "put" and "get" so that they have good mechanical sympathy. // In particular, the writing/"put" side is broken into two phases, so that the abstraction // makes it easy to begin to write data before the hash that will identify it is fully computed. type ( // BlockReadOpener defines the shape of a function used to // open a reader for a block of data. // // In a content-addressed system, the Link parameter should be only // determiner of what block body is returned. // // The LinkContext may be zero, or may be used to carry extra information: // it may be used to carry info which hints at different storage pools; // it may be used to carry authentication data; etc. // (Any such behaviors are something that a BlockReadOpener implementation // will needs to document at a higher detail level than this interface specifies. // In this interface, we can only note that it is possible to pass such information opaquely // via the LinkContext or by attachments to the general-purpose Context it contains.) // The LinkContext should not have effect on the block body returned, however; // at most should only affect data availability // (e.g. whether any block body is returned, versus an error). // // Reads are cancellable by cancelling the LinkContext.Context. // // Other parts of the IPLD library suite (such as the traversal package, and all its functions) // will typically take a Context as a parameter or piece of config from the caller, // and will pass that down through the LinkContext, meaning this can be used to // carry information as well as cancellation control all the way through the system. // // BlockReadOpener is typically not used directly, but is instead // composed in a LinkSystem and used via the methods of LinkSystem. // LinkSystem methods will helpfully handle the entire process of opening block readers, // verifying the hash of the data stream, and applying a Decoder to build Nodes -- all as one step. // // BlockReadOpener implementations are not required to validate that // the contents which will be streamed out of the reader actually match // and hash in the Link parameter before returning. // (This is something that the LinkSystem composition will handle if you're using it.) // // BlockReadOpener can also be created out of storage.ReadableStorage and attached to a LinkSystem // via the LinkSystem.SetReadStorage method. // // Users of a BlockReadOpener function should also check the io.Reader // for matching the io.Closer interface, and use the Close function as appropriate if present. BlockReadOpener func(LinkContext, datamodel.Link) (io.Reader, error) // BlockWriteOpener defines the shape of a function used to open a writer // into which data can be streamed, and which will eventually be "commited". // Committing is done using the BlockWriteCommitter returned by using the BlockWriteOpener, // and finishes the write along with requiring stating the Link which should identify this data for future reading. // // The LinkContext may be zero, or may be used to carry extra information: // it may be used to carry info which hints at different storage pools; // it may be used to carry authentication data; etc. // // Writes are cancellable by cancelling the LinkContext.Context. // // Other parts of the IPLD library suite (such as the traversal package, and all its functions) // will typically take a Context as a parameter or piece of config from the caller, // and will pass that down through the LinkContext, meaning this can be used to // carry information as well as cancellation control all the way through the system. // // BlockWriteOpener is typically not used directly, but is instead // composed in a LinkSystem and used via the methods of LinkSystem. // LinkSystem methods will helpfully handle the entire process of traversing a Node tree, // encoding this data, hashing it, streaming it to the writer, and committing it -- all as one step. // // BlockWriteOpener implementations are expected to start writing their content immediately, // and later, the returned BlockWriteCommitter should also be able to expect that // the Link which it is given is a reasonable hash of the content. // (To give an example of how this might be efficiently implemented: // One might imagine that if implementing a disk storage mechanism, // the io.Writer returned from a BlockWriteOpener will be writing a new tempfile, // and when the BlockWriteCommiter is called, it will flush the writes // and then use a rename operation to place the tempfile in a permanent path based the Link.) // // BlockWriteOpener can also be created out of storage.WritableStorage and attached to a LinkSystem // via the LinkSystem.SetWriteStorage method. BlockWriteOpener func(LinkContext) (io.Writer, BlockWriteCommitter, error) // BlockWriteCommitter defines the shape of a function which, together // with BlockWriteOpener, handles the writing and "committing" of a write // to a content-addressable storage system. // // BlockWriteCommitter is a function which is will be called at the end of a write process. // It should flush any buffers and close the io.Writer which was // made available earlier from the BlockWriteOpener call that also returned this BlockWriteCommitter. // // BlockWriteCommitter takes a Link parameter. // This Link is expected to be a reasonable hash of the content, // so that the BlockWriteCommitter can use this to commit the data to storage // in a content-addressable fashion. // See the documentation of BlockWriteOpener for more description of this // and an example of how this is likely to be reduced to practice. BlockWriteCommitter func(datamodel.Link) error // NodeReifier defines the shape of a function that given a node with no schema // or a basic schema, constructs Advanced Data Layout node // // The LinkSystem itself is passed to the NodeReifier along with a link context // because Node interface methods on an ADL may actually traverse links to other // pieces of context addressed data that need to be loaded with the Link system // // A NodeReifier return one of three things: // - original node, no error = no reification occurred, just use original node // - reified node, no error = the simple node was converted to an ADL // - nil, error = the simple node should have been converted to an ADL but something // went wrong when we tried to do so // NodeReifier func(LinkContext, datamodel.Node, *LinkSystem) (datamodel.Node, error) ) // LinkContext is a structure carrying ancilary information that may be used // while loading or storing data -- see its usage in BlockReadOpener, BlockWriteOpener, // and in the methods on LinkSystem which handle loading and storing data. // // A zero value for LinkContext is generally acceptable in any functions that use it. // In this case, any operations that need a context.Context will quietly use Context.Background // (thus being uncancellable) and simply have no additional information to work with. type LinkContext struct { // Ctx is the familiar golang Context pattern. // Use this for cancellation, or attaching additional info // (for example, perhaps to pass auth tokens through to the storage functions). Ctx context.Context // Path where the link was encountered. May be zero. // // Functions in the traversal package will set this automatically. LinkPath datamodel.Path // When traversing data or encoding: the Node containing the link -- // it may have additional type info, etc, that can be accessed. // When building / decoding: not present. // // Functions in the traversal package will set this automatically. LinkNode datamodel.Node // When building data or decoding: the NodeAssembler that will be receiving the link -- // it may have additional type info, etc, that can be accessed. // When traversing / encoding: not present. // // Functions in the traversal package will set this automatically. LinkNodeAssembler datamodel.NodeAssembler // Parent of the LinkNode. May be zero. // // Functions in the traversal package will set this automatically. ParentNode datamodel.Node // REVIEW: ParentNode in LinkContext -- so far, this has only ever been hypothetically useful. Keep or drop? } ================================================ FILE: linking.go ================================================ package ipld import ( "github.com/ipld/go-ipld-prime/linking" ) type ( LinkSystem = linking.LinkSystem LinkContext = linking.LinkContext ) type ( BlockReadOpener = linking.BlockReadOpener BlockWriteOpener = linking.BlockWriteOpener BlockWriteCommitter = linking.BlockWriteCommitter NodeReifier = linking.NodeReifier ) ================================================ FILE: multicodec/defaultRegistry.go ================================================ package multicodec import ( "github.com/ipld/go-ipld-prime/codec" ) // DefaultRegistry is a multicodec.Registry instance which is global to the program, // and is used as a default set of codecs. // // Some systems (for example, cidlink.DefaultLinkSystem) will use this default registry, // which makes it easier to write programs that pass fewer explicit arguments around. // However, these are *only* for default behaviors; // variations of functions which allow explicit non-default options should always be available // (for example, cidlink also has other LinkSystem constructor functions which accept an explicit multicodec.Registry, // and the LookupEncoder and LookupDecoder functions in any LinkSystem can be replaced). // // Since this registry is global, mind that there are also some necessary tradeoffs and limitations: // It can be difficult to control exactly what's present in this global registry // (Libraries may register codecs in this registry as a side-effect of importing, so even transitive dependencies can affect its content!). // Also, this registry is only considered safe to modify at package init time. // If these are concerns for your program, you can create your own multicodec.Registry values, // and eschew using the global default. var DefaultRegistry = Registry{} // RegisterEncoder updates the global DefaultRegistry to map a multicodec indicator number to the given codec.Encoder function. // The encoder functions registered can be subsequently looked up using LookupEncoder. // It is a shortcut to the RegisterEncoder method on the global DefaultRegistry. // // Packages which implement an IPLD codec and have a multicodec number associated with them // are encouraged to register themselves at package init time using this function. // (Doing this at package init time ensures the default global registry is populated // without causing race conditions for application code.) // // No effort is made to detect conflicting registrations in this map. // If your dependency tree is such that this becomes a problem, // there are two ways to address this: // If RegisterEncoder is called with the same indicator code more than once, the last call wins. // In practice, this means that if an application has a strong opinion about what implementation for a certain codec, // then this can be done by making a Register call with that effect at init time in the application's main package. // This should have the desired effect because the root of the import tree has its init time effect last. // Alternatively, one can just avoid use of this registry entirely: // do this by making a LinkSystem that uses a custom EncoderChooser function. func RegisterEncoder(indicator uint64, encodeFunc codec.Encoder) { DefaultRegistry.RegisterEncoder(indicator, encodeFunc) } // LookupEncoder yields a codec.Encoder function matching a multicodec indicator code number. // It is a shortcut to the LookupEncoder method on the global DefaultRegistry. // // To be available from this lookup function, an encoder must have been registered // for this indicator number by an earlier call to the RegisterEncoder function. func LookupEncoder(indicator uint64) (codec.Encoder, error) { return DefaultRegistry.LookupEncoder(indicator) } // ListEncoders returns a list of multicodec indicators for which a codec.Encoder is registered. // The list is in no particular order. // It is a shortcut to the ListEncoders method on the global DefaultRegistry. // // Be judicious about trying to use this function outside of debugging. // Because the global default registry is global and easily modified, // and can be changed by any of the transitive dependencies of your program, // its contents are not particularly stable. // In particular, it is not recommended to make any behaviors of your program conditional // based on information returned by this function -- if your program needs conditional // behavior based on registered codecs, you may want to consider taking more explicit control // and using your own non-default registry. func ListEncoders() []uint64 { return DefaultRegistry.ListEncoders() } // RegisterDecoder updates the global DefaultRegistry a map a multicodec indicator number to the given codec.Decoder function. // The decoder functions registered can be subsequently looked up using LookupDecoder. // It is a shortcut to the RegisterDecoder method on the global DefaultRegistry. // // Packages which implement an IPLD codec and have a multicodec number associated with them // are encouraged to register themselves in this map at package init time. // (Doing this at package init time ensures the default global registry is populated // without causing race conditions for application code.) // // No effort is made to detect conflicting registrations in this map. // If your dependency tree is such that this becomes a problem, // there are two ways to address this: // If RegisterDecoder is called with the same indicator code more than once, the last call wins. // In practice, this means that if an application has a strong opinion about what implementation for a certain codec, // then this can be done by making a Register call with that effect at init time in the application's main package. // This should have the desired effect because the root of the import tree has its init time effect last. // Alternatively, one can just avoid use of this registry entirely: // do this by making a LinkSystem that uses a custom DecoderChooser function. func RegisterDecoder(indicator uint64, decodeFunc codec.Decoder) { DefaultRegistry.RegisterDecoder(indicator, decodeFunc) } // LookupDecoder yields a codec.Decoder function matching a multicodec indicator code number. // It is a shortcut to the LookupDecoder method on the global DefaultRegistry. // // To be available from this lookup function, an decoder must have been registered // for this indicator number by an earlier call to the RegisterDecoder function. func LookupDecoder(indicator uint64) (codec.Decoder, error) { return DefaultRegistry.LookupDecoder(indicator) } // ListDecoders returns a list of multicodec indicators for which a codec.Decoder is registered. // The list is in no particular order. // It is a shortcut to the ListDecoders method on the global DefaultRegistry. // // Be judicious about trying to use this function outside of debugging. // Because the global default registry is global and easily modified, // and can be changed by any of the transitive dependencies of your program, // its contents are not particularly stable. // In particular, it is not recommended to make any behaviors of your program conditional // based on information returned by this function -- if your program needs conditional // behavior based on registered codecs, you may want to consider taking more explicit control // and using your own non-default registry. func ListDecoders() []uint64 { return DefaultRegistry.ListDecoders() } ================================================ FILE: multicodec/registry.go ================================================ package multicodec import ( "fmt" "github.com/ipld/go-ipld-prime/codec" ) // Registry is a structure for storing mappings of multicodec indicator numbers to codec.Encoder and codec.Decoder functions. // // The most typical usage of this structure is in combination with a codec.LinkSystem. // For example, a linksystem using CIDs and a custom multicodec registry can be constructed // using cidlink.LinkSystemUsingMulticodecRegistry. // // Registry includes no mutexing. If using Registry in a concurrent context, you must handle synchronization yourself. // (Typically, it is recommended to do initialization earlier in a program, before fanning out goroutines; // this avoids the need for mutexing overhead.) // // go-ipld also has a default registry, which has the same methods as this structure, but are at package scope. // Some systems, like cidlink.DefaultLinkSystem, will use this default registry. // However, this default registry is global to the entire program. // This Registry type is for helping if you wish to make your own registry which does not share that global state. // // Multicodec indicator numbers are specified in // https://github.com/multiformats/multicodec/blob/master/table.csv . // You should not use indicator numbers which are not specified in that table // (however, there is nothing in this implementation that will attempt to stop you, either; please behave). type Registry struct { encoders map[uint64]codec.Encoder decoders map[uint64]codec.Decoder } func (r *Registry) ensureInit() { if r.encoders != nil { return } r.encoders = make(map[uint64]codec.Encoder) r.decoders = make(map[uint64]codec.Decoder) } // RegisterEncoder updates a simple map of multicodec indicator number to codec.Encoder function. // The encoder functions registered can be subsequently looked up using LookupEncoder. func (r *Registry) RegisterEncoder(indicator uint64, encodeFunc codec.Encoder) { r.ensureInit() if encodeFunc == nil { panic("not sensible to attempt to register a nil function") } r.encoders[indicator] = encodeFunc } // LookupEncoder yields a codec.Encoder function matching a multicodec indicator code number. // // To be available from this lookup function, an encoder must have been registered // for this indicator number by an earlier call to the RegisterEncoder function. func (r *Registry) LookupEncoder(indicator uint64) (codec.Encoder, error) { encodeFunc, exists := r.encoders[indicator] if !exists { return nil, fmt.Errorf("no encoder registered for multicodec code %d (0x%x)", indicator, indicator) } return encodeFunc, nil } // ListEncoders returns a list of multicodec indicators for which a codec.Encoder is registered. // The list is in no particular order. func (r *Registry) ListEncoders() []uint64 { encoders := make([]uint64, 0, len(r.encoders)) for e := range r.encoders { encoders = append(encoders, e) } return encoders } // TODO(mvdan): turn most of these uint64s into multicodec.Code // RegisterDecoder updates a simple map of multicodec indicator number to codec.Decoder function. // The decoder functions registered can be subsequently looked up using LookupDecoder. func (r *Registry) RegisterDecoder(indicator uint64, decodeFunc codec.Decoder) { r.ensureInit() if decodeFunc == nil { panic("not sensible to attempt to register a nil function") } r.decoders[indicator] = decodeFunc } // LookupDecoder yields a codec.Decoder function matching a multicodec indicator code number. // // To be available from this lookup function, an decoder must have been registered // for this indicator number by an earlier call to the RegisterDecoder function. func (r *Registry) LookupDecoder(indicator uint64) (codec.Decoder, error) { decodeFunc, exists := r.decoders[indicator] if !exists { return nil, fmt.Errorf("no decoder registered for multicodec code %d (0x%x)", indicator, indicator) } return decodeFunc, nil } // ListDecoders returns a list of multicodec indicators for which a codec.Decoder is registered. // The list is in no particular order. func (r *Registry) ListDecoders() []uint64 { decoders := make([]uint64, 0, len(r.decoders)) for d := range r.decoders { decoders = append(decoders, d) } return decoders } ================================================ FILE: must/must.go ================================================ // Package 'must' provides another alternative to the 'fluent' package, // providing many helpful functions for wrapping methods with multiple returns // into a single return (converting errors into panics). // // It's useful especially for testing code and other situations where panics // are not problematic. // // Unlike the 'fluent' package, panics are not of any particular type. // There is no equivalent to the `fluent.Recover` feature in the 'must' package. // // Because golang supports implied destructuring of multiple-return functions // into arguments for another funtion of matching arity, most of the 'must' // functions can used smoothly in a pointfree/chainable form, like this: // // must.Node(SomeNodeBuilder{}.CreateString("a")) package must import ( "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/schema" ) // must.NotError simply panics if given an error. // It helps turn multi-line code into one-liner code in situations where // you simply don't care. func NotError(e error) { if e != nil { panic(e) } } // must.Node helps write pointfree/chainable-style code // by taking a Node and an error and transforming any error into a panic. // // Because golang supports implied destructuring of multiple-return functions // into arguments for another funtion of matching arity, it can be used like this: // // must.Node(SomeNodeBuilder{}.CreateString("a")) func Node(n datamodel.Node, e error) datamodel.Node { if e != nil { panic(e) } return n } // must.TypedNode helps write pointfree/chainable-style code // by taking a Node and an error and transforming any error into a panic. // It will also cast the `datamodel.Node` to a `schema.TypedNode`, panicking if impossible. // // Because golang supports implied destructuring of multiple-return functions // into arguments for another funtion of matching arity, it can be used like this: // // must.TypedNode(SomeNodeBuilder{}.CreateString("a")) func TypedNode(n datamodel.Node, e error) schema.TypedNode { if e != nil { panic(e) } return n.(schema.TypedNode) } // must.True panics if the given bool is false. func True(v bool) { if !v { panic("must.True") } } // must.String unboxes the given Node via AsString, // panicking in the case that the Node isn't of string kind, // and otherwise returning the bare native string. func String(n datamodel.Node) string { if v, err := n.AsString(); err != nil { panic(err) } else { return v } } // must.Int unboxes the given Node via AsInt, // panicking in the case that the Node isn't of int kind, // and otherwise returning the bare native int. func Int(n datamodel.Node) int64 { if v, err := n.AsInt(); err != nil { panic(err) } else { return v } } ================================================ FILE: node/basic/deprecated.go ================================================ // This is a transitional package: please move your references to `node/basicnode`. // The new package is identical: we've renamed the import path only. // // All content in this package is a thin wrapper around `node/basicnode`. // Please update at your earliest convenience. // // This package will eventually be removed. package basicnode import ( "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/linking" "github.com/ipld/go-ipld-prime/node/basicnode" ) var Prototype = basicnode.Prototype func Chooser(_ datamodel.Link, _ linking.LinkContext) (datamodel.NodePrototype, error) { return basicnode.Chooser(nil, linking.LinkContext{}) } func NewBool(value bool) datamodel.Node { return basicnode.NewBool(value) } func NewBytes(value []byte) datamodel.Node { return basicnode.NewBytes(value) } func NewFloat(value float64) datamodel.Node { return basicnode.NewFloat(value) } func NewInt(value int64) datamodel.Node { return basicnode.NewInt(value) } func NewLink(value datamodel.Link) datamodel.Node { return basicnode.NewLink(value) } func NewString(value string) datamodel.Node { return basicnode.NewString(value) } type Prototype__Any = basicnode.Prototype__Any type Prototype__Bool = basicnode.Prototype__Bool type Prototype__Bytes = basicnode.Prototype__Bytes type Prototype__Float = basicnode.Prototype__Float type Prototype__Int = basicnode.Prototype__Int type Prototype__Link = basicnode.Prototype__Link type Prototype__List = basicnode.Prototype__List type Prototype__Map = basicnode.Prototype__Map type Prototype__String = basicnode.Prototype__String ================================================ FILE: node/basicnode/HACKME.md ================================================ hackme ====== Design rationale are documented here. This doc is not necessary reading for users of this package, but if you're considering submitting patches -- or just trying to understand why it was written this way, and check for reasoning that might be dated -- then it might be useful reading. ### scalars are just typedefs This is noteworthy because in codegen, this is typically *not* the case: in codegen, even scalar types are boxed in a struct, such that it prevents casting values into those types. This casting is not a concern for the node implementations in this package, because - A) we don't have any kind of validation rules to make such casting worrying; and - B) since our types are unexported, casting is still blocked by this anyway. ### about builders for scalars The assembler types for scalars (string, int, etc) are pretty funny-looking. You might wish to make them work without any state at all! The reason this doesn't fly is that we have to keep the "wip" value in hand just long enough to return it from the `NodeBuilder.Build` method -- the `NodeAssembler` contract for `Assign*` methods doesn't permit just returning their results immediately. (Another possible reason is if we expected to use these assemblers on slab-style allocations (say, `[]plainString`)... however, this is inapplicable at present, because A) we don't (except places that have special-case internal paths anyway); and B) the types aren't exported, so users can't either.) Does this mean that using `NodeBuilder` for scalars has a completely unnecessary second allocation, which is laughably inefficient? Yes. It's unfortunate the interfaces constrain us to this. **But**: one typically doesn't actually use builders for scalars much; they're just here for completeness. So this is less of a problem in practice than it might at first seem. More often, one will use the "any" builder (which is has a whole different set of design constraints and tradeoffs); or, if one is writing code and knows which scalar they need, the exported direct constructor function for that kind (e.g., `String("foo")` instead of `Prototype__String{}.NewBuilder().AssignString("foo")`) will do the right thing and do it in one allocation (and it's less to type, too). ### maps and list keyAssembler and valueAssemblers have custom scalar handling Related to the above heading. Maps and lists in this package do their own internal handling of scalars, using unexported features inside the package, because they can more efficient. ### when to invalidate the 'w' pointers The 'w' pointer -- short for 'wip' node pointer -- has an interesting lifecycle. In a NodeAssembler, the 'w' pointer should be intialized before the assembler is used. This means either the matching NodeBuilder type does so; or, if we're inside recursive structure, the parent assembler did so. The 'w' pointer is used throughout the life of the assembler. Setting the 'w' pointer to nil is one of two mechanisms used internally to mark that assembly has become "finished" (the other mechanism is using an internal state enum field). Setting the 'w' pointer to nil has two advantages: one is that it makes it *impossible* to continue to mutate the target node; the other is that we need no *additional* memory to track this state change. However, we can't use the strategy of nilling 'w' in all cases: in particular, when in the NodeBuilder at the root of some construction, we need to continue to hold onto the node between when it becomes "finished" and when Build is called; otherwise we can't actually return the value! Different stratgies are therefore used in different parts of this package. Maps and lists use an internal state enum, because they already have one, and so they might as well; there's no additional cost to this. Since they can use this state to guard against additional mutations after "finish", the map and list assemblers don't bother to nil their own 'w' at all. During recursion to assemble values _inside_ maps and lists, it's interesting: the child assembler wrapper type takes reponsibility for nilling out the 'w' pointer in the child assembler's state, doing this at the same time as it updates the parent's state machine to clear proceeding with the next entry. In the case of scalars at the root of a build, we took a shortcut: we actually don't fence against repeat mutations at all. *You can actually use the assign method more than once*. We can do this without breaking safety contracts because the scalars all have a pass-by-value phase somewhere in their lifecycle (calling `nb.AssignString("x")`, then `n := nb.Build()`, then `nb.AssignString("y")` won't error if `nb` is a freestanding builder for strings... but it also won't result in mutating `n` to contain `"y"`, so overall, it's safe). We could normalize the case with scalars at the root of a tree so that they error more aggressively... but currently we haven't bothered, since this would require adding another piece of memory to the scalar builders; and meanwhile we're not in trouble on compositional correctness. Note that these remarks are for the `basicnode` package, but may also apply to other implementations too (e.g., our codegen output follows similar overall logic). ### NodePrototypes are available through a singleton Every NodePrototype available from this package is exposed as a field in a struct of which there's one public exported instance available, called 'Prototype'. This means you can use it like this: ```go nbm := basicnode.Prototype.Map.NewBuilder() nbs := basicnode.Prototype.String.NewBuilder() nba := basicnode.Prototype.Any.NewBuilder() // etc ``` (If you're interested in the performance of this: it's free! Methods called at the end of the chain are inlinable. Since all of the types of the structures on the way there are zero-member structs, the compiler can effectively treat them as constants, and thus freely elide any memory dereferences that would otherwise be necessary to get methods on such a value.) ### NodePrototypes are (also) available as exported concrete types The 'Prototype' singleton is one way to access the NodePrototype in this package; their exported types are another equivalent way. ```go basicnode.Prototype.Map = basicnode.Prototype.Map ``` It is recommended to use the singleton style; they compile to identical assembly, and the singleton is syntactically prettier. We may make these concrete types unexported in the future. A decision on this is deferred until some time has passed and we can accumulate reasonable certainty that there's no need for an exported type (such as type assertions, etc). ================================================ FILE: node/basicnode/any.go ================================================ package basicnode import ( "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/linking" ) var ( //_ datamodel.Node = &anyNode{} _ datamodel.NodePrototype = Prototype__Any{} _ datamodel.NodeBuilder = &anyBuilder{} //_ datamodel.NodeAssembler = &anyAssembler{} ) // Note that we don't use a "var _" declaration to assert that Chooser // implements traversal.LinkTargetNodePrototypeChooser, to keep basicnode's // dependencies fairly light. // Chooser implements traversal.LinkTargetNodePrototypeChooser. // // It can be used directly when loading links into the "any" prototype, // or with another chooser layer on top, such as: // // prototypeChooser := dagpb.AddSupportToChooser(basicnode.Chooser) func Chooser(_ datamodel.Link, _ linking.LinkContext) (datamodel.NodePrototype, error) { return Prototype.Any, nil } // -- Node interface methods --> // Unimplemented at present -- see "REVIEW" comment on anyNode. // -- NodePrototype --> type Prototype__Any struct{} func (Prototype__Any) NewBuilder() datamodel.NodeBuilder { return &anyBuilder{} } // -- NodeBuilder --> // anyBuilder is a builder for any kind of node. // // anyBuilder is a little unusual in its internal workings: // unlike most builders, it doesn't embed the corresponding assembler, // nor will it end up using anyNode, // but instead embeds a builder for each of the kinds it might contain. // This is because we want a more granular return at the end: // if we used anyNode, and returned a pointer to just the relevant part of it, // we'd have all the extra bytes of anyNode still reachable in GC terms // for as long as that handle to the interior of it remains live. type anyBuilder struct { // kind is set on first interaction, and used to select which builder to delegate 'Build' to! // As soon as it's been set to a value other than zero (being "Invalid"), all other Assign/Begin calls will fail since something is already in progress. // May also be set to the magic value '99', which means "i dunno, I'm just carrying another node of unknown prototype". kind datamodel.Kind // Only one of the following ends up being used... // but we don't know in advance which one, so all are embeded here. // This uses excessive space, but amortizes allocations, and all will be // freed as soon as the builder is done. // Builders are only used for recursives; // scalars are simple enough we just do them directly. // 'scalarNode' may also hold another Node of unknown prototype (possibly not even from this package), // in which case this is indicated by 'kind==99'. mapBuilder plainMap__Builder listBuilder plainList__Builder scalarNode datamodel.Node } func (nb *anyBuilder) Reset() { *nb = anyBuilder{} } func (nb *anyBuilder) BeginMap(sizeHint int64) (datamodel.MapAssembler, error) { if nb.kind != datamodel.Kind_Invalid { panic("misuse") } nb.kind = datamodel.Kind_Map nb.mapBuilder.w = &plainMap{} return nb.mapBuilder.BeginMap(sizeHint) } func (nb *anyBuilder) BeginList(sizeHint int64) (datamodel.ListAssembler, error) { if nb.kind != datamodel.Kind_Invalid { panic("misuse") } nb.kind = datamodel.Kind_List nb.listBuilder.w = &plainList{} return nb.listBuilder.BeginList(sizeHint) } func (nb *anyBuilder) AssignNull() error { if nb.kind != datamodel.Kind_Invalid { panic("misuse") } nb.kind = datamodel.Kind_Null return nil } func (nb *anyBuilder) AssignBool(v bool) error { if nb.kind != datamodel.Kind_Invalid { panic("misuse") } nb.kind = datamodel.Kind_Bool nb.scalarNode = NewBool(v) return nil } func (nb *anyBuilder) AssignInt(v int64) error { if nb.kind != datamodel.Kind_Invalid { panic("misuse") } nb.kind = datamodel.Kind_Int nb.scalarNode = NewInt(v) return nil } func (nb *anyBuilder) AssignFloat(v float64) error { if nb.kind != datamodel.Kind_Invalid { panic("misuse") } nb.kind = datamodel.Kind_Float nb.scalarNode = NewFloat(v) return nil } func (nb *anyBuilder) AssignString(v string) error { if nb.kind != datamodel.Kind_Invalid { panic("misuse") } nb.kind = datamodel.Kind_String nb.scalarNode = NewString(v) return nil } func (nb *anyBuilder) AssignBytes(v []byte) error { if nb.kind != datamodel.Kind_Invalid { panic("misuse") } nb.kind = datamodel.Kind_Bytes nb.scalarNode = NewBytes(v) return nil } func (nb *anyBuilder) AssignLink(v datamodel.Link) error { if nb.kind != datamodel.Kind_Invalid { panic("misuse") } nb.kind = datamodel.Kind_Link nb.scalarNode = NewLink(v) return nil } func (nb *anyBuilder) AssignNode(v datamodel.Node) error { if nb.kind != datamodel.Kind_Invalid { panic("misuse") } nb.kind = 99 nb.scalarNode = v return nil } func (anyBuilder) Prototype() datamodel.NodePrototype { return Prototype.Any } func (nb *anyBuilder) Build() datamodel.Node { switch nb.kind { case datamodel.Kind_Invalid: panic("misuse") case datamodel.Kind_Map: return nb.mapBuilder.Build() case datamodel.Kind_List: return nb.listBuilder.Build() case datamodel.Kind_Null: return datamodel.Null case datamodel.Kind_Bool: return nb.scalarNode case datamodel.Kind_Int: return nb.scalarNode case datamodel.Kind_Float: return nb.scalarNode case datamodel.Kind_String: return nb.scalarNode case datamodel.Kind_Bytes: return nb.scalarNode case datamodel.Kind_Link: return nb.scalarNode case 99: return nb.scalarNode default: panic("unreachable") } } // -- NodeAssembler --> // ... oddly enough, we seem to be able to put off implementing this // until we also implement something that goes full-hog on amortization // and actually has a slab of `anyNode`. Which so far, nothing does. // See "REVIEW" comment on anyNode. // type anyAssembler struct { // w *anyNode // } ================================================ FILE: node/basicnode/any_test.go ================================================ package basicnode_test import ( "testing" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/node/tests" ) func TestAnyBeingString(t *testing.T) { tests.SpecTestString(t, basicnode.Prototype.Any) } func TestAnyBeingMapStrInt(t *testing.T) { tests.SpecTestMapStrInt(t, basicnode.Prototype.Any) } func TestAnyBeingMapStrMapStrInt(t *testing.T) { tests.SpecTestMapStrMapStrInt(t, basicnode.Prototype.Any) } ================================================ FILE: node/basicnode/bench_test.go ================================================ package basicnode_test import ( "testing" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/node/tests" ) func BenchmarkSpec_Walk_Map3StrInt(b *testing.B) { tests.BenchmarkSpec_Walk_Map3StrInt(b, basicnode.Prototype.Any) } func BenchmarkSpec_Walk_MapNStrMap3StrInt(b *testing.B) { tests.BenchmarkSpec_Walk_MapNStrMap3StrInt(b, basicnode.Prototype.Any) } ================================================ FILE: node/basicnode/bool.go ================================================ package basicnode import ( "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/node/mixins" ) var ( _ datamodel.Node = plainBool(false) _ datamodel.NodePrototype = Prototype__Bool{} _ datamodel.NodeBuilder = &plainBool__Builder{} _ datamodel.NodeAssembler = &plainBool__Assembler{} ) func NewBool(value bool) datamodel.Node { v := plainBool(value) return &v } // plainBool is a simple boxed boolean that complies with datamodel.Node. type plainBool bool // -- Node interface methods --> func (plainBool) Kind() datamodel.Kind { return datamodel.Kind_Bool } func (plainBool) LookupByString(string) (datamodel.Node, error) { return mixins.Bool{TypeName: "bool"}.LookupByString("") } func (plainBool) LookupByNode(key datamodel.Node) (datamodel.Node, error) { return mixins.Bool{TypeName: "bool"}.LookupByNode(nil) } func (plainBool) LookupByIndex(idx int64) (datamodel.Node, error) { return mixins.Bool{TypeName: "bool"}.LookupByIndex(0) } func (plainBool) LookupBySegment(seg datamodel.PathSegment) (datamodel.Node, error) { return mixins.Bool{TypeName: "bool"}.LookupBySegment(seg) } func (plainBool) MapIterator() datamodel.MapIterator { return nil } func (plainBool) ListIterator() datamodel.ListIterator { return nil } func (plainBool) Length() int64 { return -1 } func (plainBool) IsAbsent() bool { return false } func (plainBool) IsNull() bool { return false } func (n plainBool) AsBool() (bool, error) { return bool(n), nil } func (plainBool) AsInt() (int64, error) { return mixins.Bool{TypeName: "bool"}.AsInt() } func (plainBool) AsFloat() (float64, error) { return mixins.Bool{TypeName: "bool"}.AsFloat() } func (plainBool) AsString() (string, error) { return mixins.Bool{TypeName: "bool"}.AsString() } func (plainBool) AsBytes() ([]byte, error) { return mixins.Bool{TypeName: "bool"}.AsBytes() } func (plainBool) AsLink() (datamodel.Link, error) { return mixins.Bool{TypeName: "bool"}.AsLink() } func (plainBool) Prototype() datamodel.NodePrototype { return Prototype__Bool{} } // -- NodePrototype --> type Prototype__Bool struct{} func (Prototype__Bool) NewBuilder() datamodel.NodeBuilder { var w plainBool return &plainBool__Builder{plainBool__Assembler{w: &w}} } // -- NodeBuilder --> type plainBool__Builder struct { plainBool__Assembler } func (nb *plainBool__Builder) Build() datamodel.Node { return nb.w } func (nb *plainBool__Builder) Reset() { var w plainBool *nb = plainBool__Builder{plainBool__Assembler{w: &w}} } // -- NodeAssembler --> type plainBool__Assembler struct { w *plainBool } func (plainBool__Assembler) BeginMap(sizeHint int64) (datamodel.MapAssembler, error) { return mixins.BoolAssembler{TypeName: "bool"}.BeginMap(0) } func (plainBool__Assembler) BeginList(sizeHint int64) (datamodel.ListAssembler, error) { return mixins.BoolAssembler{TypeName: "bool"}.BeginList(0) } func (plainBool__Assembler) AssignNull() error { return mixins.BoolAssembler{TypeName: "bool"}.AssignNull() } func (na *plainBool__Assembler) AssignBool(v bool) error { *na.w = plainBool(v) return nil } func (plainBool__Assembler) AssignInt(int64) error { return mixins.BoolAssembler{TypeName: "bool"}.AssignInt(0) } func (plainBool__Assembler) AssignFloat(float64) error { return mixins.BoolAssembler{TypeName: "bool"}.AssignFloat(0) } func (plainBool__Assembler) AssignString(string) error { return mixins.BoolAssembler{TypeName: "bool"}.AssignString("") } func (plainBool__Assembler) AssignBytes([]byte) error { return mixins.BoolAssembler{TypeName: "bool"}.AssignBytes(nil) } func (plainBool__Assembler) AssignLink(datamodel.Link) error { return mixins.BoolAssembler{TypeName: "bool"}.AssignLink(nil) } func (na *plainBool__Assembler) AssignNode(v datamodel.Node) error { if v2, err := v.AsBool(); err != nil { return err } else { *na.w = plainBool(v2) return nil } } func (plainBool__Assembler) Prototype() datamodel.NodePrototype { return Prototype__Bool{} } ================================================ FILE: node/basicnode/bytes.go ================================================ package basicnode import ( "bytes" "io" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/node/mixins" ) var ( _ datamodel.Node = plainBytes(nil) _ datamodel.NodePrototype = Prototype__Bytes{} _ datamodel.NodeBuilder = &plainBytes__Builder{} _ datamodel.NodeAssembler = &plainBytes__Assembler{} ) func NewBytes(value []byte) datamodel.Node { v := plainBytes(value) return &v } // plainBytes is a simple boxed byte slice that complies with datamodel.Node. type plainBytes []byte // -- Node interface methods --> func (plainBytes) Kind() datamodel.Kind { return datamodel.Kind_Bytes } func (plainBytes) LookupByString(string) (datamodel.Node, error) { return mixins.Bytes{TypeName: "bytes"}.LookupByString("") } func (plainBytes) LookupByNode(key datamodel.Node) (datamodel.Node, error) { return mixins.Bytes{TypeName: "bytes"}.LookupByNode(nil) } func (plainBytes) LookupByIndex(idx int64) (datamodel.Node, error) { return mixins.Bytes{TypeName: "bytes"}.LookupByIndex(0) } func (plainBytes) LookupBySegment(seg datamodel.PathSegment) (datamodel.Node, error) { return mixins.Bytes{TypeName: "bytes"}.LookupBySegment(seg) } func (plainBytes) MapIterator() datamodel.MapIterator { return nil } func (plainBytes) ListIterator() datamodel.ListIterator { return nil } func (plainBytes) Length() int64 { return -1 } func (plainBytes) IsAbsent() bool { return false } func (plainBytes) IsNull() bool { return false } func (plainBytes) AsBool() (bool, error) { return mixins.Bytes{TypeName: "bytes"}.AsBool() } func (plainBytes) AsInt() (int64, error) { return mixins.Bytes{TypeName: "bytes"}.AsInt() } func (plainBytes) AsFloat() (float64, error) { return mixins.Bytes{TypeName: "bytes"}.AsFloat() } func (plainBytes) AsString() (string, error) { return mixins.Bytes{TypeName: "bytes"}.AsString() } func (n plainBytes) AsBytes() ([]byte, error) { return []byte(n), nil } func (plainBytes) AsLink() (datamodel.Link, error) { return mixins.Bytes{TypeName: "bytes"}.AsLink() } func (plainBytes) Prototype() datamodel.NodePrototype { return Prototype__Bytes{} } func (n plainBytes) AsLargeBytes() (io.ReadSeeker, error) { return bytes.NewReader(n), nil } // -- NodePrototype --> type Prototype__Bytes struct{} func (Prototype__Bytes) NewBuilder() datamodel.NodeBuilder { var w plainBytes return &plainBytes__Builder{plainBytes__Assembler{w: &w}} } // -- NodeBuilder --> type plainBytes__Builder struct { plainBytes__Assembler } func (nb *plainBytes__Builder) Build() datamodel.Node { return nb.w } func (nb *plainBytes__Builder) Reset() { var w plainBytes *nb = plainBytes__Builder{plainBytes__Assembler{w: &w}} } // -- NodeAssembler --> type plainBytes__Assembler struct { w datamodel.Node } func (plainBytes__Assembler) BeginMap(sizeHint int64) (datamodel.MapAssembler, error) { return mixins.BytesAssembler{TypeName: "bytes"}.BeginMap(0) } func (plainBytes__Assembler) BeginList(sizeHint int64) (datamodel.ListAssembler, error) { return mixins.BytesAssembler{TypeName: "bytes"}.BeginList(0) } func (plainBytes__Assembler) AssignNull() error { return mixins.BytesAssembler{TypeName: "bytes"}.AssignNull() } func (plainBytes__Assembler) AssignBool(bool) error { return mixins.BytesAssembler{TypeName: "bytes"}.AssignBool(false) } func (plainBytes__Assembler) AssignInt(int64) error { return mixins.BytesAssembler{TypeName: "bytes"}.AssignInt(0) } func (plainBytes__Assembler) AssignFloat(float64) error { return mixins.BytesAssembler{TypeName: "bytes"}.AssignFloat(0) } func (plainBytes__Assembler) AssignString(string) error { return mixins.BytesAssembler{TypeName: "bytes"}.AssignString("") } func (na *plainBytes__Assembler) AssignBytes(v []byte) error { na.w = datamodel.Node(plainBytes(v)) return nil } func (plainBytes__Assembler) AssignLink(datamodel.Link) error { return mixins.BytesAssembler{TypeName: "bytes"}.AssignLink(nil) } func (na *plainBytes__Assembler) AssignNode(v datamodel.Node) error { if lb, ok := v.(datamodel.LargeBytesNode); ok { lbn, err := lb.AsLargeBytes() if err == nil { na.w = streamBytes{lbn} return nil } } if v2, err := v.AsBytes(); err != nil { return err } else { na.w = plainBytes(v2) return nil } } func (plainBytes__Assembler) Prototype() datamodel.NodePrototype { return Prototype__Bytes{} } ================================================ FILE: node/basicnode/bytes_stream.go ================================================ package basicnode import ( "io" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/node/mixins" ) var ( _ datamodel.Node = streamBytes{nil} _ datamodel.NodePrototype = Prototype__Bytes{} _ datamodel.NodeBuilder = &plainBytes__Builder{} _ datamodel.NodeAssembler = &plainBytes__Assembler{} ) func NewBytesFromReader(rs io.ReadSeeker) datamodel.Node { return streamBytes{rs} } // streamBytes is a boxed reader that complies with datamodel.Node. type streamBytes struct { io.ReadSeeker } // -- Node interface methods --> func (streamBytes) Kind() datamodel.Kind { return datamodel.Kind_Bytes } func (streamBytes) LookupByString(string) (datamodel.Node, error) { return mixins.Bytes{TypeName: "bytes"}.LookupByString("") } func (streamBytes) LookupByNode(key datamodel.Node) (datamodel.Node, error) { return mixins.Bytes{TypeName: "bytes"}.LookupByNode(nil) } func (streamBytes) LookupByIndex(idx int64) (datamodel.Node, error) { return mixins.Bytes{TypeName: "bytes"}.LookupByIndex(0) } func (streamBytes) LookupBySegment(seg datamodel.PathSegment) (datamodel.Node, error) { return mixins.Bytes{TypeName: "bytes"}.LookupBySegment(seg) } func (streamBytes) MapIterator() datamodel.MapIterator { return nil } func (streamBytes) ListIterator() datamodel.ListIterator { return nil } func (streamBytes) Length() int64 { return -1 } func (streamBytes) IsAbsent() bool { return false } func (streamBytes) IsNull() bool { return false } func (streamBytes) AsBool() (bool, error) { return mixins.Bytes{TypeName: "bytes"}.AsBool() } func (streamBytes) AsInt() (int64, error) { return mixins.Bytes{TypeName: "bytes"}.AsInt() } func (streamBytes) AsFloat() (float64, error) { return mixins.Bytes{TypeName: "bytes"}.AsFloat() } func (streamBytes) AsString() (string, error) { return mixins.Bytes{TypeName: "bytes"}.AsString() } func (n streamBytes) AsBytes() ([]byte, error) { return io.ReadAll(n) } func (streamBytes) AsLink() (datamodel.Link, error) { return mixins.Bytes{TypeName: "bytes"}.AsLink() } func (streamBytes) Prototype() datamodel.NodePrototype { return Prototype__Bytes{} } func (n streamBytes) AsLargeBytes() (io.ReadSeeker, error) { return n.ReadSeeker, nil } ================================================ FILE: node/basicnode/bytes_test.go ================================================ package basicnode_test import ( "testing" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/node/tests" ) func TestBytes(t *testing.T) { tests.SpecTestBytes(t, basicnode.Prototype__Bytes{}) } ================================================ FILE: node/basicnode/float.go ================================================ package basicnode import ( "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/node/mixins" ) var ( _ datamodel.Node = plainFloat(0) _ datamodel.NodePrototype = Prototype__Float{} _ datamodel.NodeBuilder = &plainFloat__Builder{} _ datamodel.NodeAssembler = &plainFloat__Assembler{} ) func NewFloat(value float64) datamodel.Node { v := plainFloat(value) return &v } // plainFloat is a simple boxed float that complies with datamodel.Node. type plainFloat float64 // -- Node interface methods --> func (plainFloat) Kind() datamodel.Kind { return datamodel.Kind_Float } func (plainFloat) LookupByString(string) (datamodel.Node, error) { return mixins.Float{TypeName: "float"}.LookupByString("") } func (plainFloat) LookupByNode(key datamodel.Node) (datamodel.Node, error) { return mixins.Float{TypeName: "float"}.LookupByNode(nil) } func (plainFloat) LookupByIndex(idx int64) (datamodel.Node, error) { return mixins.Float{TypeName: "float"}.LookupByIndex(0) } func (plainFloat) LookupBySegment(seg datamodel.PathSegment) (datamodel.Node, error) { return mixins.Float{TypeName: "float"}.LookupBySegment(seg) } func (plainFloat) MapIterator() datamodel.MapIterator { return nil } func (plainFloat) ListIterator() datamodel.ListIterator { return nil } func (plainFloat) Length() int64 { return -1 } func (plainFloat) IsAbsent() bool { return false } func (plainFloat) IsNull() bool { return false } func (plainFloat) AsBool() (bool, error) { return mixins.Float{TypeName: "float"}.AsBool() } func (plainFloat) AsInt() (int64, error) { return mixins.Float{TypeName: "float"}.AsInt() } func (n plainFloat) AsFloat() (float64, error) { return float64(n), nil } func (plainFloat) AsString() (string, error) { return mixins.Float{TypeName: "float"}.AsString() } func (plainFloat) AsBytes() ([]byte, error) { return mixins.Float{TypeName: "float"}.AsBytes() } func (plainFloat) AsLink() (datamodel.Link, error) { return mixins.Float{TypeName: "float"}.AsLink() } func (plainFloat) Prototype() datamodel.NodePrototype { return Prototype__Float{} } // -- NodePrototype --> type Prototype__Float struct{} func (Prototype__Float) NewBuilder() datamodel.NodeBuilder { var w plainFloat return &plainFloat__Builder{plainFloat__Assembler{w: &w}} } // -- NodeBuilder --> type plainFloat__Builder struct { plainFloat__Assembler } func (nb *plainFloat__Builder) Build() datamodel.Node { return nb.w } func (nb *plainFloat__Builder) Reset() { var w plainFloat *nb = plainFloat__Builder{plainFloat__Assembler{w: &w}} } // -- NodeAssembler --> type plainFloat__Assembler struct { w *plainFloat } func (plainFloat__Assembler) BeginMap(sizeHint int64) (datamodel.MapAssembler, error) { return mixins.FloatAssembler{TypeName: "float"}.BeginMap(0) } func (plainFloat__Assembler) BeginList(sizeHint int64) (datamodel.ListAssembler, error) { return mixins.FloatAssembler{TypeName: "float"}.BeginList(0) } func (plainFloat__Assembler) AssignNull() error { return mixins.FloatAssembler{TypeName: "float"}.AssignNull() } func (plainFloat__Assembler) AssignBool(bool) error { return mixins.FloatAssembler{TypeName: "float"}.AssignBool(false) } func (plainFloat__Assembler) AssignInt(int64) error { return mixins.FloatAssembler{TypeName: "float"}.AssignInt(0) } func (na *plainFloat__Assembler) AssignFloat(v float64) error { *na.w = plainFloat(v) return nil } func (plainFloat__Assembler) AssignString(string) error { return mixins.FloatAssembler{TypeName: "float"}.AssignString("") } func (plainFloat__Assembler) AssignBytes([]byte) error { return mixins.FloatAssembler{TypeName: "float"}.AssignBytes(nil) } func (plainFloat__Assembler) AssignLink(datamodel.Link) error { return mixins.FloatAssembler{TypeName: "float"}.AssignLink(nil) } func (na *plainFloat__Assembler) AssignNode(v datamodel.Node) error { if v2, err := v.AsFloat(); err != nil { return err } else { *na.w = plainFloat(v2) return nil } } func (plainFloat__Assembler) Prototype() datamodel.NodePrototype { return Prototype__Float{} } ================================================ FILE: node/basicnode/int.go ================================================ package basicnode import ( "fmt" "math" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/node/mixins" ) var ( _ datamodel.Node = plainInt(0) _ datamodel.Node = plainUint(0) _ datamodel.UintNode = plainUint(0) _ datamodel.NodePrototype = Prototype__Int{} _ datamodel.NodeBuilder = &plainInt__Builder{} _ datamodel.NodeAssembler = &plainInt__Assembler{} ) func NewInt(value int64) datamodel.Node { v := plainInt(value) return &v } // NewUint creates a new uint64-backed Node which will behave as a plain Int // node but also conforms to the datamodel.UintNode interface which can access // the full uint64 range. // // EXPERIMENTAL: this API is experimental and may be changed or removed in a // future release. func NewUint(value uint64) datamodel.Node { return plainUint(value) } // plainInt is a simple boxed int that complies with datamodel.Node. type plainInt int64 // -- Node interface methods for plainInt --> func (plainInt) Kind() datamodel.Kind { return datamodel.Kind_Int } func (plainInt) LookupByString(string) (datamodel.Node, error) { return mixins.Int{TypeName: "int"}.LookupByString("") } func (plainInt) LookupByNode(key datamodel.Node) (datamodel.Node, error) { return mixins.Int{TypeName: "int"}.LookupByNode(nil) } func (plainInt) LookupByIndex(idx int64) (datamodel.Node, error) { return mixins.Int{TypeName: "int"}.LookupByIndex(0) } func (plainInt) LookupBySegment(seg datamodel.PathSegment) (datamodel.Node, error) { return mixins.Int{TypeName: "int"}.LookupBySegment(seg) } func (plainInt) MapIterator() datamodel.MapIterator { return nil } func (plainInt) ListIterator() datamodel.ListIterator { return nil } func (plainInt) Length() int64 { return -1 } func (plainInt) IsAbsent() bool { return false } func (plainInt) IsNull() bool { return false } func (plainInt) AsBool() (bool, error) { return mixins.Int{TypeName: "int"}.AsBool() } func (n plainInt) AsInt() (int64, error) { return int64(n), nil } func (plainInt) AsFloat() (float64, error) { return mixins.Int{TypeName: "int"}.AsFloat() } func (plainInt) AsString() (string, error) { return mixins.Int{TypeName: "int"}.AsString() } func (plainInt) AsBytes() ([]byte, error) { return mixins.Int{TypeName: "int"}.AsBytes() } func (plainInt) AsLink() (datamodel.Link, error) { return mixins.Int{TypeName: "int"}.AsLink() } func (plainInt) Prototype() datamodel.NodePrototype { return Prototype__Int{} } // plainUint is a simple boxed uint64 that complies with datamodel.Node, // allowing representation of the uint64 range above the int64 maximum via the // UintNode interface type plainUint uint64 // -- Node interface methods for plainUint --> func (plainUint) Kind() datamodel.Kind { return datamodel.Kind_Int } func (plainUint) LookupByString(string) (datamodel.Node, error) { return mixins.Int{TypeName: "int"}.LookupByString("") } func (plainUint) LookupByNode(key datamodel.Node) (datamodel.Node, error) { return mixins.Int{TypeName: "int"}.LookupByNode(nil) } func (plainUint) LookupByIndex(idx int64) (datamodel.Node, error) { return mixins.Int{TypeName: "int"}.LookupByIndex(0) } func (plainUint) LookupBySegment(seg datamodel.PathSegment) (datamodel.Node, error) { return mixins.Int{TypeName: "int"}.LookupBySegment(seg) } func (plainUint) MapIterator() datamodel.MapIterator { return nil } func (plainUint) ListIterator() datamodel.ListIterator { return nil } func (plainUint) Length() int64 { return -1 } func (plainUint) IsAbsent() bool { return false } func (plainUint) IsNull() bool { return false } func (plainUint) AsBool() (bool, error) { return mixins.Int{TypeName: "int"}.AsBool() } func (n plainUint) AsInt() (int64, error) { if uint64(n) > uint64(math.MaxInt64) { return -1, fmt.Errorf("unsigned integer out of range of int64 type") } return int64(n), nil } func (plainUint) AsFloat() (float64, error) { return mixins.Int{TypeName: "int"}.AsFloat() } func (plainUint) AsString() (string, error) { return mixins.Int{TypeName: "int"}.AsString() } func (plainUint) AsBytes() ([]byte, error) { return mixins.Int{TypeName: "int"}.AsBytes() } func (plainUint) AsLink() (datamodel.Link, error) { return mixins.Int{TypeName: "int"}.AsLink() } func (plainUint) Prototype() datamodel.NodePrototype { return Prototype__Int{} } // allows plainUint to conform to the plainUint interface func (n plainUint) AsUint() (uint64, error) { return uint64(n), nil } // -- NodePrototype --> type Prototype__Int struct{} func (Prototype__Int) NewBuilder() datamodel.NodeBuilder { var w plainInt return &plainInt__Builder{plainInt__Assembler{w: &w}} } // -- NodeBuilder --> type plainInt__Builder struct { plainInt__Assembler } func (nb *plainInt__Builder) Build() datamodel.Node { return nb.w } func (nb *plainInt__Builder) Reset() { var w plainInt *nb = plainInt__Builder{plainInt__Assembler{w: &w}} } // -- NodeAssembler --> type plainInt__Assembler struct { w *plainInt } func (plainInt__Assembler) BeginMap(sizeHint int64) (datamodel.MapAssembler, error) { return mixins.IntAssembler{TypeName: "int"}.BeginMap(0) } func (plainInt__Assembler) BeginList(sizeHint int64) (datamodel.ListAssembler, error) { return mixins.IntAssembler{TypeName: "int"}.BeginList(0) } func (plainInt__Assembler) AssignNull() error { return mixins.IntAssembler{TypeName: "int"}.AssignNull() } func (plainInt__Assembler) AssignBool(bool) error { return mixins.IntAssembler{TypeName: "int"}.AssignBool(false) } func (na *plainInt__Assembler) AssignInt(v int64) error { *na.w = plainInt(v) return nil } func (plainInt__Assembler) AssignFloat(float64) error { return mixins.IntAssembler{TypeName: "int"}.AssignFloat(0) } func (plainInt__Assembler) AssignString(string) error { return mixins.IntAssembler{TypeName: "int"}.AssignString("") } func (plainInt__Assembler) AssignBytes([]byte) error { return mixins.IntAssembler{TypeName: "int"}.AssignBytes(nil) } func (plainInt__Assembler) AssignLink(datamodel.Link) error { return mixins.IntAssembler{TypeName: "int"}.AssignLink(nil) } func (na *plainInt__Assembler) AssignNode(v datamodel.Node) error { if v2, err := v.AsInt(); err != nil { return err } else { *na.w = plainInt(v2) return nil } } func (plainInt__Assembler) Prototype() datamodel.NodePrototype { return Prototype__Int{} } ================================================ FILE: node/basicnode/int_test.go ================================================ package basicnode_test import ( "testing" qt "github.com/frankban/quicktest" "github.com/ipld/go-ipld-prime/must" "github.com/ipld/go-ipld-prime/node/basicnode" ) func TestBasicInt(t *testing.T) { m := basicnode.NewInt(3) b := m.Prototype().NewBuilder() b.AssignInt(4) n := b.Build() u := basicnode.NewUint(5) qt.Check(t, must.Int(m), qt.Equals, int64(3)) qt.Check(t, must.Int(n), qt.Equals, int64(4)) qt.Check(t, must.Int(u), qt.Equals, int64(5)) } func TestIntErrors(t *testing.T) { x := basicnode.NewInt(3) _, err := x.LookupByIndex(0) errExpect := `func called on wrong kind: "LookupByIndex" called on a int node \(kind: int\), but only makes sense on list` qt.Check(t, err, qt.ErrorMatches, errExpect) _, err = x.LookupByString("n") errExpect = `func called on wrong kind: "LookupByString" called on a int node \(kind: int\), but only makes sense on map` qt.Check(t, err, qt.ErrorMatches, errExpect) _, err = x.LookupByNode(x) errExpect = `func called on wrong kind: "LookupByNode" called on a int node \(kind: int\), but only makes sense on map` qt.Check(t, err, qt.ErrorMatches, errExpect) } ================================================ FILE: node/basicnode/link.go ================================================ package basicnode import ( "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/node/mixins" ) var ( _ datamodel.Node = &plainLink{} _ datamodel.NodePrototype = Prototype__Link{} _ datamodel.NodeBuilder = &plainLink__Builder{} _ datamodel.NodeAssembler = &plainLink__Assembler{} ) func NewLink(value datamodel.Link) datamodel.Node { return &plainLink{value} } // plainLink is a simple box around a Link that complies with datamodel.Node. type plainLink struct { x datamodel.Link } // -- Node interface methods --> func (plainLink) Kind() datamodel.Kind { return datamodel.Kind_Link } func (plainLink) LookupByString(string) (datamodel.Node, error) { return mixins.Link{TypeName: "link"}.LookupByString("") } func (plainLink) LookupByNode(key datamodel.Node) (datamodel.Node, error) { return mixins.Link{TypeName: "link"}.LookupByNode(nil) } func (plainLink) LookupByIndex(idx int64) (datamodel.Node, error) { return mixins.Link{TypeName: "link"}.LookupByIndex(0) } func (plainLink) LookupBySegment(seg datamodel.PathSegment) (datamodel.Node, error) { return mixins.Link{TypeName: "link"}.LookupBySegment(seg) } func (plainLink) MapIterator() datamodel.MapIterator { return nil } func (plainLink) ListIterator() datamodel.ListIterator { return nil } func (plainLink) Length() int64 { return -1 } func (plainLink) IsAbsent() bool { return false } func (plainLink) IsNull() bool { return false } func (plainLink) AsBool() (bool, error) { return mixins.Link{TypeName: "link"}.AsBool() } func (plainLink) AsInt() (int64, error) { return mixins.Link{TypeName: "link"}.AsInt() } func (plainLink) AsFloat() (float64, error) { return mixins.Link{TypeName: "link"}.AsFloat() } func (plainLink) AsString() (string, error) { return mixins.Link{TypeName: "link"}.AsString() } func (plainLink) AsBytes() ([]byte, error) { return mixins.Link{TypeName: "link"}.AsBytes() } func (n *plainLink) AsLink() (datamodel.Link, error) { return n.x, nil } func (plainLink) Prototype() datamodel.NodePrototype { return Prototype__Link{} } // -- NodePrototype --> type Prototype__Link struct{} func (Prototype__Link) NewBuilder() datamodel.NodeBuilder { var w plainLink return &plainLink__Builder{plainLink__Assembler{w: &w}} } // -- NodeBuilder --> type plainLink__Builder struct { plainLink__Assembler } func (nb *plainLink__Builder) Build() datamodel.Node { return nb.w } func (nb *plainLink__Builder) Reset() { var w plainLink *nb = plainLink__Builder{plainLink__Assembler{w: &w}} } // -- NodeAssembler --> type plainLink__Assembler struct { w *plainLink } func (plainLink__Assembler) BeginMap(sizeHint int64) (datamodel.MapAssembler, error) { return mixins.LinkAssembler{TypeName: "link"}.BeginMap(0) } func (plainLink__Assembler) BeginList(sizeHint int64) (datamodel.ListAssembler, error) { return mixins.LinkAssembler{TypeName: "link"}.BeginList(0) } func (plainLink__Assembler) AssignNull() error { return mixins.LinkAssembler{TypeName: "link"}.AssignNull() } func (plainLink__Assembler) AssignBool(bool) error { return mixins.LinkAssembler{TypeName: "link"}.AssignBool(false) } func (plainLink__Assembler) AssignInt(int64) error { return mixins.LinkAssembler{TypeName: "link"}.AssignInt(0) } func (plainLink__Assembler) AssignFloat(float64) error { return mixins.LinkAssembler{TypeName: "link"}.AssignFloat(0) } func (plainLink__Assembler) AssignString(string) error { return mixins.LinkAssembler{TypeName: "link"}.AssignString("") } func (plainLink__Assembler) AssignBytes([]byte) error { return mixins.LinkAssembler{TypeName: "link"}.AssignBytes(nil) } func (na *plainLink__Assembler) AssignLink(v datamodel.Link) error { na.w.x = v return nil } func (na *plainLink__Assembler) AssignNode(v datamodel.Node) error { if v2, err := v.AsLink(); err != nil { return err } else { na.w.x = v2 return nil } } func (plainLink__Assembler) Prototype() datamodel.NodePrototype { return Prototype__Link{} } ================================================ FILE: node/basicnode/list.go ================================================ package basicnode import ( "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/node/mixins" ) var ( _ datamodel.Node = &plainList{} _ datamodel.NodePrototype = Prototype__List{} _ datamodel.NodeBuilder = &plainList__Builder{} _ datamodel.NodeAssembler = &plainList__Assembler{} ) // plainList is a concrete type that provides a list-kind datamodel.Node. // It can contain any kind of value. // plainList is also embedded in the 'any' struct and usable from there. type plainList struct { x []datamodel.Node } // -- Node interface methods --> func (plainList) Kind() datamodel.Kind { return datamodel.Kind_List } func (plainList) LookupByString(string) (datamodel.Node, error) { return mixins.List{TypeName: "list"}.LookupByString("") } func (plainList) LookupByNode(datamodel.Node) (datamodel.Node, error) { return mixins.List{TypeName: "list"}.LookupByNode(nil) } func (n *plainList) LookupByIndex(idx int64) (datamodel.Node, error) { if idx < 0 { return nil, datamodel.ErrNotExists{Segment: datamodel.PathSegmentOfInt(idx)} } if n.Length() <= idx { return nil, datamodel.ErrNotExists{Segment: datamodel.PathSegmentOfInt(idx)} } return n.x[idx], nil } func (n *plainList) LookupBySegment(seg datamodel.PathSegment) (datamodel.Node, error) { idx, err := seg.Index() if err != nil { return nil, datamodel.ErrInvalidSegmentForList{TroubleSegment: seg, Reason: err} } return n.LookupByIndex(idx) } func (plainList) MapIterator() datamodel.MapIterator { return nil } func (n *plainList) ListIterator() datamodel.ListIterator { return &plainList_ListIterator{n, 0} } func (n *plainList) Length() int64 { return int64(len(n.x)) } func (plainList) IsAbsent() bool { return false } func (plainList) IsNull() bool { return false } func (plainList) AsBool() (bool, error) { return mixins.List{TypeName: "list"}.AsBool() } func (plainList) AsInt() (int64, error) { return mixins.List{TypeName: "list"}.AsInt() } func (plainList) AsFloat() (float64, error) { return mixins.List{TypeName: "list"}.AsFloat() } func (plainList) AsString() (string, error) { return mixins.List{TypeName: "list"}.AsString() } func (plainList) AsBytes() ([]byte, error) { return mixins.List{TypeName: "list"}.AsBytes() } func (plainList) AsLink() (datamodel.Link, error) { return mixins.List{TypeName: "list"}.AsLink() } func (plainList) Prototype() datamodel.NodePrototype { return Prototype.List } type plainList_ListIterator struct { n *plainList idx int } func (itr *plainList_ListIterator) Next() (idx int64, v datamodel.Node, _ error) { if itr.Done() { return -1, nil, datamodel.ErrIteratorOverread{} } v = itr.n.x[itr.idx] idx = int64(itr.idx) itr.idx++ return } func (itr *plainList_ListIterator) Done() bool { return itr.idx >= len(itr.n.x) } // -- NodePrototype --> type Prototype__List struct{} func (Prototype__List) NewBuilder() datamodel.NodeBuilder { return &plainList__Builder{plainList__Assembler{w: &plainList{}}} } // -- NodeBuilder --> type plainList__Builder struct { plainList__Assembler } func (nb *plainList__Builder) Build() datamodel.Node { if nb.state != laState_finished { panic("invalid state: assembler must be 'finished' before Build can be called!") } return nb.w } func (nb *plainList__Builder) Reset() { *nb = plainList__Builder{} nb.w = &plainList{} } // -- NodeAssembler --> type plainList__Assembler struct { w *plainList va plainList__ValueAssembler state laState } type plainList__ValueAssembler struct { la *plainList__Assembler } // laState is an enum of the state machine for a list assembler. // (this might be something to export reusably, but it's also very much an impl detail that need not be seen, so, dubious.) // it's similar to maState for maps, but has fewer states because we never have keys to assemble. type laState uint8 const ( laState_initial laState = iota // also the 'expect value or finish' state laState_midValue // waiting for a 'finished' state in the ValueAssembler. laState_finished // 'w' will also be nil, but this is a politer statement ) func (plainList__Assembler) BeginMap(sizeHint int64) (datamodel.MapAssembler, error) { return mixins.ListAssembler{TypeName: "list"}.BeginMap(0) } func (na *plainList__Assembler) BeginList(sizeHint int64) (datamodel.ListAssembler, error) { if sizeHint < 0 { sizeHint = 0 } // Allocate storage space. na.w.x = make([]datamodel.Node, 0, sizeHint) // That's it; return self as the ListAssembler. We already have all the right methods on this structure. return na, nil } func (plainList__Assembler) AssignNull() error { return mixins.ListAssembler{TypeName: "list"}.AssignNull() } func (plainList__Assembler) AssignBool(bool) error { return mixins.ListAssembler{TypeName: "list"}.AssignBool(false) } func (plainList__Assembler) AssignInt(int64) error { return mixins.ListAssembler{TypeName: "list"}.AssignInt(0) } func (plainList__Assembler) AssignFloat(float64) error { return mixins.ListAssembler{TypeName: "list"}.AssignFloat(0) } func (plainList__Assembler) AssignString(string) error { return mixins.ListAssembler{TypeName: "list"}.AssignString("") } func (plainList__Assembler) AssignBytes([]byte) error { return mixins.ListAssembler{TypeName: "list"}.AssignBytes(nil) } func (plainList__Assembler) AssignLink(datamodel.Link) error { return mixins.ListAssembler{TypeName: "list"}.AssignLink(nil) } func (na *plainList__Assembler) AssignNode(v datamodel.Node) error { // Sanity check, then update, assembler state. // Update of state to 'finished' comes later; where exactly depends on if shortcuts apply. if na.state != laState_initial { panic("misuse") } // Copy the content. if v2, ok := v.(*plainList); ok { // if our own type: shortcut. // Copy the structure by value. // This means we'll have pointers into the same internal maps and slices; // this is okay, because the Node type promises it's immutable, and we are going to instantly finish ourselves to also maintain that. // FIXME: the shortcut behaves differently than the long way: it discards any existing progress. Doesn't violate immut, but is odd. *na.w = *v2 na.state = laState_finished return nil } // If the above shortcut didn't work, resort to a generic copy. // We call AssignNode for all the child values, giving them a chance to hit shortcuts even if we didn't. if v.Kind() != datamodel.Kind_List { return datamodel.ErrWrongKind{TypeName: "list", MethodName: "AssignNode", AppropriateKind: datamodel.KindSet_JustList, ActualKind: v.Kind()} } itr := v.ListIterator() for !itr.Done() { _, v, err := itr.Next() if err != nil { return err } if err := na.AssembleValue().AssignNode(v); err != nil { return err } } return na.Finish() } func (plainList__Assembler) Prototype() datamodel.NodePrototype { return Prototype.List } // -- ListAssembler --> // AssembleValue is part of conforming to ListAssembler, which we do on // plainList__Assembler so that BeginList can just return a retyped pointer rather than new object. func (la *plainList__Assembler) AssembleValue() datamodel.NodeAssembler { // Sanity check, then update, assembler state. if la.state != laState_initial { panic("misuse") } la.state = laState_midValue // Make value assembler valid by giving it pointer back to whole 'la'; yield it. la.va.la = la return &la.va } // Finish is part of conforming to ListAssembler, which we do on // plainList__Assembler so that BeginList can just return a retyped pointer rather than new object. func (la *plainList__Assembler) Finish() error { // Sanity check, then update, assembler state. if la.state != laState_initial { panic("misuse") } la.state = laState_finished // validators could run and report errors promptly, if this type had any. return nil } func (plainList__Assembler) ValuePrototype(_ int64) datamodel.NodePrototype { return Prototype.Any } // -- ListAssembler.ValueAssembler --> func (lva *plainList__ValueAssembler) BeginMap(sizeHint int64) (datamodel.MapAssembler, error) { ma := plainList__ValueAssemblerMap{} ma.ca.w = &plainMap{} ma.p = lva.la _, err := ma.ca.BeginMap(sizeHint) return &ma, err } func (lva *plainList__ValueAssembler) BeginList(sizeHint int64) (datamodel.ListAssembler, error) { la := plainList__ValueAssemblerList{} la.ca.w = &plainList{} la.p = lva.la _, err := la.ca.BeginList(sizeHint) return &la, err } func (lva *plainList__ValueAssembler) AssignNull() error { return lva.AssignNode(datamodel.Null) } func (lva *plainList__ValueAssembler) AssignBool(v bool) error { vb := plainBool(v) return lva.AssignNode(&vb) } func (lva *plainList__ValueAssembler) AssignInt(v int64) error { vb := plainInt(v) return lva.AssignNode(&vb) } func (lva *plainList__ValueAssembler) AssignFloat(v float64) error { vb := plainFloat(v) return lva.AssignNode(&vb) } func (lva *plainList__ValueAssembler) AssignString(v string) error { vb := plainString(v) return lva.AssignNode(&vb) } func (lva *plainList__ValueAssembler) AssignBytes(v []byte) error { vb := plainBytes(v) return lva.AssignNode(&vb) } func (lva *plainList__ValueAssembler) AssignLink(v datamodel.Link) error { vb := plainLink{v} return lva.AssignNode(&vb) } func (lva *plainList__ValueAssembler) AssignNode(v datamodel.Node) error { lva.la.w.x = append(lva.la.w.x, v) lva.la.state = laState_initial lva.la = nil // invalidate self to prevent further incorrect use. return nil } func (plainList__ValueAssembler) Prototype() datamodel.NodePrototype { return Prototype.Any } type plainList__ValueAssemblerMap struct { ca plainMap__Assembler p *plainList__Assembler // pointer back to parent, for final insert and state bump } // we briefly state only the methods we need to delegate here. // just embedding plainMap__Assembler also behaves correctly, // but causes a lot of unnecessary autogenerated functions in the final binary. func (ma *plainList__ValueAssemblerMap) AssembleEntry(k string) (datamodel.NodeAssembler, error) { return ma.ca.AssembleEntry(k) } func (ma *plainList__ValueAssemblerMap) AssembleKey() datamodel.NodeAssembler { return ma.ca.AssembleKey() } func (ma *plainList__ValueAssemblerMap) AssembleValue() datamodel.NodeAssembler { return ma.ca.AssembleValue() } func (plainList__ValueAssemblerMap) KeyPrototype() datamodel.NodePrototype { return Prototype__String{} } func (plainList__ValueAssemblerMap) ValuePrototype(_ string) datamodel.NodePrototype { return Prototype.Any } func (ma *plainList__ValueAssemblerMap) Finish() error { if err := ma.ca.Finish(); err != nil { return err } w := ma.ca.w ma.ca.w = nil return ma.p.va.AssignNode(w) } type plainList__ValueAssemblerList struct { ca plainList__Assembler p *plainList__Assembler // pointer back to parent, for final insert and state bump } // we briefly state only the methods we need to delegate here. // just embedding plainList__Assembler also behaves correctly, // but causes a lot of unnecessary autogenerated functions in the final binary. func (la *plainList__ValueAssemblerList) AssembleValue() datamodel.NodeAssembler { return la.ca.AssembleValue() } func (plainList__ValueAssemblerList) ValuePrototype(_ int64) datamodel.NodePrototype { return Prototype.Any } func (la *plainList__ValueAssemblerList) Finish() error { if err := la.ca.Finish(); err != nil { return err } w := la.ca.w la.ca.w = nil return la.p.va.AssignNode(w) } ================================================ FILE: node/basicnode/list_test.go ================================================ package basicnode_test import ( "testing" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/node/tests" ) func TestList(t *testing.T) { tests.SpecTestListString(t, basicnode.Prototype.List) } ================================================ FILE: node/basicnode/map.go ================================================ package basicnode import ( "fmt" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/node/mixins" ) var ( _ datamodel.Node = &plainMap{} _ datamodel.NodePrototype = Prototype__Map{} _ datamodel.NodeBuilder = &plainMap__Builder{} _ datamodel.NodeAssembler = &plainMap__Assembler{} ) // plainMap is a concrete type that provides a map-kind datamodel.Node. // It can contain any kind of value. // plainMap is also embedded in the 'any' struct and usable from there. type plainMap struct { m map[string]datamodel.Node // string key -- even if a runtime schema wrapper is using us for storage, we must have a comparable type here, and string is all we know. t []plainMap__Entry // table for fast iteration, order keeping, and yielding pointers to enable alloc/conv amortization. } type plainMap__Entry struct { k plainString // address of this used when we return keys as nodes, such as in iterators. Need in one place to amortize shifts to heap when ptr'ing for iface. v datamodel.Node // identical to map values. keeping them here simplifies iteration. (in codegen'd maps, this position is also part of amortization, but in this implementation, that's less useful.) // note on alternate implementations: 'v' could also use the 'any' type, and thus amortize value allocations. the memory size trade would be large however, so we don't, here. } // -- Node interface methods --> func (plainMap) Kind() datamodel.Kind { return datamodel.Kind_Map } func (n *plainMap) LookupByString(key string) (datamodel.Node, error) { v, exists := n.m[key] if !exists { return nil, datamodel.ErrNotExists{Segment: datamodel.PathSegmentOfString(key)} } return v, nil } func (n *plainMap) LookupByNode(key datamodel.Node) (datamodel.Node, error) { ks, err := key.AsString() if err != nil { return nil, err } return n.LookupByString(ks) } func (plainMap) LookupByIndex(idx int64) (datamodel.Node, error) { return mixins.Map{TypeName: "map"}.LookupByIndex(0) } func (n *plainMap) LookupBySegment(seg datamodel.PathSegment) (datamodel.Node, error) { return n.LookupByString(seg.String()) } func (n *plainMap) MapIterator() datamodel.MapIterator { return &plainMap_MapIterator{n, 0} } func (plainMap) ListIterator() datamodel.ListIterator { return nil } func (n *plainMap) Length() int64 { return int64(len(n.t)) } func (plainMap) IsAbsent() bool { return false } func (plainMap) IsNull() bool { return false } func (plainMap) AsBool() (bool, error) { return mixins.Map{TypeName: "map"}.AsBool() } func (plainMap) AsInt() (int64, error) { return mixins.Map{TypeName: "map"}.AsInt() } func (plainMap) AsFloat() (float64, error) { return mixins.Map{TypeName: "map"}.AsFloat() } func (plainMap) AsString() (string, error) { return mixins.Map{TypeName: "map"}.AsString() } func (plainMap) AsBytes() ([]byte, error) { return mixins.Map{TypeName: "map"}.AsBytes() } func (plainMap) AsLink() (datamodel.Link, error) { return mixins.Map{TypeName: "map"}.AsLink() } func (plainMap) Prototype() datamodel.NodePrototype { return Prototype.Map } type plainMap_MapIterator struct { n *plainMap idx int } func (itr *plainMap_MapIterator) Next() (k datamodel.Node, v datamodel.Node, _ error) { if itr.Done() { return nil, nil, datamodel.ErrIteratorOverread{} } k = &itr.n.t[itr.idx].k v = itr.n.t[itr.idx].v itr.idx++ return } func (itr *plainMap_MapIterator) Done() bool { return itr.idx >= len(itr.n.t) } // -- NodePrototype --> type Prototype__Map struct{} func (Prototype__Map) NewBuilder() datamodel.NodeBuilder { return &plainMap__Builder{plainMap__Assembler{w: &plainMap{}}} } // -- NodeBuilder --> type plainMap__Builder struct { plainMap__Assembler } func (nb *plainMap__Builder) Build() datamodel.Node { if nb.state != maState_finished { panic("invalid state: assembler must be 'finished' before Build can be called!") } return nb.w } func (nb *plainMap__Builder) Reset() { *nb = plainMap__Builder{} nb.w = &plainMap{} } // -- NodeAssembler --> type plainMap__Assembler struct { w *plainMap ka plainMap__KeyAssembler va plainMap__ValueAssembler state maState } type plainMap__KeyAssembler struct { ma *plainMap__Assembler } type plainMap__ValueAssembler struct { ma *plainMap__Assembler } // maState is an enum of the state machine for a map assembler. // (this might be something to export reusably, but it's also very much an impl detail that need not be seen, so, dubious.) type maState uint8 const ( maState_initial maState = iota // also the 'expect key or finish' state maState_midKey // waiting for a 'finished' state in the KeyAssembler. maState_expectValue // 'AssembleValue' is the only valid next step maState_midValue // waiting for a 'finished' state in the ValueAssembler. maState_finished // 'w' will also be nil, but this is a politer statement ) func (na *plainMap__Assembler) BeginMap(sizeHint int64) (datamodel.MapAssembler, error) { if sizeHint < 0 { sizeHint = 0 } // Allocate storage space. na.w.t = make([]plainMap__Entry, 0, sizeHint) na.w.m = make(map[string]datamodel.Node, sizeHint) // That's it; return self as the MapAssembler. We already have all the right methods on this structure. return na, nil } func (plainMap__Assembler) BeginList(sizeHint int64) (datamodel.ListAssembler, error) { return mixins.MapAssembler{TypeName: "map"}.BeginList(0) } func (plainMap__Assembler) AssignNull() error { return mixins.MapAssembler{TypeName: "map"}.AssignNull() } func (plainMap__Assembler) AssignBool(bool) error { return mixins.MapAssembler{TypeName: "map"}.AssignBool(false) } func (plainMap__Assembler) AssignInt(int64) error { return mixins.MapAssembler{TypeName: "map"}.AssignInt(0) } func (plainMap__Assembler) AssignFloat(float64) error { return mixins.MapAssembler{TypeName: "map"}.AssignFloat(0) } func (plainMap__Assembler) AssignString(string) error { return mixins.MapAssembler{TypeName: "map"}.AssignString("") } func (plainMap__Assembler) AssignBytes([]byte) error { return mixins.MapAssembler{TypeName: "map"}.AssignBytes(nil) } func (plainMap__Assembler) AssignLink(datamodel.Link) error { return mixins.MapAssembler{TypeName: "map"}.AssignLink(nil) } func (na *plainMap__Assembler) AssignNode(v datamodel.Node) error { // Sanity check assembler state. // Update of state to 'finished' comes later; where exactly depends on if shortcuts apply. if na.state != maState_initial { panic("misuse") } // Copy the content. if v2, ok := v.(*plainMap); ok { // if our own type: shortcut. // Copy the structure by value. // This means we'll have pointers into the same internal maps and slices; // this is okay, because the Node type promises it's immutable, and we are going to instantly finish ourselves to also maintain that. // FIXME: the shortcut behaves differently than the long way: it discards any existing progress. Doesn't violate immut, but is odd. *na.w = *v2 na.state = maState_finished return nil } // If the above shortcut didn't work, resort to a generic copy. // We call AssignNode for all the child values, giving them a chance to hit shortcuts even if we didn't. if v.Kind() != datamodel.Kind_Map { return datamodel.ErrWrongKind{TypeName: "map", MethodName: "AssignNode", AppropriateKind: datamodel.KindSet_JustMap, ActualKind: v.Kind()} } itr := v.MapIterator() for !itr.Done() { k, v, err := itr.Next() if err != nil { return err } if err := na.AssembleKey().AssignNode(k); err != nil { return err } if err := na.AssembleValue().AssignNode(v); err != nil { return err } } return na.Finish() } func (plainMap__Assembler) Prototype() datamodel.NodePrototype { return Prototype.Map } // -- MapAssembler --> // AssembleEntry is part of conforming to MapAssembler, which we do on // plainMap__Assembler so that BeginMap can just return a retyped pointer rather than new object. func (ma *plainMap__Assembler) AssembleEntry(k string) (datamodel.NodeAssembler, error) { // Sanity check assembler state. // Update of state comes after possible key rejection. if ma.state != maState_initial { panic("misuse") } // Check for dup keys; error if so. _, exists := ma.w.m[k] if exists { return nil, datamodel.ErrRepeatedMapKey{Key: plainString(k)} } ma.state = maState_midValue ma.w.t = append(ma.w.t, plainMap__Entry{k: plainString(k)}) // Make value assembler valid by giving it pointer back to whole 'ma'; yield it. ma.va.ma = ma return &ma.va, nil } // AssembleKey is part of conforming to MapAssembler, which we do on // plainMap__Assembler so that BeginMap can just return a retyped pointer rather than new object. func (ma *plainMap__Assembler) AssembleKey() datamodel.NodeAssembler { // Sanity check, then update, assembler state. if ma.state != maState_initial { panic("misuse") } ma.state = maState_midKey // Make key assembler valid by giving it pointer back to whole 'ma'; yield it. ma.ka.ma = ma return &ma.ka } // AssembleValue is part of conforming to MapAssembler, which we do on // plainMap__Assembler so that BeginMap can just return a retyped pointer rather than new object. func (ma *plainMap__Assembler) AssembleValue() datamodel.NodeAssembler { // Sanity check, then update, assembler state. if ma.state != maState_expectValue { panic("misuse") } ma.state = maState_midValue // Make value assembler valid by giving it pointer back to whole 'ma'; yield it. ma.va.ma = ma return &ma.va } // Finish is part of conforming to MapAssembler, which we do on // plainMap__Assembler so that BeginMap can just return a retyped pointer rather than new object. func (ma *plainMap__Assembler) Finish() error { // Sanity check, then update, assembler state. if ma.state != maState_initial { panic("misuse") } ma.state = maState_finished // validators could run and report errors promptly, if this type had any. return nil } func (plainMap__Assembler) KeyPrototype() datamodel.NodePrototype { return Prototype__String{} } func (plainMap__Assembler) ValuePrototype(_ string) datamodel.NodePrototype { return Prototype.Any } // -- MapAssembler.KeyAssembler --> func (plainMap__KeyAssembler) BeginMap(sizeHint int64) (datamodel.MapAssembler, error) { return mixins.StringAssembler{TypeName: "string"}.BeginMap(0) } func (plainMap__KeyAssembler) BeginList(sizeHint int64) (datamodel.ListAssembler, error) { return mixins.StringAssembler{TypeName: "string"}.BeginList(0) } func (plainMap__KeyAssembler) AssignNull() error { return mixins.StringAssembler{TypeName: "string"}.AssignNull() } func (plainMap__KeyAssembler) AssignBool(bool) error { return mixins.StringAssembler{TypeName: "string"}.AssignBool(false) } func (plainMap__KeyAssembler) AssignInt(int64) error { return mixins.StringAssembler{TypeName: "string"}.AssignInt(0) } func (plainMap__KeyAssembler) AssignFloat(float64) error { return mixins.StringAssembler{TypeName: "string"}.AssignFloat(0) } func (mka *plainMap__KeyAssembler) AssignString(v string) error { // Check for dup keys; error if so. // (And, backtrack state to accepting keys again so we don't get eternally wedged here.) _, exists := mka.ma.w.m[v] if exists { mka.ma.state = maState_initial mka.ma = nil // invalidate self to prevent further incorrect use. return datamodel.ErrRepeatedMapKey{Key: plainString(v)} } // Assign the key into the end of the entry table; // we'll be doing map insertions after we get the value in hand. // (There's no need to delegate to another assembler for the key type, // because we're just at Data Model level here, which only regards plain strings.) mka.ma.w.t = append(mka.ma.w.t, plainMap__Entry{}) mka.ma.w.t[len(mka.ma.w.t)-1].k = plainString(v) // Update parent assembler state: clear to proceed. mka.ma.state = maState_expectValue mka.ma = nil // invalidate self to prevent further incorrect use. return nil } func (plainMap__KeyAssembler) AssignBytes([]byte) error { return mixins.StringAssembler{TypeName: "string"}.AssignBytes(nil) } func (plainMap__KeyAssembler) AssignLink(datamodel.Link) error { return mixins.StringAssembler{TypeName: "string"}.AssignLink(nil) } func (mka *plainMap__KeyAssembler) AssignNode(v datamodel.Node) error { vs, err := v.AsString() if err != nil { return fmt.Errorf("cannot assign non-string node into map key assembler") // FIXME:errors: this doesn't quite fit in ErrWrongKind cleanly; new error type? } return mka.AssignString(vs) } func (plainMap__KeyAssembler) Prototype() datamodel.NodePrototype { return Prototype__String{} } // -- MapAssembler.ValueAssembler --> func (mva *plainMap__ValueAssembler) BeginMap(sizeHint int64) (datamodel.MapAssembler, error) { ma := plainMap__ValueAssemblerMap{} ma.ca.w = &plainMap{} ma.p = mva.ma _, err := ma.ca.BeginMap(sizeHint) return &ma, err } func (mva *plainMap__ValueAssembler) BeginList(sizeHint int64) (datamodel.ListAssembler, error) { la := plainMap__ValueAssemblerList{} la.ca.w = &plainList{} la.p = mva.ma _, err := la.ca.BeginList(sizeHint) return &la, err } func (mva *plainMap__ValueAssembler) AssignNull() error { return mva.AssignNode(datamodel.Null) } func (mva *plainMap__ValueAssembler) AssignBool(v bool) error { vb := plainBool(v) return mva.AssignNode(&vb) } func (mva *plainMap__ValueAssembler) AssignInt(v int64) error { vb := plainInt(v) return mva.AssignNode(&vb) } func (mva *plainMap__ValueAssembler) AssignFloat(v float64) error { vb := plainFloat(v) return mva.AssignNode(&vb) } func (mva *plainMap__ValueAssembler) AssignString(v string) error { vb := plainString(v) return mva.AssignNode(&vb) } func (mva *plainMap__ValueAssembler) AssignBytes(v []byte) error { vb := plainBytes(v) return mva.AssignNode(&vb) } func (mva *plainMap__ValueAssembler) AssignLink(v datamodel.Link) error { vb := plainLink{v} return mva.AssignNode(&vb) } func (mva *plainMap__ValueAssembler) AssignNode(v datamodel.Node) error { l := len(mva.ma.w.t) - 1 mva.ma.w.t[l].v = v mva.ma.w.m[string(mva.ma.w.t[l].k)] = v mva.ma.state = maState_initial mva.ma = nil // invalidate self to prevent further incorrect use. return nil } func (plainMap__ValueAssembler) Prototype() datamodel.NodePrototype { return Prototype.Any } type plainMap__ValueAssemblerMap struct { ca plainMap__Assembler p *plainMap__Assembler // pointer back to parent, for final insert and state bump } // we briefly state only the methods we need to delegate here. // just embedding plainMap__Assembler also behaves correctly, // but causes a lot of unnecessary autogenerated functions in the final binary. func (ma *plainMap__ValueAssemblerMap) AssembleEntry(k string) (datamodel.NodeAssembler, error) { return ma.ca.AssembleEntry(k) } func (ma *plainMap__ValueAssemblerMap) AssembleKey() datamodel.NodeAssembler { return ma.ca.AssembleKey() } func (ma *plainMap__ValueAssemblerMap) AssembleValue() datamodel.NodeAssembler { return ma.ca.AssembleValue() } func (plainMap__ValueAssemblerMap) KeyPrototype() datamodel.NodePrototype { return Prototype__String{} } func (plainMap__ValueAssemblerMap) ValuePrototype(_ string) datamodel.NodePrototype { return Prototype.Any } func (ma *plainMap__ValueAssemblerMap) Finish() error { if err := ma.ca.Finish(); err != nil { return err } w := ma.ca.w ma.ca.w = nil return ma.p.va.AssignNode(w) } type plainMap__ValueAssemblerList struct { ca plainList__Assembler p *plainMap__Assembler // pointer back to parent, for final insert and state bump } // we briefly state only the methods we need to delegate here. // just embedding plainList__Assembler also behaves correctly, // but causes a lot of unnecessary autogenerated functions in the final binary. func (la *plainMap__ValueAssemblerList) AssembleValue() datamodel.NodeAssembler { return la.ca.AssembleValue() } func (plainMap__ValueAssemblerList) ValuePrototype(_ int64) datamodel.NodePrototype { return Prototype.Any } func (la *plainMap__ValueAssemblerList) Finish() error { if err := la.ca.Finish(); err != nil { return err } w := la.ca.w la.ca.w = nil return la.p.va.AssignNode(w) } ================================================ FILE: node/basicnode/map_test.go ================================================ package basicnode_test import ( "fmt" "testing" qt "github.com/frankban/quicktest" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/must" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/node/tests" "github.com/ipld/go-ipld-prime/printer" ) func TestMap(t *testing.T) { tests.SpecTestMapStrInt(t, basicnode.Prototype.Map) tests.SpecTestMapStrMapStrInt(t, basicnode.Prototype.Map) tests.SpecTestMapStrListStr(t, basicnode.Prototype.Map) } func BenchmarkMapStrInt_3n_AssembleStandard(b *testing.B) { tests.SpecBenchmarkMapStrInt_3n_AssembleStandard(b, basicnode.Prototype.Map) } func BenchmarkMapStrInt_3n_AssembleEntry(b *testing.B) { tests.SpecBenchmarkMapStrInt_3n_AssembleEntry(b, basicnode.Prototype.Map) } func BenchmarkMapStrInt_3n_Iteration(b *testing.B) { tests.SpecBenchmarkMapStrInt_3n_Iteration(b, basicnode.Prototype.Map) } func BenchmarkMapStrInt_25n_AssembleStandard(b *testing.B) { tests.SpecBenchmarkMapStrInt_25n_AssembleStandard(b, basicnode.Prototype.Map) } func BenchmarkMapStrInt_25n_AssembleEntry(b *testing.B) { tests.SpecBenchmarkMapStrInt_25n_AssembleEntry(b, basicnode.Prototype.Map) } func BenchmarkMapStrInt_25n_Iteration(b *testing.B) { tests.SpecBenchmarkMapStrInt_25n_Iteration(b, basicnode.Prototype.Map) } func BenchmarkSpec_Marshal_Map3StrInt(b *testing.B) { tests.BenchmarkSpec_Marshal_Map3StrInt(b, basicnode.Prototype.Map) } func BenchmarkSpec_Marshal_MapNStrMap3StrInt(b *testing.B) { tests.BenchmarkSpec_Marshal_MapNStrMap3StrInt(b, basicnode.Prototype.Map) } func BenchmarkSpec_Unmarshal_Map3StrInt(b *testing.B) { tests.BenchmarkSpec_Unmarshal_Map3StrInt(b, basicnode.Prototype.Map) } func BenchmarkSpec_Unmarshal_MapNStrMap3StrInt(b *testing.B) { tests.BenchmarkSpec_Unmarshal_MapNStrMap3StrInt(b, basicnode.Prototype.Map) } // Test that the map builder cannot be assigned arbitrary values, and trying to // will result in a sensible error func TestMapAssignError(t *testing.T) { b := basicnode.Prototype.Map.NewBuilder() err := b.AssignBool(true) errExpect := `func called on wrong kind: "AssignBool" called on a map node \(kind: map\), but only makes sense on bool` qt.Check(t, err, qt.ErrorMatches, errExpect) err = b.AssignInt(3) errExpect = `func called on wrong kind: "AssignInt" called on a map node \(kind: map\), but only makes sense on int` qt.Check(t, err, qt.ErrorMatches, errExpect) err = b.AssignFloat(5.7) errExpect = `func called on wrong kind: "AssignFloat" called on a map node \(kind: map\), but only makes sense on float` qt.Check(t, err, qt.ErrorMatches, errExpect) err = b.AssignString("hi") errExpect = `func called on wrong kind: "AssignString" called on a map node \(kind: map\), but only makes sense on string` qt.Check(t, err, qt.ErrorMatches, errExpect) err = b.AssignNode(basicnode.NewInt(3)) errExpect = `func called on wrong kind: "AssignNode" called on a map node \(kind: int\), but only makes sense on map` qt.Check(t, err, qt.ErrorMatches, errExpect) // TODO(dustmop): BeginList, AssignNull, AssignBytes, AssignLink } // Test that the map builder can create map nodes, and AssignNode will copy // such a node, and lookup methods on that node will work correctly func TestMapBuilder(t *testing.T) { b := basicnode.Prototype.Map.NewBuilder() // construct a map of three keys, using the MapBuilder ma, err := b.BeginMap(3) if err != nil { t.Fatal(err) } a := ma.AssembleKey() a.AssignString("cat") a = ma.AssembleValue() a.AssignString("meow") a, err = ma.AssembleEntry("dog") if err != nil { t.Fatal(err) } a.AssignString("bark") a = ma.AssembleKey() a.AssignString("eel") a = ma.AssembleValue() a.AssignString("zap") err = ma.Finish() if err != nil { t.Fatal(err) } // test the builder's prototypes and its key and value prototypes, while we're here np := b.Prototype() qt.Check(t, fmt.Sprintf("%T", np), qt.Equals, "basicnode.Prototype__Map") np = ma.KeyPrototype() qt.Check(t, fmt.Sprintf("%T", np), qt.Equals, "basicnode.Prototype__String") np = ma.ValuePrototype("") qt.Check(t, fmt.Sprintf("%T", np), qt.Equals, "basicnode.Prototype__Any") // compare the printed map mapNode := b.Build() actual := printer.Sprint(mapNode) expect := `map{ string{"cat"}: string{"meow"} string{"dog"}: string{"bark"} string{"eel"}: string{"zap"} }` qt.Check(t, expect, qt.Equals, actual) // copy the map using AssignNode c := basicnode.Prototype.Map.NewBuilder() err = c.AssignNode(mapNode) if err != nil { t.Fatal(err) } anotherNode := c.Build() actual = printer.Sprint(anotherNode) qt.Assert(t, expect, qt.Equals, actual) // access values of map, using string r, err := anotherNode.LookupByString("cat") if err != nil { t.Fatal(err) } qt.Check(t, "meow", qt.Equals, must.String(r)) // access values of map, using node r, err = anotherNode.LookupByNode(basicnode.NewString("dog")) if err != nil { t.Fatal(err) } qt.Check(t, "bark", qt.Equals, must.String(r)) // access values of map, using PathSegment r, err = anotherNode.LookupBySegment(datamodel.ParsePathSegment("eel")) if err != nil { t.Fatal(err) } qt.Check(t, "zap", qt.Equals, must.String(r)) // validate the node's prototype np = anotherNode.Prototype() qt.Check(t, fmt.Sprintf("%T", np), qt.Equals, "basicnode.Prototype__Map") } // test that AssignNode will fail if called twice, it expects an empty // node to assign to func TestMapCantAssignNodeTwice(t *testing.T) { b := basicnode.Prototype.Map.NewBuilder() // construct a map of three keys, using the MapBuilder ma, err := b.BeginMap(3) if err != nil { t.Fatal(err) } a := ma.AssembleKey() a.AssignString("cat") a = ma.AssembleValue() a.AssignString("meow") a, err = ma.AssembleEntry("dog") if err != nil { t.Fatal(err) } a.AssignString("bark") a = ma.AssembleKey() a.AssignString("eel") a = ma.AssembleValue() a.AssignString("zap") err = ma.Finish() if err != nil { t.Fatal(err) } mapNode := b.Build() // copy the map using AssignNode, works the first time c := basicnode.Prototype.Map.NewBuilder() err = c.AssignNode(mapNode) if err != nil { t.Fatal(err) } qt.Assert(t, func() { _ = c.AssignNode(mapNode) }, qt.PanicMatches, // TODO(dustmop): Error message here should be better `misuse`) } func TestMapLookupError(t *testing.T) { b := basicnode.Prototype.Map.NewBuilder() // construct a map of three keys, using the MapBuilder ma, err := b.BeginMap(3) if err != nil { t.Fatal(err) } a := ma.AssembleKey() a.AssignString("cat") a = ma.AssembleValue() a.AssignString("meow") a, err = ma.AssembleEntry("dog") if err != nil { t.Fatal(err) } a.AssignString("bark") a = ma.AssembleKey() a.AssignString("eel") a = ma.AssembleValue() a.AssignString("zap") err = ma.Finish() if err != nil { t.Fatal(err) } mapNode := b.Build() _, err = mapNode.LookupByString("frog") qt.Check(t, err, qt.ErrorMatches, `key not found: "frog"`) _, err = mapNode.LookupByNode(basicnode.NewInt(3)) // TODO(dustmop): This error message is not great. It's about how the // int node could not be converted when the real problem is that this // method should not accept ints as parameters qt.Check(t, err, qt.ErrorMatches, `func called on wrong kind: "AsString" called on a int node \(kind: int\), but only makes sense on string`) _, err = mapNode.LookupByIndex(0) qt.Check(t, err, qt.ErrorMatches, `func called on wrong kind: "LookupByIndex" called on a map node \(kind: map\), but only makes sense on list`) } func TestMapNewBuilderUsageError(t *testing.T) { qt.Assert(t, func() { b := basicnode.Prototype.Map.NewBuilder() _ = b.Build() }, qt.PanicMatches, `invalid state: assembler must be 'finished' before Build can be called!`) // construct an empty map b := basicnode.Prototype.Map.NewBuilder() ma, err := b.BeginMap(0) if err != nil { t.Fatal(err) } err = ma.Finish() if err != nil { t.Fatal(err) } mapNode := b.Build() actual := printer.Sprint(mapNode) expect := `map{}` qt.Check(t, expect, qt.Equals, actual) // reset will return the state to 'initial', so Build will panic once again b.Reset() qt.Assert(t, func() { _ = b.Build() }, qt.PanicMatches, `invalid state: assembler must be 'finished' before Build can be called!`) // assembling a key without a value will cause Finish to panic b.Reset() ma, err = b.BeginMap(0) if err != nil { t.Fatal(err) } a := ma.AssembleKey() a.AssignString("cat") qt.Assert(t, func() { _ = ma.Finish() }, qt.PanicMatches, // TODO(dustmop): Error message here should be better `misuse`) } func TestMapDupKeyError(t *testing.T) { b := basicnode.Prototype.Map.NewBuilder() // construct a map with duplicate keys ma, err := b.BeginMap(3) if err != nil { t.Fatal(err) } a := ma.AssembleKey() a.AssignString("cat") a = ma.AssembleValue() a.AssignString("meow") a = ma.AssembleKey() err = a.AssignString("cat") qt.Check(t, err, qt.ErrorMatches, `cannot repeat map key "cat"`) } ================================================ FILE: node/basicnode/prototypes.go ================================================ package basicnode // Prototype embeds a NodePrototype for every kind of Node implementation in this package. // You can use it like this: // // basicnode.Prototype.Map.NewBuilder().BeginMap() //... // // and: // // basicnode.Prototype.String.NewBuilder().AssignString("x") // ... // // Most of the prototypes are for one particular Kind of node (e.g. string, int, etc); // you can use the "Any" style if you want a builder that can accept any kind of data. var Prototype prototype type prototype struct { Any Prototype__Any Map Prototype__Map List Prototype__List Bool Prototype__Bool Int Prototype__Int Float Prototype__Float String Prototype__String Bytes Prototype__Bytes Link Prototype__Link } ================================================ FILE: node/basicnode/string.go ================================================ package basicnode import ( "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/node/mixins" ) var ( _ datamodel.Node = plainString("") _ datamodel.NodePrototype = Prototype__String{} _ datamodel.NodeBuilder = &plainString__Builder{} _ datamodel.NodeAssembler = &plainString__Assembler{} ) func NewString(value string) datamodel.Node { v := plainString(value) return &v } // plainString is a simple boxed string that complies with datamodel.Node. // It's useful for many things, such as boxing map keys. // // The implementation is a simple typedef of a string; // handling it as a Node incurs 'runtime.convTstring', // which is about the best we can do. type plainString string // -- Node interface methods --> func (plainString) Kind() datamodel.Kind { return datamodel.Kind_String } func (plainString) LookupByString(string) (datamodel.Node, error) { return mixins.String{TypeName: "string"}.LookupByString("") } func (plainString) LookupByNode(key datamodel.Node) (datamodel.Node, error) { return mixins.String{TypeName: "string"}.LookupByNode(nil) } func (plainString) LookupByIndex(idx int64) (datamodel.Node, error) { return mixins.String{TypeName: "string"}.LookupByIndex(0) } func (plainString) LookupBySegment(seg datamodel.PathSegment) (datamodel.Node, error) { return mixins.String{TypeName: "string"}.LookupBySegment(seg) } func (plainString) MapIterator() datamodel.MapIterator { return nil } func (plainString) ListIterator() datamodel.ListIterator { return nil } func (plainString) Length() int64 { return -1 } func (plainString) IsAbsent() bool { return false } func (plainString) IsNull() bool { return false } func (plainString) AsBool() (bool, error) { return mixins.String{TypeName: "string"}.AsBool() } func (plainString) AsInt() (int64, error) { return mixins.String{TypeName: "string"}.AsInt() } func (plainString) AsFloat() (float64, error) { return mixins.String{TypeName: "string"}.AsFloat() } func (x plainString) AsString() (string, error) { return string(x), nil } func (plainString) AsBytes() ([]byte, error) { return mixins.String{TypeName: "string"}.AsBytes() } func (plainString) AsLink() (datamodel.Link, error) { return mixins.String{TypeName: "string"}.AsLink() } func (plainString) Prototype() datamodel.NodePrototype { return Prototype__String{} } // -- NodePrototype --> type Prototype__String struct{} func (Prototype__String) NewBuilder() datamodel.NodeBuilder { var w plainString return &plainString__Builder{plainString__Assembler{w: &w}} } // -- NodeBuilder --> type plainString__Builder struct { plainString__Assembler } func (nb *plainString__Builder) Build() datamodel.Node { return nb.w } func (nb *plainString__Builder) Reset() { var w plainString *nb = plainString__Builder{plainString__Assembler{w: &w}} } // -- NodeAssembler --> type plainString__Assembler struct { w *plainString } func (plainString__Assembler) BeginMap(sizeHint int64) (datamodel.MapAssembler, error) { return mixins.StringAssembler{TypeName: "string"}.BeginMap(0) } func (plainString__Assembler) BeginList(sizeHint int64) (datamodel.ListAssembler, error) { return mixins.StringAssembler{TypeName: "string"}.BeginList(0) } func (plainString__Assembler) AssignNull() error { return mixins.StringAssembler{TypeName: "string"}.AssignNull() } func (plainString__Assembler) AssignBool(bool) error { return mixins.StringAssembler{TypeName: "string"}.AssignBool(false) } func (plainString__Assembler) AssignInt(int64) error { return mixins.StringAssembler{TypeName: "string"}.AssignInt(0) } func (plainString__Assembler) AssignFloat(float64) error { return mixins.StringAssembler{TypeName: "string"}.AssignFloat(0) } func (na *plainString__Assembler) AssignString(v string) error { *na.w = plainString(v) return nil } func (plainString__Assembler) AssignBytes([]byte) error { return mixins.StringAssembler{TypeName: "string"}.AssignBytes(nil) } func (plainString__Assembler) AssignLink(datamodel.Link) error { return mixins.StringAssembler{TypeName: "string"}.AssignLink(nil) } func (na *plainString__Assembler) AssignNode(v datamodel.Node) error { if v2, err := v.AsString(); err != nil { return err } else { *na.w = plainString(v2) return nil } } func (plainString__Assembler) Prototype() datamodel.NodePrototype { return Prototype__String{} } ================================================ FILE: node/basicnode/string_test.go ================================================ package basicnode_test import ( "testing" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/node/tests" ) func TestString(t *testing.T) { tests.SpecTestString(t, basicnode.Prototype__String{}) } ================================================ FILE: node/bindnode/api.go ================================================ // Package bindnode provides a datamodel.Node implementation via Go reflection. // // This package is EXPERIMENTAL; its behavior and API might change as it's still // in development. package bindnode import ( "reflect" "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/schema" ) // Prototype implements a schema.TypedPrototype given a Go pointer type and an // IPLD schema type. Note that the result is also a datamodel.NodePrototype. // // If both the Go type and schema type are supplied, it is assumed that they are // compatible with one another. // // If either the Go type or schema type are nil, we infer the missing type from // the other provided type. For example, we can infer an unnamed Go struct type // for a schema struct type, and we can infer a schema Int type for a Go int64 // type. The inferring logic is still a work in progress and subject to change. // At this time, inferring IPLD Unions and Enums from Go types is not supported. // // When supplying a non-nil ptrType, Prototype only obtains the Go pointer type // from it, so its underlying value will typically be nil. For example: // // proto := bindnode.Prototype((*goType)(nil), schemaType) func Prototype(ptrType interface{}, schemaType schema.Type, options ...Option) schema.TypedPrototype { if ptrType == nil && schemaType == nil { panic("bindnode: either ptrType or schemaType must not be nil") } cfg := applyOptions(options...) // TODO: if both are supplied, verify that they are compatible var goType reflect.Type if ptrType == nil { goType = inferGoType(schemaType, make(map[schema.TypeName]inferredStatus), 0) } else { goPtrType := reflect.TypeOf(ptrType) if goPtrType.Kind() != reflect.Ptr { panic("bindnode: ptrType must be a pointer") } goType = goPtrType.Elem() if goType.Kind() == reflect.Ptr { panic("bindnode: ptrType must not be a pointer to a pointer") } if schemaType == nil { schemaType = inferSchema(goType, 0) } else { verifyCompatibility(cfg, make(map[seenEntry]bool), goType, schemaType) } } return &_prototype{cfg: cfg, schemaType: schemaType, goType: goType} } type converter struct { kind schema.TypeKind customFromBool func(bool) (interface{}, error) customToBool func(interface{}) (bool, error) customFromInt func(int64) (interface{}, error) customToInt func(interface{}) (int64, error) customFromFloat func(float64) (interface{}, error) customToFloat func(interface{}) (float64, error) customFromString func(string) (interface{}, error) customToString func(interface{}) (string, error) customFromBytes func([]byte) (interface{}, error) customToBytes func(interface{}) ([]byte, error) customFromLink func(cid.Cid) (interface{}, error) customToLink func(interface{}) (cid.Cid, error) customFromAny func(datamodel.Node) (interface{}, error) customToAny func(interface{}) (datamodel.Node, error) } type config struct { namedConverters map[schema.TypeName]*converter typeConverters map[reflect.Type]*converter } // this mainly exists to short-circuit the nonPtrType() call; the `Type()` variant // exists for completeness func (c *config) converterFor(typeName schema.TypeName, val reflect.Value) *converter { if c == nil { return nil } if namedConverter, ok := c.namedConverters[typeName]; ok { return namedConverter } return c.typeConverters[nonPtrType(val)] } func (c *config) converterForType(typeName schema.TypeName, typ reflect.Type) *converter { if c == nil { return nil } if namedConverter, ok := c.namedConverters[typeName]; ok { return namedConverter } return c.typeConverters[typ] } // Option is able to apply custom options to the bindnode API type Option func(*config) // TypedBoolConverter adds custom converter functions for a particular // type as identified by a pointer in the first argument. // The fromFunc is of the form: func(bool) (interface{}, error) // and toFunc is of the form: func(interface{}) (bool, error) // where interface{} is a pointer form of the type we are converting. // // TypedBoolConverter is an EXPERIMENTAL API and may be removed or // changed in a future release. func TypedBoolConverter(ptrVal interface{}, from func(bool) (interface{}, error), to func(interface{}) (bool, error)) Option { customType := nonPtrType(reflect.ValueOf(ptrVal)) converter := &converter{ kind: schema.TypeKind_Bool, customFromBool: from, customToBool: to, } return func(cfg *config) { cfg.typeConverters[customType] = converter } } // TypedIntConverter adds custom converter functions for a particular // type as identified by a pointer in the first argument. // The fromFunc is of the form: func(int64) (interface{}, error) // and toFunc is of the form: func(interface{}) (int64, error) // where interface{} is a pointer form of the type we are converting. // // TypedIntConverter is an EXPERIMENTAL API and may be removed or // changed in a future release. func TypedIntConverter(ptrVal interface{}, from func(int64) (interface{}, error), to func(interface{}) (int64, error)) Option { customType := nonPtrType(reflect.ValueOf(ptrVal)) converter := &converter{ kind: schema.TypeKind_Int, customFromInt: from, customToInt: to, } return func(cfg *config) { cfg.typeConverters[customType] = converter } } // TypedFloatConverter adds custom converter functions for a particular // type as identified by a pointer in the first argument. // The fromFunc is of the form: func(float64) (interface{}, error) // and toFunc is of the form: func(interface{}) (float64, error) // where interface{} is a pointer form of the type we are converting. // // TypedFloatConverter is an EXPERIMENTAL API and may be removed or // changed in a future release. func TypedFloatConverter(ptrVal interface{}, from func(float64) (interface{}, error), to func(interface{}) (float64, error)) Option { customType := nonPtrType(reflect.ValueOf(ptrVal)) converter := &converter{ kind: schema.TypeKind_Float, customFromFloat: from, customToFloat: to, } return func(cfg *config) { cfg.typeConverters[customType] = converter } } // TypedStringConverter adds custom converter functions for a particular // type as identified by a pointer in the first argument. // The fromFunc is of the form: func(string) (interface{}, error) // and toFunc is of the form: func(interface{}) (string, error) // where interface{} is a pointer form of the type we are converting. // // TypedStringConverter is an EXPERIMENTAL API and may be removed or // changed in a future release. func TypedStringConverter(ptrVal interface{}, from func(string) (interface{}, error), to func(interface{}) (string, error)) Option { customType := nonPtrType(reflect.ValueOf(ptrVal)) converter := &converter{ kind: schema.TypeKind_String, customFromString: from, customToString: to, } return func(cfg *config) { cfg.typeConverters[customType] = converter } } // TypedBytesConverter adds custom converter functions for a particular // type as identified by a pointer in the first argument. // The fromFunc is of the form: func([]byte) (interface{}, error) // and toFunc is of the form: func(interface{}) ([]byte, error) // where interface{} is a pointer form of the type we are converting. // // TypedBytesConverter is an EXPERIMENTAL API and may be removed or // changed in a future release. func TypedBytesConverter(ptrVal interface{}, from func([]byte) (interface{}, error), to func(interface{}) ([]byte, error)) Option { customType := nonPtrType(reflect.ValueOf(ptrVal)) converter := &converter{ kind: schema.TypeKind_Bytes, customFromBytes: from, customToBytes: to, } return func(cfg *config) { cfg.typeConverters[customType] = converter } } // TypedLinkConverter adds custom converter functions for a particular // type as identified by a pointer in the first argument. // The fromFunc is of the form: func([]byte) (interface{}, error) // and toFunc is of the form: func(interface{}) ([]byte, error) // where interface{} is a pointer form of the type we are converting. // // Beware that this API is only compatible with cidlink.Link types in the data // model and may result in errors if attempting to convert from other // datamodel.Link types. // // TypedLinkConverter is an EXPERIMENTAL API and may be removed or // changed in a future release. func TypedLinkConverter(ptrVal interface{}, from func(cid.Cid) (interface{}, error), to func(interface{}) (cid.Cid, error)) Option { customType := nonPtrType(reflect.ValueOf(ptrVal)) converter := &converter{ kind: schema.TypeKind_Link, customFromLink: from, customToLink: to, } return func(cfg *config) { cfg.typeConverters[customType] = converter } } // TypedAnyConverter adds custom converter functions for a particular // type as identified by a pointer in the first argument. // The fromFunc is of the form: func(datamodel.Node) (interface{}, error) // and toFunc is of the form: func(interface{}) (datamodel.Node, error) // where interface{} is a pointer form of the type we are converting. // // This method should be able to deal with all forms of Any and return an error // if the expected data forms don't match the expected. // // TypedAnyConverter is an EXPERIMENTAL API and may be removed or // changed in a future release. func TypedAnyConverter(ptrVal interface{}, from func(datamodel.Node) (interface{}, error), to func(interface{}) (datamodel.Node, error)) Option { customType := nonPtrType(reflect.ValueOf(ptrVal)) converter := &converter{ kind: schema.TypeKind_Any, customFromAny: from, customToAny: to, } return func(cfg *config) { cfg.typeConverters[customType] = converter } } // NamedBoolConverter adds custom converter functions for given // named schema type. // The fromFunc is of the form: func(bool) (interface{}, error) // and toFunc is of the form: func(interface{}) (bool, error) // where interface{} is a pointer form of the type we are converting. // // NamedBoolConverter is an EXPERIMENTAL API and may be removed or // changed in a future release. func NamedBoolConverter(typeName schema.TypeName, from func(bool) (interface{}, error), to func(interface{}) (bool, error)) Option { converter := &converter{ kind: schema.TypeKind_Bool, customFromBool: from, customToBool: to, } return func(cfg *config) { cfg.namedConverters[typeName] = converter } } // NamedIntConverter adds custom converter functions for given // named schema type. // The fromFunc is of the form: func(int64) (interface{}, error) // and toFunc is of the form: func(interface{}) (int64, error) // where interface{} is a pointer form of the type we are converting. // // NamedIntConverter is an EXPERIMENTAL API and may be removed or // changed in a future release. func NamedIntConverter(typeName schema.TypeName, from func(int64) (interface{}, error), to func(interface{}) (int64, error)) Option { converter := &converter{ kind: schema.TypeKind_Int, customFromInt: from, customToInt: to, } return func(cfg *config) { cfg.namedConverters[typeName] = converter } } // NamedFloatConverter adds custom converter functions for given // named schema type. // The fromFunc is of the form: func(float64) (interface{}, error) // and toFunc is of the form: func(interface{}) (float64, error) // where interface{} is a pointer form of the type we are converting. // // NamedFloatConverter is an EXPERIMENTAL API and may be removed or // changed in a future release. func NamedFloatConverter(typeName schema.TypeName, from func(float64) (interface{}, error), to func(interface{}) (float64, error)) Option { converter := &converter{ kind: schema.TypeKind_Float, customFromFloat: from, customToFloat: to, } return func(cfg *config) { cfg.namedConverters[typeName] = converter } } // NamedStringConverter adds custom converter functions for given // named schema type. // The fromFunc is of the form: func(string) (interface{}, error) // and toFunc is of the form: func(interface{}) (string, error) // where interface{} is a pointer form of the type we are converting. // // NamedStringConverter is an EXPERIMENTAL API and may be removed or // changed in a future release. func NamedStringConverter(typeName schema.TypeName, from func(string) (interface{}, error), to func(interface{}) (string, error)) Option { converter := &converter{ kind: schema.TypeKind_String, customFromString: from, customToString: to, } return func(cfg *config) { cfg.namedConverters[typeName] = converter } } // NamedBytesConverter adds custom converter functions for given // named schema type. // The fromFunc is of the form: func([]byte) (interface{}, error) // and toFunc is of the form: func(interface{}) ([]byte, error) // where interface{} is a pointer form of the type we are converting. // // NamedBytesConverter is an EXPERIMENTAL API and may be removed or // changed in a future release. func NamedBytesConverter(typeName schema.TypeName, from func([]byte) (interface{}, error), to func(interface{}) ([]byte, error)) Option { converter := &converter{ kind: schema.TypeKind_Bytes, customFromBytes: from, customToBytes: to, } return func(cfg *config) { cfg.namedConverters[typeName] = converter } } // NamedLinkConverter adds custom converter functions for given // named schema type. // The fromFunc is of the form: func([]byte) (interface{}, error) // and toFunc is of the form: func(interface{}) ([]byte, error) // where interface{} is a pointer form of the type we are converting. // // Beware that this API is only compatible with cidlink.Link types in the data // model and may result in errors if attempting to convert from other // datamodel.Link types. // // NamedLinkConverter is an EXPERIMENTAL API and may be removed or // changed in a future release. func NamedLinkConverter(typeName schema.TypeName, from func(cid.Cid) (interface{}, error), to func(interface{}) (cid.Cid, error)) Option { converter := &converter{ kind: schema.TypeKind_Link, customFromLink: from, customToLink: to, } return func(cfg *config) { cfg.namedConverters[typeName] = converter } } // NamedAnyConverter adds custom converter functions for given // named schema type. // The fromFunc is of the form: func(datamodel.Node) (interface{}, error) // and toFunc is of the form: func(interface{}) (datamodel.Node, error) // where interface{} is a pointer form of the type we are converting. // // This method should be able to deal with all forms of Any and return an error // if the expected data forms don't match the expected. // // NamedAnyConverter is an EXPERIMENTAL API and may be removed or // changed in a future release. func NamedAnyConverter(typeName schema.TypeName, from func(datamodel.Node) (interface{}, error), to func(interface{}) (datamodel.Node, error)) Option { converter := &converter{ kind: schema.TypeKind_Any, customFromAny: from, customToAny: to, } return func(cfg *config) { cfg.namedConverters[typeName] = converter } } func applyOptions(opt ...Option) *config { if len(opt) == 0 { // no need to allocate, we access it via converterFor and converterForType // which are safe for nil maps return nil } cfg := &config{ namedConverters: make(map[string]*converter), typeConverters: make(map[reflect.Type]*converter), } for _, o := range opt { o(cfg) } return cfg } // Wrap implements a schema.TypedNode given a non-nil pointer to a Go value and an // IPLD schema type. Note that the result is also a datamodel.Node. // // Wrap is meant to be used when one already has a Go value with data. // As such, ptrVal must not be nil. // // Similar to Prototype, if schemaType is non-nil it is assumed to be compatible // with the Go type, and otherwise it's inferred from the Go type. func Wrap(ptrVal interface{}, schemaType schema.Type, options ...Option) schema.TypedNode { if ptrVal == nil { panic("bindnode: ptrVal must not be nil") } goPtrVal := reflect.ValueOf(ptrVal) if goPtrVal.Kind() != reflect.Ptr { panic("bindnode: ptrVal must be a pointer") } if goPtrVal.IsNil() { // Note that this can happen if ptrVal was a typed nil. panic("bindnode: ptrVal must not be nil") } cfg := applyOptions(options...) goVal := goPtrVal.Elem() if goVal.Kind() == reflect.Ptr { panic("bindnode: ptrVal must not be a pointer to a pointer") } if schemaType == nil { schemaType = inferSchema(goVal.Type(), 0) } else { // TODO(rvagg): explore ways to make this skippable by caching in the schema.Type // passed in to this function; e.g. if you call Prototype(), then you've gone through // this already, then calling .Type() on that could return a bindnode version of // schema.Type that has the config cached and can be assumed to have been checked or // inferred. verifyCompatibility(cfg, make(map[seenEntry]bool), goVal.Type(), schemaType) } return newNode(cfg, schemaType, goVal) } // TODO: consider making our own Node interface, like: // // type WrappedNode interface { // datamodel.Node // Unwrap() (ptrVal interface) // } // // Pros: API is easier to understand, harder to mix up with other datamodel.Nodes. // Cons: One usually only has a datamodel.Node, and type assertions can be weird. // Unwrap takes a datamodel.Node implemented by Prototype or Wrap, // and returns a pointer to the inner Go value. // // Unwrap returns nil if the node isn't implemented by this package. func Unwrap(node datamodel.Node) (ptrVal interface{}) { var val reflect.Value switch node := node.(type) { case *_node: val = node.val case *_nodeRepr: val = node.val default: return nil } if val.Kind() == reflect.Ptr { panic("bindnode: didn't expect val to be a pointer") } return val.Addr().Interface() } ================================================ FILE: node/bindnode/api_test.go ================================================ package bindnode_test import ( "encoding/hex" "math" "testing" qt "github.com/frankban/quicktest" "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/codec/dagcbor" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/node/bindnode" ) func TestEnumError(t *testing.T) { type Action string const ( ActionPresent = Action("p") ActionMissing = Action("m") ) type S struct{ Action Action } schema := ` type S struct { Action Action } representation tuple type Action enum { | Present ("p") | Missing ("m") } representation string ` typeSystem, err := ipld.LoadSchemaBytes([]byte(schema)) qt.Assert(t, err, qt.IsNil) schemaType := typeSystem.TypeByName("S") node := bindnode.Wrap(&S{Action: ActionPresent}, schemaType).Representation() _, err = ipld.Encode(node, dagcbor.Encode) qt.Assert(t, err, qt.IsNotNil) qt.Assert(t, err.Error(), qt.Equals, `AsString: "p" is not a valid member of enum Action (bindnode works at the type level; did you mean "Present"?)`) } func TestSubNodeWalkAndUnwrap(t *testing.T) { type F struct { F bool } type B struct { B int } type A struct { A string } type S struct { F F B B A *A Any datamodel.Node Bytes []byte String string } schema := ` type F struct { F Bool } representation tuple type B struct { B Int } representation tuple type A struct { A String } representation tuple type S struct { F F B B A optional A Any Any Bytes Bytes String String } representation tuple ` encodedHex := "8681f581186581624141fb4069466666666666430102036f636f6e7374616e7420737472696e67" // [[true],[101],["AA"],202.2,[]byte{1,2,3},"constant string"] byts := []byte{0, 1, 2, 3} const constStr = "constant string" expected := S{ F: F{true}, B: B{101}, A: &A{"AAA"[1:]}, Any: basicnode.NewFloat(202.2), Bytes: byts[1:], String: constStr, } typeSystem, err := ipld.LoadSchemaBytes([]byte(schema)) qt.Assert(t, err, qt.IsNil) schemaType := typeSystem.TypeByName("S") verifyMap := func(node datamodel.Node) { mi := node.MapIterator() key, value, err := mi.Next() qt.Assert(t, err, qt.IsNil) str, err := key.AsString() qt.Assert(t, err, qt.IsNil) qt.Assert(t, str, qt.Equals, "F") typ := bindnode.Unwrap(value) instF, ok := typ.(*F) qt.Assert(t, ok, qt.IsTrue) qt.Assert(t, *instF, qt.Equals, F{true}) key, value, err = mi.Next() qt.Assert(t, err, qt.IsNil) str, err = key.AsString() qt.Assert(t, err, qt.IsNil) qt.Assert(t, str, qt.Equals, "B") typ = bindnode.Unwrap(value) instB, ok := typ.(*B) qt.Assert(t, ok, qt.IsTrue) qt.Assert(t, *instB, qt.Equals, B{101}) key, value, err = mi.Next() qt.Assert(t, err, qt.IsNil) str, err = key.AsString() qt.Assert(t, err, qt.IsNil) qt.Assert(t, str, qt.Equals, "A") typ = bindnode.Unwrap(value) instA, ok := typ.(*A) qt.Assert(t, ok, qt.IsTrue) qt.Assert(t, *instA, qt.Equals, A{"AA"}) key, value, err = mi.Next() qt.Assert(t, err, qt.IsNil) str, err = key.AsString() qt.Assert(t, err, qt.IsNil) qt.Assert(t, str, qt.Equals, "Any") qt.Assert(t, ipld.DeepEqual(basicnode.NewFloat(202.2), value), qt.IsTrue) key, value, err = mi.Next() qt.Assert(t, err, qt.IsNil) str, err = key.AsString() qt.Assert(t, err, qt.IsNil) qt.Assert(t, str, qt.Equals, "Bytes") typ = bindnode.Unwrap(value) instByts, ok := typ.(*[]byte) qt.Assert(t, ok, qt.IsTrue) qt.Assert(t, *instByts, qt.DeepEquals, []byte{1, 2, 3}) key, value, err = mi.Next() qt.Assert(t, err, qt.IsNil) str, err = key.AsString() qt.Assert(t, err, qt.IsNil) qt.Assert(t, str, qt.Equals, "String") typ = bindnode.Unwrap(value) instStr, ok := typ.(*string) qt.Assert(t, ok, qt.IsTrue) qt.Assert(t, *instStr, qt.DeepEquals, "constant string") } t.Run("decode", func(t *testing.T) { encoded, _ := hex.DecodeString(encodedHex) proto := bindnode.Prototype(&S{}, schemaType) node, err := ipld.DecodeUsingPrototype([]byte(encoded), dagcbor.Decode, proto) qt.Assert(t, err, qt.IsNil) typ := bindnode.Unwrap(node) instS, ok := typ.(*S) qt.Assert(t, ok, qt.IsTrue) qt.Assert(t, *instS, qt.DeepEquals, expected) verifyMap(node) }) t.Run("encode", func(t *testing.T) { node := bindnode.Wrap(&expected, schemaType) byts, err := ipld.Encode(node, dagcbor.Encode) qt.Assert(t, err, qt.IsNil) qt.Assert(t, hex.EncodeToString(byts), qt.Equals, encodedHex) verifyMap(node) }) } func TestUint64Struct(t *testing.T) { t.Run("in struct", func(t *testing.T) { type IntHolder struct { Int32 int32 Int64 int64 Uint64 uint64 } schema := ` type IntHolder struct { Int32 Int Int64 Int Uint64 Int } ` maxExpectedHex := "a365496e7433321a7fffffff65496e7436341b7fffffffffffffff6655696e7436341bffffffffffffffff" maxExpected, err := hex.DecodeString(maxExpectedHex) qt.Assert(t, err, qt.IsNil) typeSystem, err := ipld.LoadSchemaBytes([]byte(schema)) qt.Assert(t, err, qt.IsNil) schemaType := typeSystem.TypeByName("IntHolder") proto := bindnode.Prototype(&IntHolder{}, schemaType) node, err := ipld.DecodeUsingPrototype([]byte(maxExpected), dagcbor.Decode, proto) qt.Assert(t, err, qt.IsNil) typ := bindnode.Unwrap(node) inst, ok := typ.(*IntHolder) qt.Assert(t, ok, qt.IsTrue) qt.Assert(t, *inst, qt.DeepEquals, IntHolder{ Int32: math.MaxInt32, Int64: math.MaxInt64, Uint64: math.MaxUint64, }) node = bindnode.Wrap(inst, schemaType).Representation() byt, err := ipld.Encode(node, dagcbor.Encode) qt.Assert(t, err, qt.IsNil) qt.Assert(t, hex.EncodeToString(byt), qt.Equals, maxExpectedHex) }) t.Run("plain", func(t *testing.T) { type IntHolder uint64 schema := `type IntHolder int` maxExpectedHex := "1bffffffffffffffff" maxExpected, err := hex.DecodeString(maxExpectedHex) qt.Assert(t, err, qt.IsNil) typeSystem, err := ipld.LoadSchemaBytes([]byte(schema)) qt.Assert(t, err, qt.IsNil) schemaType := typeSystem.TypeByName("IntHolder") proto := bindnode.Prototype((*IntHolder)(nil), schemaType) node, err := ipld.DecodeUsingPrototype([]byte(maxExpected), dagcbor.Decode, proto) qt.Assert(t, err, qt.IsNil) typ := bindnode.Unwrap(node) inst, ok := typ.(*IntHolder) qt.Assert(t, ok, qt.IsTrue) qt.Assert(t, *inst, qt.Equals, IntHolder(math.MaxUint64)) node = bindnode.Wrap(inst, schemaType).Representation() byt, err := ipld.Encode(node, dagcbor.Encode) qt.Assert(t, err, qt.IsNil) qt.Assert(t, hex.EncodeToString(byt), qt.Equals, maxExpectedHex) }) } ================================================ FILE: node/bindnode/custom_test.go ================================================ package bindnode_test import ( "bytes" "encoding/hex" "errors" "fmt" "math/big" "strings" "testing" "github.com/google/go-cmp/cmp" "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/codec/dagcbor" "github.com/ipld/go-ipld-prime/codec/dagjson" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/fluent/qp" basicnode "github.com/ipld/go-ipld-prime/node/basic" "github.com/ipld/go-ipld-prime/node/bindnode" "github.com/ipld/go-ipld-prime/schema" "github.com/multiformats/go-multihash" qt "github.com/frankban/quicktest" ) type BoolSubst int var errorDefault = errors.New("something went wrong") const ( BoolSubst_Yes = 100 BoolSubst_No = -100 ) func BoolSubstFromBool(b bool) (interface{}, error) { if b { return BoolSubst_Yes, nil } return BoolSubst_No, nil } func BoolSubstFromBoolError(b bool) (interface{}, error) { return BoolSubst_No, errorDefault } func BoolToBoolSubst(b interface{}) (bool, error) { bp, ok := b.(*BoolSubst) if !ok { return true, fmt.Errorf("expected *BoolSubst value") } switch *bp { case BoolSubst_Yes: return true, nil case BoolSubst_No: return false, nil default: return true, fmt.Errorf("bad BoolSubst") } } func BoolToBoolSubstError(b interface{}) (bool, error) { return false, errorDefault } type IntSubst string func IntSubstFromInt(i int64) (interface{}, error) { if i == 1000 { return "one thousand", nil } else if i == 2000 { return "two thousand", nil } return nil, fmt.Errorf("unexpected value of IntSubst") } func IntSubstFromIntError(i int64) (interface{}, error) { return nil, errorDefault } func IntToIntSubst(i interface{}) (int64, error) { ip, ok := i.(*IntSubst) if !ok { return 0, fmt.Errorf("expected *IntSubst value") } switch *ip { case "one thousand": return 1000, nil case "two thousand": return 2000, nil default: return 0, fmt.Errorf("bad IntSubst") } } func IntToIntSubstError(i interface{}) (int64, error) { return 0, errorDefault } type BigFloat struct{ *big.Float } func BigFloatFromFloat(f float64) (interface{}, error) { bf := big.NewFloat(f) return &BigFloat{bf}, nil } func BigFloatFromFloatError(f float64) (interface{}, error) { return nil, errorDefault } func FloatFromBigFloat(f interface{}) (float64, error) { fp, ok := f.(*BigFloat) if !ok { return 0, fmt.Errorf("expected *BigFloat value") } f64, _ := fp.Float64() return f64, nil } func FloatFromBigFloatError(f interface{}) (float64, error) { return 0, errorDefault } type ByteArray [][]byte func ByteArrayFromString(s string) (interface{}, error) { sa := strings.Split(s, "|") ba := make([][]byte, 0) for _, a := range sa { ba = append(ba, []byte(a)) } return ba, nil } func ByteArrayFromStringError(s string) (interface{}, error) { return nil, errorDefault } func StringFromByteArray(b interface{}) (string, error) { bap, ok := b.(*ByteArray) if !ok { return "", fmt.Errorf("expected *ByteArray value") } sb := strings.Builder{} for i, b := range *bap { sb.WriteString(string(b)) if i != len(*bap)-1 { sb.WriteString("|") } } return sb.String(), nil } func StringFromByteArrayError(b interface{}) (string, error) { return "", errorDefault } // similar to cid/Cid, go-address/Address, go-graphsync/RequestID type Boop struct{ str string } func NewBoop(b []byte) *Boop { return &Boop{string(b)} } func (b Boop) Bytes() []byte { return []byte(b.str) } func (b Boop) String() string { return b.str } // similar to go-state-types/big/Int type Frop struct{ *big.Int } func NewFropFromString(str string) Frop { v, _ := big.NewInt(0).SetString(str, 10) return Frop{v} } func NewFropFromBytes(buf []byte) *Frop { var negative bool switch buf[0] { case 0: negative = false case 1: negative = true default: panic("can't handle this") } i := big.NewInt(0).SetBytes(buf[1:]) if negative { i.Neg(i) } return &Frop{i} } func (b *Frop) Bytes() []byte { switch { case b.Sign() > 0: return append([]byte{0}, b.Int.Bytes()...) case b.Sign() < 0: return append([]byte{1}, b.Int.Bytes()...) default: return []byte{} } } func BoopFromBytes(b []byte) (interface{}, error) { return NewBoop(b), nil } func BoopFromBytesError(b []byte) (interface{}, error) { return nil, errorDefault } func BoopToBytes(iface interface{}) ([]byte, error) { if boop, ok := iface.(*Boop); ok { return boop.Bytes(), nil } return nil, fmt.Errorf("did not get expected type") } func BoopToBytesError(iface interface{}) ([]byte, error) { return nil, errorDefault } func FropFromBytes(b []byte) (interface{}, error) { return NewFropFromBytes(b), nil } func FropFromBytesError(b []byte) (interface{}, error) { return nil, errorDefault } func FropToBytes(iface interface{}) ([]byte, error) { if frop, ok := iface.(*Frop); ok { return frop.Bytes(), nil } return nil, fmt.Errorf("did not get expected type") } func FropToBytesError(iface interface{}) ([]byte, error) { return nil, errorDefault } // Bitcoin's version of "links" is a hex form of the dbl-sha2-256 digest reversed type BtcId string func FromCidToBtcId(c cid.Cid) (interface{}, error) { if c.Prefix().Codec != cid.BitcoinBlock { // should be able to do BitcoinTx too .. but .. return nil, fmt.Errorf("can only convert IDs for BitcoinBlock codecs") } // and multihash must be dbl-sha2-256 dig, err := multihash.Decode(c.Hash()) if err != nil { return nil, err } hid := make([]byte, 0) for i := len(dig.Digest) - 1; i >= 0; i-- { hid = append(hid, dig.Digest[i]) } return BtcId(hex.EncodeToString(hid)), nil } func FromCidToBtcIdError(c cid.Cid) (interface{}, error) { return BtcId(""), errorDefault } func FromBtcIdToCid(iface interface{}) (cid.Cid, error) { bid, ok := iface.(*BtcId) if !ok { return cid.Undef, fmt.Errorf("expected *BtcId value") } dig := make([]byte, 0) hid, err := hex.DecodeString(string(*bid)) if err != nil { return cid.Undef, err } for i := len(hid) - 1; i >= 0; i-- { dig = append(dig, hid[i]) } mh, err := multihash.Encode(dig, multihash.DBL_SHA2_256) if err != nil { return cid.Undef, err } return cid.NewCidV1(cid.BitcoinBlock, mh), nil } func FromBtcIdToCidError(iface interface{}) (cid.Cid, error) { return cid.Undef, errorDefault } type Boom struct { S string St ByteArray B Boop Bo BoolSubst Bptr *Boop F Frop Fl BigFloat I int In IntSubst L BtcId } const boomSchema = ` type ByteArray string type Boop bytes type BoolSubst bool type Frop bytes type BigFloat float type IntSubst int type BtcId &Any type Boom struct { S String St ByteArray B Boop Bo BoolSubst Bptr nullable Boop F Frop Fl BigFloat I Int In IntSubst L BtcId } representation map ` const boomFixtureDagJson = `{"B":{"/":{"bytes":"dGhlc2UgYXJlIGJ5dGVz"}},"Bo":false,"Bptr":{"/":{"bytes":"dGhlc2UgYXJlIHBvaW50ZXIgYnl0ZXM"}},"F":{"/":{"bytes":"AAH3fubjrGlwOMpClAkh/ro13L5Uls4/CtI"}},"Fl":1.12,"I":10101,"In":2000,"L":{"/":"bagyacvra2e6qt2fohajauxceox55t3gedsyqap2phmv7q2qaaaaaaaaaaaaa"},"S":"a string here","St":"a|byte|array"}` var boomFixtureInstance = Boom{ B: *NewBoop([]byte("these are bytes")), Bo: BoolSubst_No, Bptr: NewBoop([]byte("these are pointer bytes")), F: NewFropFromString("12345678901234567891234567890123456789012345678901234567890"), Fl: BigFloat{big.NewFloat(1.12)}, I: 10101, In: IntSubst("two thousand"), S: "a string here", St: ByteArray([][]byte{[]byte("a"), []byte("byte"), []byte("array")}), L: BtcId("00000000000000006af82b3b4f3f00b11cc4ecd9fb75445c0a1238aee8093dd1"), } func TestCustom(t *testing.T) { opts := []bindnode.Option{ bindnode.TypedBytesConverter(&Boop{}, BoopFromBytes, BoopToBytes), bindnode.TypedBytesConverter(&Frop{}, FropFromBytes, FropToBytes), bindnode.TypedBoolConverter(BoolSubst(0), BoolSubstFromBool, BoolToBoolSubst), bindnode.TypedIntConverter(IntSubst(""), IntSubstFromInt, IntToIntSubst), bindnode.TypedFloatConverter(&BigFloat{}, BigFloatFromFloat, FloatFromBigFloat), bindnode.TypedStringConverter(&ByteArray{}, ByteArrayFromString, StringFromByteArray), bindnode.TypedLinkConverter(BtcId(""), FromCidToBtcId, FromBtcIdToCid), } typeSystem, err := ipld.LoadSchemaBytes([]byte(boomSchema)) qt.Assert(t, err, qt.IsNil) schemaType := typeSystem.TypeByName("Boom") proto := bindnode.Prototype(&Boom{}, schemaType, opts...) builder := proto.Representation().NewBuilder() err = dagjson.Decode(builder, bytes.NewReader([]byte(boomFixtureDagJson))) qt.Assert(t, err, qt.IsNil) typ := bindnode.Unwrap(builder.Build()) inst, ok := typ.(*Boom) qt.Assert(t, ok, qt.IsTrue) cmpr := qt.CmpEquals( cmp.Comparer(func(x, y Boop) bool { return x.String() == y.String() }), cmp.Comparer(func(x, y Frop) bool { return x.String() == y.String() }), cmp.Comparer(func(x, y BigFloat) bool { return x.String() == y.String() }), ) qt.Assert(t, *inst, cmpr, boomFixtureInstance) tn := bindnode.Wrap(inst, schemaType, opts...) var buf bytes.Buffer err = dagjson.Encode(tn.Representation(), &buf) qt.Assert(t, err, qt.IsNil) qt.Assert(t, buf.String(), qt.Equals, boomFixtureDagJson) } func TestCustomNamed(t *testing.T) { opts := []bindnode.Option{ bindnode.NamedBytesConverter("Boop", BoopFromBytes, BoopToBytes), bindnode.NamedBytesConverter("Frop", FropFromBytes, FropToBytes), bindnode.NamedBoolConverter("BoolSubst", BoolSubstFromBool, BoolToBoolSubst), bindnode.NamedIntConverter("IntSubst", IntSubstFromInt, IntToIntSubst), bindnode.NamedFloatConverter("BigFloat", BigFloatFromFloat, FloatFromBigFloat), bindnode.NamedStringConverter("ByteArray", ByteArrayFromString, StringFromByteArray), bindnode.NamedLinkConverter("BtcId", FromCidToBtcId, FromBtcIdToCid), // these will error, but shouldn't get called cause the named converters take precedence bindnode.TypedBytesConverter(&Boop{}, BoopFromBytesError, BoopToBytesError), bindnode.TypedBytesConverter(&Frop{}, FropFromBytesError, FropToBytesError), bindnode.TypedBoolConverter(BoolSubst(0), BoolSubstFromBoolError, BoolToBoolSubstError), bindnode.TypedIntConverter(IntSubst(""), IntSubstFromIntError, IntToIntSubstError), bindnode.TypedFloatConverter(&BigFloat{}, BigFloatFromFloatError, FloatFromBigFloatError), bindnode.TypedStringConverter(&ByteArray{}, ByteArrayFromStringError, StringFromByteArrayError), bindnode.TypedLinkConverter(BtcId(""), FromCidToBtcIdError, FromBtcIdToCidError), } typeSystem, err := ipld.LoadSchemaBytes([]byte(boomSchema)) qt.Assert(t, err, qt.IsNil) schemaType := typeSystem.TypeByName("Boom") proto := bindnode.Prototype(&Boom{}, schemaType, opts...) builder := proto.Representation().NewBuilder() err = dagjson.Decode(builder, bytes.NewReader([]byte(boomFixtureDagJson))) qt.Assert(t, err, qt.IsNil) typ := bindnode.Unwrap(builder.Build()) inst, ok := typ.(*Boom) qt.Assert(t, ok, qt.IsTrue) cmpr := qt.CmpEquals( cmp.Comparer(func(x, y Boop) bool { return x.String() == y.String() }), cmp.Comparer(func(x, y Frop) bool { return x.String() == y.String() }), cmp.Comparer(func(x, y BigFloat) bool { return x.String() == y.String() }), ) qt.Assert(t, *inst, cmpr, boomFixtureInstance) tn := bindnode.Wrap(inst, schemaType, opts...) var buf bytes.Buffer err = dagjson.Encode(tn.Representation(), &buf) qt.Assert(t, err, qt.IsNil) qt.Assert(t, buf.String(), qt.Equals, boomFixtureDagJson) } type AnyExtend struct { Name string Blob AnyExtendBlob Count int Null AnyCborEncoded NullPtr *AnyCborEncoded NullableWith *AnyCborEncoded Bool AnyCborEncoded Int AnyCborEncoded Float AnyCborEncoded String AnyCborEncoded Bytes AnyCborEncoded Link AnyCborEncoded Map AnyCborEncoded List AnyCborEncoded BoolPtr *BoolSubst // included to test that a null entry won't call a non-Any converter XListAny []AnyCborEncoded XMapAny anyMap } type anyMap struct { Keys []string Values map[string]*AnyCborEncoded } const anyExtendSchema = ` type AnyExtend struct { Name String Blob Any Count Int Null nullable Any NullPtr nullable Any NullableWith nullable Any Bool Any Int Any Float Any String Any Bytes Any Link Any Map Any List Any BoolPtr nullable Bool XListAny [Any] XMapAny {String:Any} } ` type AnyExtendBlob struct { f string x int64 y int64 z int64 } func AnyExtendBlobFromNode(node datamodel.Node) (interface{}, error) { foo, err := node.LookupByString("foo") if err != nil { return nil, err } fooStr, err := foo.AsString() if err != nil { return nil, err } baz, err := node.LookupByString("baz") if err != nil { return nil, err } x, err := baz.LookupByIndex(0) if err != nil { return nil, err } xi, err := x.AsInt() if err != nil { return nil, err } y, err := baz.LookupByIndex(1) if err != nil { return nil, err } yi, err := y.AsInt() if err != nil { return nil, err } z, err := baz.LookupByIndex(2) if err != nil { return nil, err } zi, err := z.AsInt() if err != nil { return nil, err } return &AnyExtendBlob{f: fooStr, x: xi, y: yi, z: zi}, nil } func (aeb AnyExtendBlob) ToNode() (datamodel.Node, error) { return qp.BuildMap(basicnode.Prototype.Any, -1, func(ma datamodel.MapAssembler) { qp.MapEntry(ma, "foo", qp.String(aeb.f)) qp.MapEntry(ma, "baz", qp.List(-1, func(la datamodel.ListAssembler) { qp.ListEntry(la, qp.Int(aeb.x)) qp.ListEntry(la, qp.Int(aeb.y)) qp.ListEntry(la, qp.Int(aeb.z)) })) }) } func AnyExtendBlobToNode(ptr interface{}) (datamodel.Node, error) { aeb, ok := ptr.(*AnyExtendBlob) if !ok { return nil, fmt.Errorf("expected *AnyExtendBlob type") } return aeb.ToNode() } // take a datamodel.Node, dag-cbor encode it and store it here, do the reverse // to get the datamodel.Node back type AnyCborEncoded struct{ str []byte } func AnyCborEncodedFromNode(node datamodel.Node) (interface{}, error) { if tn, ok := node.(schema.TypedNode); ok { node = tn.Representation() } var buf bytes.Buffer err := dagcbor.Encode(node, &buf) if err != nil { return nil, err } acb := AnyCborEncoded{str: buf.Bytes()} return &acb, nil } func AnyCborEncodedToNode(ptr interface{}) (datamodel.Node, error) { acb, ok := ptr.(*AnyCborEncoded) if !ok { return nil, fmt.Errorf("expected *AnyCborEncoded type") } na := basicnode.Prototype.Any.NewBuilder() err := dagcbor.Decode(na, bytes.NewReader(acb.str)) if err != nil { return nil, err } return na.Build(), nil } const anyExtendDagJson = `{"Blob":{"baz":[2,3,4],"foo":"bar"},"Bool":false,"BoolPtr":null,"Bytes":{"/":{"bytes":"AgMEBQYHCA"}},"Count":101,"Float":2.34,"Int":123456789,"Link":{"/":"bagyacvra2e6qt2fohajauxceox55t3gedsyqap2phmv7q2qaaaaaaaaaaaaa"},"List":[null,"one","two","three",1,2,3,true],"Map":{"foo":"bar","one":1,"three":3,"two":2},"Name":"Any extend test","Null":null,"NullPtr":null,"NullableWith":123456789,"String":"this is a string","XListAny":[1,2,true,null,"bop"],"XMapAny":{"a":1,"b":2,"c":true,"d":null,"e":"bop"}}` var anyExtendFixtureInstance = AnyExtend{ Name: "Any extend test", Count: 101, Blob: AnyExtendBlob{f: "bar", x: 2, y: 3, z: 4}, Null: AnyCborEncoded{mustFromHex("f6")}, // normally these two fields would be `nil`, but we now get to decide whether it should be something concrete NullPtr: &AnyCborEncoded{mustFromHex("f6")}, NullableWith: &AnyCborEncoded{mustFromHex("1a075bcd15")}, Bool: AnyCborEncoded{mustFromHex("f4")}, Int: AnyCborEncoded{mustFromHex("1a075bcd15")}, // 123456789 Float: AnyCborEncoded{mustFromHex("fb4002b851eb851eb8")}, // 2.34 String: AnyCborEncoded{mustFromHex("7074686973206973206120737472696e67")}, // "this is a string" Bytes: AnyCborEncoded{mustFromHex("4702030405060708")}, // [2,3,4,5,6,7,8] Link: AnyCborEncoded{mustFromHex("d82a58260001b0015620d13d09e8ae38120a5c4475fbd9ecc41cb1003f4f3b2bf86a0000000000000000")}, // bagyacvra2e6qt2fohajauxceox55t3gedsyqap2phmv7q2qaaaaaaaaaaaaa Map: AnyCborEncoded{mustFromHex("a463666f6f63626172636f6e65016374776f0265746872656503")}, // {"one":1,"two":2,"three":3,"foo":"bar"} List: AnyCborEncoded{mustFromHex("88f6636f6e656374776f657468726565010203f5")}, // [null,'one','two','three',1,2,3,true] BoolPtr: nil, XListAny: []AnyCborEncoded{{mustFromHex("01")}, {mustFromHex("02")}, {mustFromHex("f5")}, {mustFromHex("f6")}, {mustFromHex("63626f70")}}, // [1,2,true,null,"bop"] XMapAny: anyMap{ Keys: []string{"a", "b", "c", "d", "e"}, Values: map[string]*AnyCborEncoded{ "a": {mustFromHex("01")}, "b": {mustFromHex("02")}, "c": {mustFromHex("f5")}, "d": {mustFromHex("f6")}, "e": {mustFromHex("63626f70")}}}, // {"a":1,"b":2,"c":true,"d":null,"e":"bop"} } func TestCustomAny(t *testing.T) { opts := []bindnode.Option{ bindnode.TypedAnyConverter(&AnyExtendBlob{}, AnyExtendBlobFromNode, AnyExtendBlobToNode), bindnode.TypedAnyConverter(&AnyCborEncoded{}, AnyCborEncodedFromNode, AnyCborEncodedToNode), bindnode.TypedBoolConverter(BoolSubst(0), BoolSubstFromBool, BoolToBoolSubst), } typeSystem, err := ipld.LoadSchemaBytes([]byte(anyExtendSchema)) qt.Assert(t, err, qt.IsNil) schemaType := typeSystem.TypeByName("AnyExtend") proto := bindnode.Prototype(&AnyExtend{}, schemaType, opts...) builder := proto.Representation().NewBuilder() err = dagjson.Decode(builder, bytes.NewReader([]byte(anyExtendDagJson))) qt.Assert(t, err, qt.IsNil) typ := bindnode.Unwrap(builder.Build()) inst, ok := typ.(*AnyExtend) qt.Assert(t, ok, qt.IsTrue) cmpr := qt.CmpEquals( cmp.Comparer(func(x, y AnyExtendBlob) bool { return x.f == y.f && x.x == y.x && x.y == y.y && x.z == y.z }), cmp.Comparer(func(x, y AnyCborEncoded) bool { return bytes.Equal(x.str, y.str) }), ) qt.Assert(t, *inst, cmpr, anyExtendFixtureInstance) tn := bindnode.Wrap(inst, schemaType, opts...) var buf bytes.Buffer err = dagjson.Encode(tn.Representation(), &buf) qt.Assert(t, err, qt.IsNil) qt.Assert(t, buf.String(), qt.Equals, anyExtendDagJson) } func mustFromHex(hexStr string) []byte { byt, err := hex.DecodeString(hexStr) if err != nil { panic(err) } return byt } type ClosedUnion interface { isClosedUnion() } type intUnion uint64 func (intUnion) isClosedUnion() {} type stringUnion string func (stringUnion) isClosedUnion() {} func ClosedUnionFromNode(node datamodel.Node) (interface{}, error) { asInt, err := node.AsInt() if err == nil { return intUnion(asInt), nil } asString, err := node.AsString() if err == nil { return stringUnion(asString), nil } return nil, errors.New("unrecognized type") } func ClosedUnionToNode(val interface{}) (datamodel.Node, error) { cu, ok := val.(*ClosedUnion) if !ok { return nil, errors.New("should be a ClosedUnion") } switch concrete := (*cu).(type) { case intUnion: return basicnode.NewInt(int64(concrete)), nil case stringUnion: return basicnode.NewString(string(concrete)), nil default: return nil, errors.New("unexpected union type") } } type StructWithUnion struct { Cu ClosedUnion } const closedUnionSchema = ` type ClosedUnion any type StructWithUnion struct { cu ClosedUnion } ` const closedUnionFixtureIntDagJson = `{"cu":8}` const closedUnionFixtureStringDagJson = `{"cu":"happy"}` var closedUnionIntInst = StructWithUnion{ Cu: intUnion(8), } var closedUnionStringInst = StructWithUnion{ Cu: stringUnion("happy"), } func TestCustomAnyWithInterface(t *testing.T) { opts := []bindnode.Option{ bindnode.NamedAnyConverter("ClosedUnion", ClosedUnionFromNode, ClosedUnionToNode), } typeSystem, err := ipld.LoadSchemaBytes([]byte(closedUnionSchema)) qt.Assert(t, err, qt.IsNil) schemaType := typeSystem.TypeByName("StructWithUnion") proto := bindnode.Prototype(&StructWithUnion{}, schemaType, opts...) // test one union variant builder := proto.Representation().NewBuilder() err = dagjson.Decode(builder, bytes.NewReader([]byte(closedUnionFixtureIntDagJson))) qt.Assert(t, err, qt.IsNil) typ := bindnode.Unwrap(builder.Build()) inst, ok := typ.(*StructWithUnion) qt.Assert(t, ok, qt.IsTrue) cmpr := qt.CmpEquals() qt.Assert(t, *inst, cmpr, closedUnionIntInst) tn := bindnode.Wrap(inst, schemaType, opts...) var buf bytes.Buffer err = dagjson.Encode(tn.Representation(), &buf) qt.Assert(t, err, qt.IsNil) qt.Assert(t, buf.String(), qt.Equals, closedUnionFixtureIntDagJson) // test other union variant builder = proto.Representation().NewBuilder() err = dagjson.Decode(builder, bytes.NewReader([]byte(closedUnionFixtureStringDagJson))) qt.Assert(t, err, qt.IsNil) typ = bindnode.Unwrap(builder.Build()) inst, ok = typ.(*StructWithUnion) qt.Assert(t, ok, qt.IsTrue) cmpr = qt.CmpEquals() qt.Assert(t, *inst, cmpr, closedUnionStringInst) tn = bindnode.Wrap(inst, schemaType, opts...) buf = bytes.Buffer{} err = dagjson.Encode(tn.Representation(), &buf) qt.Assert(t, err, qt.IsNil) qt.Assert(t, buf.String(), qt.Equals, closedUnionFixtureStringDagJson) } ================================================ FILE: node/bindnode/example_test.go ================================================ package bindnode_test import ( "fmt" "os" "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/codec/dagjson" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/fluent/qp" "github.com/ipld/go-ipld-prime/node/bindnode" "github.com/ipld/go-ipld-prime/schema" ) func ExampleWrap_withSchema() { ts, err := ipld.LoadSchemaBytes([]byte(` type Person struct { Name String Age optional Int Friends optional [String] } `)) if err != nil { panic(err) } schemaType := ts.TypeByName("Person") type Person struct { Name string Age *int64 // optional Friends []string // optional; no need for a pointer as slices are nilable } person := &Person{ Name: "Michael", Friends: []string{"Sarah", "Alex"}, } node := bindnode.Wrap(person, schemaType) nodeRepr := node.Representation() dagjson.Encode(nodeRepr, os.Stdout) // Output: // {"Friends":["Sarah","Alex"],"Name":"Michael"} } func ExampleWrap_noSchema() { type Person struct { Name string Age int64 // TODO: optional to match other examples Friends []string } person := &Person{ Name: "Michael", Friends: []string{"Sarah", "Alex"}, } node := bindnode.Wrap(person, nil) nodeRepr := node.Representation() dagjson.Encode(nodeRepr, os.Stdout) // Output: // {"Age":0,"Friends":["Sarah","Alex"],"Name":"Michael"} } func ExamplePrototype_onlySchema() { ts, err := ipld.LoadSchemaBytes([]byte(` type Person struct { Name String Age optional Int Friends [String] } `)) if err != nil { panic(err) } schemaType := ts.TypeByName("Person") proto := bindnode.Prototype(nil, schemaType) node, err := qp.BuildMap(proto, -1, func(ma datamodel.MapAssembler) { qp.MapEntry(ma, "Name", qp.String("Michael")) qp.MapEntry(ma, "Friends", qp.List(-1, func(la datamodel.ListAssembler) { qp.ListEntry(la, qp.String("Sarah")) qp.ListEntry(la, qp.String("Alex")) })) }) if err != nil { panic(err) } nodeRepr := node.(schema.TypedNode).Representation() dagjson.Encode(nodeRepr, os.Stdout) // Output: // {"Friends":["Sarah","Alex"],"Name":"Michael"} } func ExamplePrototype_union() { ts, err := ipld.LoadSchemaBytes([]byte(` type StringOrInt union { | String "hasString" | Int "hasInt" } representation keyed `)) if err != nil { panic(err) } schemaType := ts.TypeByName("StringOrInt") type CustomIntType int64 type StringOrInt struct { String *string Int *CustomIntType // We can use custom types, too. } proto := bindnode.Prototype((*StringOrInt)(nil), schemaType) node, err := qp.BuildMap(proto.Representation(), -1, func(ma datamodel.MapAssembler) { qp.MapEntry(ma, "hasInt", qp.Int(123)) }) if err != nil { panic(err) } fmt.Print("Type level DAG-JSON: ") dagjson.Encode(node, os.Stdout) fmt.Println() fmt.Print("Representation level DAG-JSON: ") nodeRepr := node.(schema.TypedNode).Representation() dagjson.Encode(nodeRepr, os.Stdout) fmt.Println() // Inspect what the underlying Go value contains. union := bindnode.Unwrap(node).(*StringOrInt) switch { case union.String != nil: fmt.Printf("Go StringOrInt.String: %v\n", *union.String) case union.Int != nil: fmt.Printf("Go StringOrInt.Int: %v\n", *union.Int) } // Output: // Type level DAG-JSON: {"Int":123} // Representation level DAG-JSON: {"hasInt":123} // Go StringOrInt.Int: 123 } ================================================ FILE: node/bindnode/fuzz_test.go ================================================ //go:build go1.18 package bindnode_test import ( "bytes" "fmt" "reflect" "strings" "testing" "github.com/ipld/go-ipld-prime/codec/dagcbor" "github.com/ipld/go-ipld-prime/codec/dagjson" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/node/bindnode" "github.com/ipld/go-ipld-prime/schema" schemadmt "github.com/ipld/go-ipld-prime/schema/dmt" schemadsl "github.com/ipld/go-ipld-prime/schema/dsl" ) var fuzzInputs = []struct { schemaDSL, nodeDagJSON string }{ { schemaDSL: `type Root bool`, nodeDagJSON: `true`, }, { schemaDSL: `type Root int`, nodeDagJSON: `123`, }, { schemaDSL: `type Root float`, nodeDagJSON: `45.67`, }, { schemaDSL: `type Root string`, nodeDagJSON: `"foo"`, }, { schemaDSL: `type Root bytes`, nodeDagJSON: `{"/":{"bytes":"ZGVhZGJlZWY"}}`, }, { schemaDSL: `type Root [Int]`, nodeDagJSON: `[3,2,1]`, }, { schemaDSL: `type Root [String]`, nodeDagJSON: `["x","y","z"]`, }, { schemaDSL: `type Root {String:Int}`, nodeDagJSON: `{"a":20,"b":10}`, }, { schemaDSL: `type Root {String:Float}`, nodeDagJSON: `{"a":20.5,"b":10.2}`, }, { schemaDSL: `type Root struct { F1 Bool F2 Bytes }`, nodeDagJSON: `{"F1":true,"F2":{"/":{"bytes":"ZGVhZGJlZWY"}}}`, }, { schemaDSL: `type Root struct { F1 Int F2 Float } representation tuple`, nodeDagJSON: `[23,45.67]`, }, { schemaDSL: `type Root enum { | aa ("a") | bb ("b") } representation string`, nodeDagJSON: `"b"`, }, { schemaDSL: `type Root enum { | One ("1") | Two ("2") } representation int`, nodeDagJSON: `2`, }, { schemaDSL: `type Root union { | Int "x" | String "y" } representation keyed`, nodeDagJSON: `{"y":"foo"}`, }, { schemaDSL: `type Root union { | Float float | Bytes bytes | Bool bool | Nested map } representation kinded type Nested struct { F1 Int } `, nodeDagJSON: `true`, }, } func marshalDagCBOR(tb testing.TB, node datamodel.Node) []byte { tb.Helper() var buf bytes.Buffer if err := dagcbor.Encode(node, &buf); err != nil { tb.Fatal(err) } return buf.Bytes() } func marshalDagJSON(tb testing.TB, node datamodel.Node) []byte { tb.Helper() var buf bytes.Buffer if err := dagjson.Encode(node, &buf); err != nil { switch s := err.Error(); { case strings.Contains(s, "unsupported value: NaN"), strings.Contains(s, "unsupported value: -Inf"), strings.Contains(s, "unsupported value: +Inf"): tb.Skipf("dagcbor does not support NaN/Inf") } tb.Fatal(err) } return buf.Bytes() } // TODO: consider allowing any codec multicode instead of hard-coding dagcbor // TODO: we always infer the Go type; it would be interesting to also support // inferring the IPLD schema, or to supply both. // TODO: encoding roundtrips via codecs are a good way to exercise bindnode's // Node implementation, but they do not call all the methods on the Node // interface. Consider other ways to call the rest of the methods, akin to how // infer_test.go has useNodeAsKind. func FuzzBindnodeViaDagCBOR(f *testing.F) { for _, input := range fuzzInputs { // f.Logf("debug: %#v\n", input) schemaDMT, err := schemadsl.ParseBytes([]byte(input.schemaDSL)) if err != nil { f.Fatal(err) } schemaNode := bindnode.Wrap(schemaDMT, schemadmt.Prototypes.Schema.Type()) schemaDagCBOR := marshalDagCBOR(f, schemaNode.Representation()) nodeBuilder := basicnode.Prototype.Any.NewBuilder() if err := dagjson.Decode(nodeBuilder, strings.NewReader(input.nodeDagJSON)); err != nil { f.Fatal(err) } node := nodeBuilder.Build() nodeDagCBOR := marshalDagCBOR(f, node) f.Add(schemaDagCBOR, nodeDagCBOR) // Verify that nodeDagCBOR actually fits the schema. // Otherwise, if any of our fuzz inputs are wrong, we might not notice. { schemaDMT := bindnode.Unwrap(schemaNode).(*schemadmt.Schema) ts := new(schema.TypeSystem) ts.Init() if err := schemadmt.Compile(ts, schemaDMT); err != nil { f.Fatal(err) } schemaType := ts.TypeByName("Root") proto := bindnode.Prototype(nil, schemaType) nodeBuilder := proto.Representation().NewBuilder() if err := dagcbor.Decode(nodeBuilder, bytes.NewReader(nodeDagCBOR)); err != nil { f.Fatal(err) } } } f.Fuzz(func(t *testing.T, schemaDagCBOR, nodeDagCBOR []byte) { schemaBuilder := schemadmt.Prototypes.Schema.Representation().NewBuilder() if err := dagcbor.Decode(schemaBuilder, bytes.NewReader(schemaDagCBOR)); err != nil { t.Skipf("invalid schema-schema dag-cbor: %v", err) } schemaNode := schemaBuilder.Build().(schema.TypedNode) schemaDMT := bindnode.Unwrap(schemaNode).(*schemadmt.Schema) // Log the input schema and node we're fuzzing with, to help debugging. // We also use dag-json, as it's more human readable. t.Logf("schema in dag-cbor: %X", schemaDagCBOR) t.Logf("node in dag-cbor: %X", nodeDagCBOR) t.Logf("schema in dag-json: %s", marshalDagJSON(t, schemaNode.Representation())) { nodeBuilder := basicnode.Prototype.Any.NewBuilder() if err := dagcbor.Decode(nodeBuilder, bytes.NewReader(nodeDagCBOR)); err != nil { // If some dag-cbor bytes don't decode into the Any prototype, // then they're just not valid dag-cbor at all. t.Skipf("invalid node dag-cbor: %v", err) } node := nodeBuilder.Build() t.Logf("node in dag-json: %s", marshalDagJSON(t, node)) } // Is nodeDagCBOR canonically encoded, i.e. strictly deterministic as // per the DAG-CBOR spec? This matters for the re-encode checks below. // Note that we want to use the non-strict decoder for fuzzing, // as that default is what the vast majority users will use. canonicalNodeDagCBOR := true canonicalDecoder := dagcbor.DecodeOptions{AllowLinks: true, ExperimentalDeterminism: true} if err := canonicalDecoder.Decode(basicnode.Prototype.Any.NewBuilder(), bytes.NewReader(nodeDagCBOR)); err != nil { canonicalNodeDagCBOR = false t.Logf("note that this node dag-cbor isn't canonical!") } ts := new(schema.TypeSystem) ts.Init() // For the time being, we're not interested in panics from // schemadmt.Compile or schema.TypeSystem. They are relatively prone to // panics at the moment, and right now we're mainly interested in bugs // in bindnode and dagcbor. func() { defer func() { if r := recover(); r != nil { t.Skipf("invalid schema: %v", r) } }() if err := schemadmt.Compile(ts, schemaDMT); err != nil { t.Skipf("invalid schema: %v", err) } }() schemaType := ts.TypeByName("Root") if schemaType == nil { t.Skipf("schema has no Root type") } var proto schema.TypedPrototype func() { defer func() { if r := recover(); r != nil { str := fmt.Sprint(r) switch { case strings.Contains(str, "bindnode: unexpected nil schema.Type"): case strings.Contains(str, "is not a valid Go identifier"): case strings.Contains(str, "bindnode: inferring Go types from cyclic schemas is not supported"): default: panic(r) } t.Skipf("invalid schema: %v", r) } }() proto = bindnode.Prototype(nil, schemaType) }() for _, repr := range []bool{false, true} { t.Logf("decode and encode roundtrip with dag-cbor repr=%v", repr) var nodeBuilder datamodel.NodeBuilder if !repr { nodeBuilder = proto.NewBuilder() } else { nodeBuilder = proto.Representation().NewBuilder() } if err := dagcbor.Decode(nodeBuilder, bytes.NewReader(nodeDagCBOR)); err != nil { // The dag-cbor isn't valid for this node. Nothing else to do. // We don't use t.Skip, because a dag-cbor might only be valid // at the repr level, but not at the type level. continue } node := nodeBuilder.Build() if repr { node = node.(schema.TypedNode).Representation() } // Unwrap returns a pointer, and %#v prints pointers as hex, // so to get useful output, use reflect to dereference them. t.Logf("decode successful: %#v", reflect.ValueOf(bindnode.Unwrap(node)).Elem().Interface()) reenc := marshalDagCBOR(t, node) switch { case canonicalNodeDagCBOR && !bytes.Equal(reenc, nodeDagCBOR): t.Errorf("node reencoded as %X rather than %X", reenc, nodeDagCBOR) case !canonicalNodeDagCBOR && bytes.Equal(reenc, nodeDagCBOR): t.Errorf("node reencoded as %X even though it's not canonical", reenc) default: t.Logf("re-encode successful: %X", reenc) } } }) } ================================================ FILE: node/bindnode/generate.go ================================================ package bindnode import ( "bytes" "fmt" "go/format" "io" "strings" "github.com/ipld/go-ipld-prime/schema" ) // TODO(mvdan): deduplicate with inferGoType once reflect supports creating named types func produceGoType(goTypes map[string]string, typ schema.Type) (name, src string) { if typ, ok := typ.(interface{ IsAnonymous() bool }); ok { if typ.IsAnonymous() { panic("TODO: does this ever happen?") } } name = string(typ.Name()) switch typ.(type) { case *schema.TypeBool: return goTypeBool.String(), "" case *schema.TypeInt: return goTypeInt.String(), "" case *schema.TypeFloat: return goTypeFloat.String(), "" case *schema.TypeString: return goTypeString.String(), "" case *schema.TypeBytes: return goTypeBytes.String(), "" case *schema.TypeLink: return goTypeLink.String(), "" // datamodel.Link case *schema.TypeAny: return goTypeNode.String(), "" // datamodel.Node } // Results are cached in goTypes. if src := goTypes[name]; src != "" { return name, src } src = produceGoTypeInner(goTypes, name, typ) goTypes[name] = src return name, src } func produceGoTypeInner(goTypes map[string]string, name string, typ schema.Type) (src string) { // Avoid infinite cycles. // produceGoType will fill in the final type later. goTypes[name] = "WIP" switch typ := typ.(type) { case *schema.TypeEnum: // TODO: also generate named constants for the members. return goTypeString.String() case *schema.TypeStruct: var b strings.Builder fmt.Fprintf(&b, "struct {\n") fields := typ.Fields() for _, field := range fields { fmt.Fprintf(&b, "%s ", fieldNameFromSchema(field.Name())) ftypGo, _ := produceGoType(goTypes, field.Type()) if field.IsNullable() { fmt.Fprintf(&b, "*") } if field.IsOptional() { fmt.Fprintf(&b, "*") } fmt.Fprintf(&b, "%s\n", ftypGo) } fmt.Fprintf(&b, "\n}") return b.String() case *schema.TypeMap: ktyp, _ := produceGoType(goTypes, typ.KeyType()) vtyp, _ := produceGoType(goTypes, typ.ValueType()) if typ.ValueIsNullable() { vtyp = "*" + vtyp } return fmt.Sprintf(`struct { Keys []%s Values map[%s]%s }`, ktyp, ktyp, vtyp) case *schema.TypeList: etyp, _ := produceGoType(goTypes, typ.ValueType()) if typ.ValueIsNullable() { etyp = "*" + etyp } return fmt.Sprintf("[]%s", etyp) case *schema.TypeUnion: var b strings.Builder fmt.Fprintf(&b, "struct{\n") members := typ.Members() for _, ftyp := range members { ftypGo, _ := produceGoType(goTypes, ftyp) fmt.Fprintf(&b, "%s ", fieldNameFromSchema(string(ftyp.Name()))) fmt.Fprintf(&b, "*%s\n", ftypGo) } fmt.Fprintf(&b, "\n}") return b.String() } panic(fmt.Sprintf("%T\n", typ)) } // ProduceGoTypes infers Go types from an IPLD schema in ts // and writes their Go source code type declarations to w. // Note that just the types are written, // without a package declaration nor any imports. // // This gives a good starting point when wanting to use bindnode with Go types, // but users will generally want to own and modify the types afterward, // so they can add documentation or tweak the types as needed. func ProduceGoTypes(w io.Writer, ts *schema.TypeSystem) error { goTypes := make(map[string]string) var buf bytes.Buffer for _, name := range ts.Names() { schemaType := ts.TypeByName(string(name)) if name != schemaType.Name() { panic(fmt.Sprintf("%s vs %s", name, schemaType.Name())) } _, src := produceGoType(goTypes, schemaType) if src == "" { continue // scalar type used directly } fmt.Fprintf(&buf, "type %s %s\n", name, src) } src, err := format.Source(buf.Bytes()) if err != nil { return err } if _, err := w.Write(src); err != nil { return err } return nil } ================================================ FILE: node/bindnode/infer.go ================================================ package bindnode import ( "fmt" "go/token" "reflect" "strings" "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime/datamodel" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipld/go-ipld-prime/schema" ) var ( goTypeBool = reflect.TypeOf(false) goTypeInt = reflect.TypeOf(int(0)) goTypeFloat = reflect.TypeOf(0.0) goTypeString = reflect.TypeOf("") goTypeBytes = reflect.TypeOf([]byte{}) goTypeLink = reflect.TypeOf((*datamodel.Link)(nil)).Elem() goTypeNode = reflect.TypeOf((*datamodel.Node)(nil)).Elem() goTypeCidLink = reflect.TypeOf((*cidlink.Link)(nil)).Elem() goTypeCid = reflect.TypeOf((*cid.Cid)(nil)).Elem() schemaTypeBool = schema.SpawnBool("Bool") schemaTypeInt = schema.SpawnInt("Int") schemaTypeFloat = schema.SpawnFloat("Float") schemaTypeString = schema.SpawnString("String") schemaTypeBytes = schema.SpawnBytes("Bytes") schemaTypeLink = schema.SpawnLink("Link") schemaTypeAny = schema.SpawnAny("Any") ) // Consider exposing these APIs later, if they might be useful. type seenEntry struct { goType reflect.Type schemaType schema.Type } // verifyCompatibility is the primary way we check that the schema type(s) // matches the Go type(s); so we do this before we can proceed operating on it. // verifyCompatibility doesn't return an error, it panics—the errors here are // not runtime errors, they're programmer errors because your schema doesn't // match your Go type func verifyCompatibility(cfg *config, seen map[seenEntry]bool, goType reflect.Type, schemaType schema.Type) { // TODO(mvdan): support **T as well? if goType.Kind() == reflect.Ptr { goType = goType.Elem() } // Avoid endless loops. // // TODO(mvdan): this is easy but fairly allocation-happy. // Plus, one map per call means we don't reuse work. if seen[seenEntry{goType, schemaType}] { return } seen[seenEntry{goType, schemaType}] = true doPanic := func(format string, args ...interface{}) { panicFormat := "bindnode: schema type %s is not compatible with Go type %s" panicArgs := []interface{}{schemaType.Name(), goType.String()} if format != "" { panicFormat += ": " + format } panicArgs = append(panicArgs, args...) panic(fmt.Sprintf(panicFormat, panicArgs...)) } switch schemaType := schemaType.(type) { case *schema.TypeBool: if customConverter := cfg.converterForType(schemaType.Name(), goType); customConverter != nil { if customConverter.kind != schema.TypeKind_Bool { doPanic("kind mismatch; custom converter for type is not for Bool") } } else if goType.Kind() != reflect.Bool { doPanic("kind mismatch; need boolean") } case *schema.TypeInt: if customConverter := cfg.converterForType(schemaType.Name(), goType); customConverter != nil { if customConverter.kind != schema.TypeKind_Int { doPanic("kind mismatch; custom converter for type is not for Int") } } else if kind := goType.Kind(); !kindInt[kind] && !kindUint[kind] { doPanic("kind mismatch; need integer") } case *schema.TypeFloat: if customConverter := cfg.converterForType(schemaType.Name(), goType); customConverter != nil { if customConverter.kind != schema.TypeKind_Float { doPanic("kind mismatch; custom converter for type is not for Float") } } else { switch goType.Kind() { case reflect.Float32, reflect.Float64: default: doPanic("kind mismatch; need float") } } case *schema.TypeString: // TODO: allow []byte? if customConverter := cfg.converterForType(schemaType.Name(), goType); customConverter != nil { if customConverter.kind != schema.TypeKind_String { doPanic("kind mismatch; custom converter for type is not for String") } } else if goType.Kind() != reflect.String { doPanic("kind mismatch; need string") } case *schema.TypeBytes: // TODO: allow string? if customConverter := cfg.converterForType(schemaType.Name(), goType); customConverter != nil { if customConverter.kind != schema.TypeKind_Bytes { doPanic("kind mismatch; custom converter for type is not for Bytes") } } else if goType.Kind() != reflect.Slice { doPanic("kind mismatch; need slice of bytes") } else if goType.Elem().Kind() != reflect.Uint8 { doPanic("kind mismatch; need slice of bytes") } case *schema.TypeEnum: if _, ok := schemaType.RepresentationStrategy().(schema.EnumRepresentation_Int); ok { if kind := goType.Kind(); kind != reflect.String && !kindInt[kind] && !kindUint[kind] { doPanic("kind mismatch; need string or integer") } } else { if goType.Kind() != reflect.String { doPanic("kind mismatch; need string") } } case *schema.TypeList: if goType.Kind() != reflect.Slice { doPanic("kind mismatch; need slice") } goType = goType.Elem() if schemaType.ValueIsNullable() { if ptr, nilable := ptrOrNilable(goType.Kind()); !nilable { doPanic("nullable types must be nilable") } else if ptr { goType = goType.Elem() } } verifyCompatibility(cfg, seen, goType, schemaType.ValueType()) case *schema.TypeMap: // struct { // Keys []K // Values map[K]V // } if goType.Kind() != reflect.Struct { doPanic("kind mismatch; need struct{Keys []K; Values map[K]V}") } if goType.NumField() != 2 { doPanic("%d vs 2 fields", goType.NumField()) } fieldKeys := goType.Field(0) if fieldKeys.Type.Kind() != reflect.Slice { doPanic("kind mismatch; need struct{Keys []K; Values map[K]V}") } verifyCompatibility(cfg, seen, fieldKeys.Type.Elem(), schemaType.KeyType()) fieldValues := goType.Field(1) if fieldValues.Type.Kind() != reflect.Map { doPanic("kind mismatch; need struct{Keys []K; Values map[K]V}") } keyType := fieldValues.Type.Key() verifyCompatibility(cfg, seen, keyType, schemaType.KeyType()) elemType := fieldValues.Type.Elem() if schemaType.ValueIsNullable() { if ptr, nilable := ptrOrNilable(elemType.Kind()); !nilable { doPanic("nullable types must be nilable") } else if ptr { elemType = elemType.Elem() } } verifyCompatibility(cfg, seen, elemType, schemaType.ValueType()) case *schema.TypeStruct: if goType.Kind() != reflect.Struct { doPanic("kind mismatch; need struct") } schemaFields := schemaType.Fields() if goType.NumField() != len(schemaFields) { doPanic("%d vs %d fields", goType.NumField(), len(schemaFields)) } for i, schemaField := range schemaFields { schemaType := schemaField.Type() goType := goType.Field(i).Type switch { case schemaField.IsOptional() && schemaField.IsNullable(): // TODO: https://github.com/ipld/go-ipld-prime/issues/340 will // help here, to avoid the double pointer. We can't use nilable // but non-pointer types because that's just one "nil" state. // TODO: deal with custom converters in this case if goType.Kind() != reflect.Ptr { doPanic("optional and nullable fields must use double pointers (**)") } goType = goType.Elem() if goType.Kind() != reflect.Ptr { doPanic("optional and nullable fields must use double pointers (**)") } goType = goType.Elem() case schemaField.IsOptional(): if ptr, nilable := ptrOrNilable(goType.Kind()); !nilable { doPanic("optional fields must be nilable") } else if ptr { goType = goType.Elem() } case schemaField.IsNullable(): if ptr, nilable := ptrOrNilable(goType.Kind()); !nilable { if customConverter := cfg.converterForType(schemaType.Name(), goType); customConverter == nil { doPanic("nullable fields must be nilable") } } else if ptr { goType = goType.Elem() } } verifyCompatibility(cfg, seen, goType, schemaType) } case *schema.TypeUnion: if goType.Kind() != reflect.Struct { doPanic("kind mismatch; need struct for an union") } schemaMembers := schemaType.Members() if goType.NumField() != len(schemaMembers) { doPanic("%d vs %d members", goType.NumField(), len(schemaMembers)) } for i, schemaType := range schemaMembers { goType := goType.Field(i).Type if ptr, nilable := ptrOrNilable(goType.Kind()); !nilable { doPanic("union members must be nilable") } else if ptr { goType = goType.Elem() } verifyCompatibility(cfg, seen, goType, schemaType) } case *schema.TypeLink: if customConverter := cfg.converterForType(schemaType.Name(), goType); customConverter != nil { if customConverter.kind != schema.TypeKind_Link { doPanic("kind mismatch; custom converter for type is not for Link") } } else if goType != goTypeLink && goType != goTypeCidLink && goType != goTypeCid { doPanic("links in Go must be datamodel.Link, cidlink.Link, or cid.Cid") } case *schema.TypeAny: if customConverter := cfg.converterForType(schemaType.Name(), goType); customConverter != nil { if customConverter.kind != schema.TypeKind_Any { doPanic("kind mismatch; custom converter for type is not for Any") } } else if goType != goTypeNode { doPanic("Any in Go must be datamodel.Node") } default: panic(fmt.Sprintf("%T", schemaType)) } } func ptrOrNilable(kind reflect.Kind) (ptr, nilable bool) { switch kind { case reflect.Ptr: return true, true case reflect.Interface, reflect.Map, reflect.Slice: return false, true default: return false, false } } // If we recurse past a large number of levels, we're mostly stuck in a loop. // Prevent burning CPU or causing OOM crashes. // If a user really wrote an IPLD schema or Go type with such deep nesting, // it's likely they are trying to abuse the system as well. const maxRecursionLevel = 1 << 10 type inferredStatus int const ( _ inferredStatus = iota inferringInProcess inferringDone ) // inferGoType can build a Go type given a schema func inferGoType(typ schema.Type, status map[schema.TypeName]inferredStatus, level int) reflect.Type { if level > maxRecursionLevel { panic(fmt.Sprintf("inferGoType: refusing to recurse past %d levels", maxRecursionLevel)) } name := typ.Name() if status[name] == inferringInProcess { panic("bindnode: inferring Go types from cyclic schemas is not supported since Go reflection does not support creating named types") } status[name] = inferringInProcess defer func() { status[name] = inferringDone }() switch typ := typ.(type) { case *schema.TypeBool: return goTypeBool case *schema.TypeInt: return goTypeInt case *schema.TypeFloat: return goTypeFloat case *schema.TypeString: return goTypeString case *schema.TypeBytes: return goTypeBytes case *schema.TypeStruct: fields := typ.Fields() fieldsGo := make([]reflect.StructField, len(fields)) for i, field := range fields { ftypGo := inferGoType(field.Type(), status, level+1) if field.IsNullable() { ftypGo = reflect.PointerTo(ftypGo) } if field.IsOptional() { ftypGo = reflect.PointerTo(ftypGo) } fieldsGo[i] = reflect.StructField{ Name: fieldNameFromSchema(field.Name()), Type: ftypGo, } } return reflect.StructOf(fieldsGo) case *schema.TypeMap: ktyp := inferGoType(typ.KeyType(), status, level+1) vtyp := inferGoType(typ.ValueType(), status, level+1) if typ.ValueIsNullable() { vtyp = reflect.PointerTo(vtyp) } // We need an extra field to keep the map ordered, // since IPLD maps must have stable iteration order. // We could sort when iterating, but that's expensive. // Keeping the insertion order is easy and intuitive. // // struct { // Keys []K // Values map[K]V // } fieldsGo := []reflect.StructField{ { Name: "Keys", Type: reflect.SliceOf(ktyp), }, { Name: "Values", Type: reflect.MapOf(ktyp, vtyp), }, } return reflect.StructOf(fieldsGo) case *schema.TypeList: etyp := inferGoType(typ.ValueType(), status, level+1) if typ.ValueIsNullable() { etyp = reflect.PointerTo(etyp) } return reflect.SliceOf(etyp) case *schema.TypeUnion: // type goUnion struct { // Type1 *Type1 // Type2 *Type2 // ... // } members := typ.Members() fieldsGo := make([]reflect.StructField, len(members)) for i, ftyp := range members { ftypGo := inferGoType(ftyp, status, level+1) fieldsGo[i] = reflect.StructField{ Name: fieldNameFromSchema(ftyp.Name()), Type: reflect.PointerTo(ftypGo), } } return reflect.StructOf(fieldsGo) case *schema.TypeLink: return goTypeLink case *schema.TypeEnum: // TODO: generate int for int reprs by default? return goTypeString case *schema.TypeAny: return goTypeNode case nil: panic("bindnode: unexpected nil schema.Type") } panic(fmt.Sprintf("%T", typ)) } // from IPLD Schema field names like "foo" to Go field names like "Foo". func fieldNameFromSchema(name string) string { fieldName := strings.Title(name) //lint:ignore SA1019 cases.Title doesn't work for this if !token.IsIdentifier(fieldName) { panic(fmt.Sprintf("bindnode: inferred field name %q is not a valid Go identifier", fieldName)) } return fieldName } var defaultTypeSystem schema.TypeSystem func init() { defaultTypeSystem.Init() defaultTypeSystem.Accumulate(schemaTypeBool) defaultTypeSystem.Accumulate(schemaTypeInt) defaultTypeSystem.Accumulate(schemaTypeFloat) defaultTypeSystem.Accumulate(schemaTypeString) defaultTypeSystem.Accumulate(schemaTypeBytes) defaultTypeSystem.Accumulate(schemaTypeLink) defaultTypeSystem.Accumulate(schemaTypeAny) } // TODO: support IPLD maps and unions in inferSchema // TODO: support bringing your own TypeSystem? // TODO: we should probably avoid re-spawning the same types if the TypeSystem // has them, and test that that works as expected // inferSchema can build a schema from a Go type func inferSchema(typ reflect.Type, level int) schema.Type { if level > maxRecursionLevel { panic(fmt.Sprintf("inferSchema: refusing to recurse past %d levels", maxRecursionLevel)) } switch typ.Kind() { case reflect.Bool: return schemaTypeBool case reflect.Int64: return schemaTypeInt case reflect.Float64: return schemaTypeFloat case reflect.String: return schemaTypeString case reflect.Struct: // these types must match exactly since we need symmetry of being able to // get the values an also assign values to them if typ == goTypeCid || typ == goTypeCidLink { return schemaTypeLink } fieldsSchema := make([]schema.StructField, typ.NumField()) for i := range fieldsSchema { field := typ.Field(i) ftyp := field.Type ftypSchema := inferSchema(ftyp, level+1) fieldsSchema[i] = schema.SpawnStructField( field.Name, // TODO: allow configuring the name with tags ftypSchema.Name(), // TODO: support nullable/optional with tags false, false, ) } name := typ.Name() if name == "" { panic("TODO: anonymous composite types") } typSchema := schema.SpawnStruct(name, fieldsSchema, nil) defaultTypeSystem.Accumulate(typSchema) return typSchema case reflect.Slice: if typ.Elem().Kind() == reflect.Uint8 { // Special case for []byte. return schemaTypeBytes } nullable := false if typ.Elem().Kind() == reflect.Ptr { nullable = true } etypSchema := inferSchema(typ.Elem(), level+1) name := typ.Name() if name == "" { name = "List_" + etypSchema.Name() } typSchema := schema.SpawnList(name, etypSchema.Name(), nullable) defaultTypeSystem.Accumulate(typSchema) return typSchema case reflect.Interface: // these types must match exactly since we need symmetry of being able to // get the values an also assign values to them if typ == goTypeLink { return schemaTypeLink } if typ == goTypeNode { return schemaTypeAny } panic("bindnode: unable to infer from interface") } panic(fmt.Sprintf("bindnode: unable to infer from type %s", typ.Kind().String())) } // There are currently 27 reflect.Kind iota values, // so 32 should be plenty to ensure we don't panic in practice. var kindInt = [32]bool{ reflect.Int: true, reflect.Int8: true, reflect.Int16: true, reflect.Int32: true, reflect.Int64: true, } var kindUint = [32]bool{ reflect.Uint: true, reflect.Uint8: true, reflect.Uint16: true, reflect.Uint32: true, reflect.Uint64: true, } ================================================ FILE: node/bindnode/infer_test.go ================================================ package bindnode_test import ( "bytes" "encoding/json" "fmt" "html/template" "os" "os/exec" "path/filepath" "reflect" "runtime/debug" "strings" "testing" qt "github.com/frankban/quicktest" "github.com/google/go-cmp/cmp" "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/codec/dagjson" "github.com/ipld/go-ipld-prime/datamodel" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/node/bindnode" "github.com/ipld/go-ipld-prime/schema" ) type anyScalar struct { Bool *bool Int *int64 Float *float64 String *string Bytes *[]byte Link *datamodel.Link } type anyRecursive struct { List *[]string Map *struct { Keys []string Values map[string]string } } var prototypeTests = []struct { name string schemaSrc string ptrType interface{} // Prettified for maintainability, valid DAG-JSON when compacted. prettyDagJSON string }{ { name: "Scalars", schemaSrc: `type Root struct { bool Bool int Int uint Int float Float string String bytes Bytes }`, ptrType: (*struct { Bool bool Int int64 Uint uint32 Float float64 String string Bytes []byte })(nil), prettyDagJSON: `{ "bool": true, "bytes": {"/": {"bytes": "34cd"}}, "float": 12.5, "int": 3, "string": "foo", "uint": 50 }`, }, { name: "Links", schemaSrc: `type Root struct { linkCID Link linkGeneric Link linkImpl Link }`, ptrType: (*struct { LinkCID cid.Cid LinkGeneric datamodel.Link LinkImpl cidlink.Link })(nil), prettyDagJSON: `{ "linkCID": {"/": "bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi"}, "linkGeneric": {"/": "bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi"}, "linkImpl": {"/": "bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi"} }`, }, { name: "Any", // TODO: also Null schemaSrc: `type Root struct { anyNodeWithBool Any anyNodeWithInt Any anyNodeWithFloat Any anyNodeWithString Any anyNodeWithBytes Any anyNodeWithList Any anyNodeWithMap Any anyNodeWithLink Any anyNodeBehindList [Any] anyNodeBehindMap {String:Any} }`, ptrType: (*struct { AnyNodeWithBool datamodel.Node AnyNodeWithInt datamodel.Node AnyNodeWithFloat datamodel.Node AnyNodeWithString datamodel.Node AnyNodeWithBytes datamodel.Node AnyNodeWithList datamodel.Node AnyNodeWithMap datamodel.Node AnyNodeWithLink datamodel.Node AnyNodeBehindList []datamodel.Node AnyNodeBehindMap struct { Keys []string Values map[string]datamodel.Node } })(nil), prettyDagJSON: `{ "anyNodeBehindList": [12.5, {"x": false}], "anyNodeBehindMap": {"x": 123, "y": [true, false]}, "anyNodeWithBool": true, "anyNodeWithBytes": {"/": {"bytes": "34cd"}}, "anyNodeWithFloat": 12.5, "anyNodeWithInt": 3, "anyNodeWithLink": {"/": "bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi"}, "anyNodeWithList": [3, 2, 1], "anyNodeWithMap": {"a": "x", "b": "y"}, "anyNodeWithString": "foo" }`, }, { name: "Enums", schemaSrc: `type Root struct { stringAsString EnumAsString stringAsStringCustom EnumAsString stringAsInt EnumAsInt intAsInt EnumAsInt uintAsInt EnumAsInt } type EnumAsString enum { | Nope ("No") | Yep ("Yes") | Maybe } type EnumAsInt enum { | Nope ("10") | Yep ("11") | Maybe ("12") } representation int`, ptrType: (*struct { StringAsString string StringAsStringCustom string StringAsInt string IntAsInt int32 UintAsInt uint16 })(nil), prettyDagJSON: `{ "intAsInt": 12, "stringAsInt": 10, "stringAsString": "Maybe", "stringAsStringCustom": "Yes", "uintAsInt": 11 }`, }, { name: "ScalarKindedUnions", // TODO: should we use an "Any" type from the prelude? schemaSrc: `type Root struct { boolAny AnyScalar intAny AnyScalar floatAny AnyScalar stringAny AnyScalar bytesAny AnyScalar linkAny AnyScalar } type AnyScalar union { | Bool bool | Int int | Float float | String string | Bytes bytes | Link link } representation kinded`, ptrType: (*struct { BoolAny anyScalar IntAny anyScalar FloatAny anyScalar StringAny anyScalar BytesAny anyScalar LinkAny anyScalar })(nil), prettyDagJSON: `{ "boolAny": true, "bytesAny": {"/": {"bytes": "34cd"}}, "floatAny": 12.5, "intAny": 3, "linkAny": {"/": "bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi"}, "stringAny": "foo" }`, }, { name: "RecursiveKindedUnions", // TODO: should we use an "Any" type from the prelude? // Especially since we use String map/list element types. // TODO: use inline map/list defs once schema and dsl+dmt support it. schemaSrc: `type Root struct { listAny AnyRecursive mapAny AnyRecursive } type List_String [String] type Map_String {String:String} type AnyRecursive union { | List_String list | Map_String map } representation kinded`, ptrType: (*struct { ListAny anyRecursive MapAny anyRecursive })(nil), prettyDagJSON: `{ "listAny": ["foo", "bar"], "mapAny": {"a": "x", "b": "y"} }`, }, } func compactJSON(t *testing.T, pretty string) string { var buf bytes.Buffer err := json.Compact(&buf, []byte(pretty)) qt.Assert(t, err, qt.IsNil) return buf.String() } func dagjsonEncode(t *testing.T, node datamodel.Node) string { var sb strings.Builder err := dagjson.Encode(node, &sb) qt.Assert(t, err, qt.IsNil) return sb.String() } func dagjsonDecode(t *testing.T, proto datamodel.NodePrototype, src string) datamodel.Node { nb := proto.NewBuilder() err := dagjson.Decode(nb, strings.NewReader(src)) qt.Assert(t, err, qt.IsNil) return nb.Build() } func TestPrototype(t *testing.T) { t.Parallel() for _, test := range prototypeTests { test := test // don't reuse the range var for _, onlySchema := range []bool{false, true} { onlySchema := onlySchema // don't reuse the range var suffix := "" if onlySchema { suffix = "_onlySchema" } t.Run(test.name+suffix, func(t *testing.T) { t.Parallel() ts, err := ipld.LoadSchemaBytes([]byte(test.schemaSrc)) qt.Assert(t, err, qt.IsNil) schemaType := ts.TypeByName("Root") qt.Assert(t, schemaType, qt.Not(qt.IsNil)) ptrType := test.ptrType // don't write to the shared test value if onlySchema { ptrType = nil } proto := bindnode.Prototype(ptrType, schemaType) wantEncoded := compactJSON(t, test.prettyDagJSON) node := dagjsonDecode(t, proto.Representation(), wantEncoded).(schema.TypedNode) // TODO: assert node type matches ptrType encoded := dagjsonEncode(t, node.Representation()) qt.Assert(t, encoded, qt.Equals, wantEncoded) // Verify that doing a dag-json encode of the non-repr node works. _ = dagjsonEncode(t, node) }) } } } func TestPrototypePointerCombinations(t *testing.T) { t.Parallel() // TODO: Null // TODO: cover more schema types and repr strategies. // Some of them are still using w.val directly without "nonPtr" calls. kindTests := []struct { name string schemaType string fieldPtrType interface{} fieldDagJSON string }{ {"Bool", "Bool", (*bool)(nil), `true`}, {"Int", "Int", (*int64)(nil), `23`}, {"Float", "Float", (*float64)(nil), `34.5`}, {"String", "String", (*string)(nil), `"foo"`}, {"Bytes", "Bytes", (*[]byte)(nil), `{"/": {"bytes": "34cd"}}`}, {"Link_CID", "Link", (*cid.Cid)(nil), `{"/": "bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi"}`}, {"Link_Impl", "Link", (*cidlink.Link)(nil), `{"/": "bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi"}`}, {"Link_Generic", "Link", (*datamodel.Link)(nil), `{"/": "bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi"}`}, {"List_String", "[String]", (*[]string)(nil), `["foo", "bar"]`}, {"Any_Node_Int", "Any", (*datamodel.Node)(nil), `23`}, // TODO: reenable once we don't require pointers for nullable // {"Any_Pointer_Int", "{String: nullable Any}", // (*struct { // Keys []string // Values map[string]datamodel.Node // })(nil), `{"x":3,"y":"bar","z":[2.3,4.5]}`}, {"Map_String_Int", "{String:Int}", (*struct { Keys []string Values map[string]int64 })(nil), `{"x":3,"y":4}`}, } // For each IPLD kind, we test a matrix of combinations for IPLD's optional // and nullable fields alongside pointer usage on the Go field side. modifiers := []struct { schemaField string // "", "optional", "nullable", "optional nullable" goPointers int // 0 (T), 1 (*T), 2 (**T) }{ {"", 0}, // regular IPLD field with Go's T {"", 1}, // regular IPLD field with Go's *T {"optional", 0}, // optional IPLD field with Go's T (skipped unless T is nilable) {"optional", 1}, // optional IPLD field with Go's *T {"nullable", 0}, // nullable IPLD field with Go's T (skipped unless T is nilable) {"nullable", 1}, // nullable IPLD field with Go's *T {"optional nullable", 2}, // optional and nullable IPLD field with Go's **T } for _, kindTest := range kindTests { for _, modifier := range modifiers { // don't reuse range vars kindTest := kindTest modifier := modifier goFieldType := reflect.TypeOf(kindTest.fieldPtrType) switch modifier.goPointers { case 0: goFieldType = goFieldType.Elem() // dereference fieldPtrType case 1: // fieldPtrType already uses one pointer case 2: goFieldType = reflect.PointerTo(goFieldType) // dereference fieldPtrType } if modifier.schemaField != "" && !nilable(goFieldType.Kind()) { continue } t.Run(fmt.Sprintf("%s/%s-%dptr", kindTest.name, modifier.schemaField, modifier.goPointers), func(t *testing.T) { t.Parallel() var buf bytes.Buffer err := template.Must(template.New("").Parse(` type Root struct { field {{.Modifier}} {{.Type}} }`)).Execute(&buf, struct { Type, Modifier string }{kindTest.schemaType, modifier.schemaField}) qt.Assert(t, err, qt.IsNil) schemaSrc := buf.String() t.Logf("IPLD schema: %s", schemaSrc) // *struct { Field {{.goFieldType}} } goType := reflect.Zero(reflect.PointerTo(reflect.StructOf([]reflect.StructField{ {Name: "Field", Type: goFieldType}, }))).Interface() t.Logf("Go type: %T", goType) ts, err := ipld.LoadSchemaBytes([]byte(schemaSrc)) qt.Assert(t, err, qt.IsNil) schemaType := ts.TypeByName("Root") qt.Assert(t, schemaType, qt.Not(qt.IsNil)) proto := bindnode.Prototype(goType, schemaType) wantEncodedBytes, err := json.Marshal(map[string]interface{}{"field": json.RawMessage(kindTest.fieldDagJSON)}) qt.Assert(t, err, qt.IsNil) wantEncoded := string(wantEncodedBytes) node := dagjsonDecode(t, proto.Representation(), wantEncoded).(schema.TypedNode) encoded := dagjsonEncode(t, node.Representation()) qt.Assert(t, encoded, qt.Equals, wantEncoded) // Assigning with the missing field should only work with optional. nb := proto.NewBuilder() err = dagjson.Decode(nb, strings.NewReader(`{}`)) switch modifier.schemaField { case "optional", "optional nullable": qt.Assert(t, err, qt.IsNil) node := nb.Build() // The resulting node should be non-nil with a nil field. nodeVal := reflect.ValueOf(bindnode.Unwrap(node)) qt.Assert(t, nodeVal.Elem().FieldByName("Field").IsNil(), qt.IsTrue) default: qt.Assert(t, err, qt.Not(qt.IsNil)) } // Assigning with a null field should only work with nullable. nb = proto.NewBuilder() err = dagjson.Decode(nb, strings.NewReader(`{"field":null}`)) switch modifier.schemaField { case "nullable", "optional nullable": qt.Assert(t, err, qt.IsNil) node := nb.Build() // The resulting node should be non-nil with a nil field. nodeVal := reflect.ValueOf(bindnode.Unwrap(node)) if modifier.schemaField == "nullable" { qt.Assert(t, nodeVal.Elem().FieldByName("Field").IsNil(), qt.IsTrue) } else { qt.Assert(t, nodeVal.Elem().FieldByName("Field").Elem().IsNil(), qt.IsTrue) } default: qt.Assert(t, err, qt.Not(qt.IsNil)) } }) } } } func nilable(kind reflect.Kind) bool { switch kind { case reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice: return true default: return false } } func assembleAsKind(proto datamodel.NodePrototype, schemaType schema.Type, asKind datamodel.Kind) (ipld.Node, error) { nb := proto.NewBuilder() switch asKind { case datamodel.Kind_Bool: if err := nb.AssignBool(true); err != nil { return nil, err } case datamodel.Kind_Int: if err := nb.AssignInt(123); err != nil { return nil, err } case datamodel.Kind_Float: if err := nb.AssignFloat(12.5); err != nil { return nil, err } case datamodel.Kind_String: if err := nb.AssignString("foo"); err != nil { return nil, err } case datamodel.Kind_Bytes: if err := nb.AssignBytes([]byte("\x00bar")); err != nil { return nil, err } case datamodel.Kind_Link: someCid, err := cid.Decode("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi") if err != nil { return nil, err } if err := nb.AssignLink(cidlink.Link{Cid: someCid}); err != nil { return nil, err } case datamodel.Kind_Map: asm, err := nb.BeginMap(-1) if err != nil { return nil, err } // First via AssembleKey. if err := asm.AssembleKey().AssignString("F1"); err != nil { return nil, err } if err := asm.AssembleValue().AssignInt(101); err != nil { return nil, err } // Then via AssembleEntry. entryAsm, err := asm.AssembleEntry("F2") if err != nil { return nil, err } if err := entryAsm.AssignInt(102); err != nil { return nil, err } // If this is a struct, using a missing field should error. if _, ok := schemaType.(*schema.TypeStruct); ok { if err := asm.AssembleKey().AssignString("MissingKey"); err != nil { return nil, err } if err := asm.AssembleValue().AssignInt(101); err == nil { return nil, fmt.Errorf("expected error on missing struct key") } } if err := asm.Finish(); err != nil { return nil, err } case datamodel.Kind_List: asm, err := nb.BeginList(-1) if err != nil { return nil, err } // Note that we want the list to have two integer elements, // which matches the map entries above, // so that the struct with tuple repr just works too. if err := asm.AssembleValue().AssignInt(101); err != nil { return nil, err } if err := asm.AssembleValue().AssignInt(102); err != nil { return nil, err } // If this is a struct, assembling one more tuple entry should error. if _, ok := schemaType.(*schema.TypeStruct); ok { if err := asm.AssembleValue().AssignInt(103); err == nil { return nil, fmt.Errorf("expected error on extra tuple entry") } } if err := asm.Finish(); err != nil { return nil, err } } node := nb.Build() if node == nil { // If we succeeded, node must never be nil. return nil, fmt.Errorf("built node is nil") } return node, nil } func useNodeAsKind(node datamodel.Node, asKind datamodel.Kind) error { if gotKind := node.Kind(); gotKind != asKind { // Return a dummy error to signal when the kind doesn't match. return datamodel.ErrWrongKind{MethodName: "TestKindMismatches_Dummy_Kind"} } // Just check that IsAbsent and IsNull don't panic, for now. _ = node.IsAbsent() _ = node.IsNull() proto := node.Prototype() if proto == nil { return fmt.Errorf("got a null Prototype") } // TODO: also check LookupByNode, LookupBySegment switch asKind { case datamodel.Kind_Bool: if _, err := node.AsBool(); err != nil { return err } case datamodel.Kind_Int: if _, err := node.AsInt(); err != nil { return err } case datamodel.Kind_Float: if _, err := node.AsFloat(); err != nil { return err } case datamodel.Kind_String: if _, err := node.AsString(); err != nil { return err } case datamodel.Kind_Bytes: if _, err := node.AsBytes(); err != nil { return err } case datamodel.Kind_Link: if _, err := node.AsLink(); err != nil { return err } case datamodel.Kind_Map: iter := node.MapIterator() if iter == nil { // Return a dummy error to signal whether iter is nil or not. return datamodel.ErrWrongKind{MethodName: "TestKindMismatches_Dummy_MapIterator"} } for !iter.Done() { _, _, err := iter.Next() if err != nil { return err } } // valid element entryNode, err := node.LookupByString("F1") if err != nil { return err } if err := useNodeAsKind(entryNode, datamodel.Kind_Int); err != nil { return err } // missing element _, missingErr := node.LookupByString("MissingKey") switch err := missingErr.(type) { case nil: return fmt.Errorf("lookup of a missing key succeeded") case datamodel.ErrNotExists: // expected for maps case schema.ErrInvalidKey: // expected for structs default: return err } switch l := node.Length(); l { case 2: case -1: // Return a dummy error to signal whether Length failed. return datamodel.ErrWrongKind{MethodName: "TestKindMismatches_Dummy_Length"} default: return fmt.Errorf("unexpected Length: %d", l) } case datamodel.Kind_List: iter := node.ListIterator() if iter == nil { // Return a dummy error to signal whether iter is nil or not. return datamodel.ErrWrongKind{MethodName: "TestKindMismatches_Dummy_ListIterator"} } for !iter.Done() { _, _, err := iter.Next() if err != nil { return err } } // valid element entryNode, err := node.LookupByIndex(1) if err != nil { return err } if err := useNodeAsKind(entryNode, datamodel.Kind_Int); err != nil { return err } // missing element _, missingErr := node.LookupByIndex(30) switch err := missingErr.(type) { case nil: return fmt.Errorf("lookup of a missing key succeeded") case datamodel.ErrNotExists: // expected for maps case schema.ErrInvalidKey: // expected for structs default: return err } switch l := node.Length(); l { case 2: case -1: // Return a dummy error to signal whether Length failed. return datamodel.ErrWrongKind{MethodName: "TestKindMismatches_Dummy_Length"} default: return fmt.Errorf("unexpected Length: %d", l) } } return nil } func TestKindMismatches(t *testing.T) { t.Parallel() kindTests := []struct { name string schemaSrc string }{ {"Bool", "type Root bool"}, {"Int", "type Root int"}, {"Float", "type Root float"}, {"String", "type Root string"}, {"Bytes", "type Root bytes"}, {"Map", ` type Root {String:Int} `}, {"Struct", ` type Root struct { F1 Int F2 Int } `}, {"Struct_Tuple", ` type Root struct { F1 Int F2 Int } representation tuple `}, {"Enum", ` type Root enum { | Foo ("foo") | Bar ("bar") | Either } `}, {"Union_Kinded_onlyString", ` type Root union { | String string } representation kinded `}, {"Union_Kinded_onlyList", ` type Root union { | List list } representation kinded `}, // TODO: more schema types and repr strategies } allKinds := []datamodel.Kind{ // datamodel.Kind_Null, TODO datamodel.Kind_Bool, datamodel.Kind_Int, datamodel.Kind_Float, datamodel.Kind_String, datamodel.Kind_Bytes, datamodel.Kind_Link, datamodel.Kind_Map, datamodel.Kind_List, } // TODO: also test for non-repr assemblers and nodes for _, kindTest := range kindTests { // don't reuse range vars kindTest := kindTest t.Run(kindTest.name, func(t *testing.T) { t.Parallel() defer func() { if r := recover(); r != nil { // Note that debug.Stack inside the recover will include the // stack trace for the original panic call. t.Errorf("caught panic:\n%v\n%s", r, debug.Stack()) } }() ts, err := ipld.LoadSchemaBytes([]byte(kindTest.schemaSrc)) qt.Assert(t, err, qt.IsNil) schemaType := ts.TypeByName("Root") qt.Assert(t, schemaType, qt.Not(qt.IsNil)) // Note that the Go type is inferred. proto := bindnode.Prototype(nil, schemaType).Representation() reprBehaviorKind := schemaType.RepresentationBehavior() if reprBehaviorKind == datamodel.Kind_Invalid { // For now, this only applies to kinded unions. // We'll need to modify this when we test with Any. members := schemaType.(*schema.TypeUnion).Members() qt.Assert(t, members, qt.HasLen, 1) reprBehaviorKind = members[0].RepresentationBehavior() } qt.Assert(t, reprBehaviorKind, qt.Not(qt.Equals), datamodel.Kind_Invalid) for _, kind := range allKinds { _, err := assembleAsKind(proto, schemaType, kind) comment := qt.Commentf("assigned as %v", kind) // Assembling should succed iff we used the right kind. if kind == reprBehaviorKind { qt.Assert(t, err, qt.IsNil, comment) } else { qt.Assert(t, err, qt.Not(qt.IsNil), comment) qt.Assert(t, err, qt.ErrorAs, new(datamodel.ErrWrongKind), comment) } } node, err := assembleAsKind(proto, schemaType, reprBehaviorKind) qt.Assert(t, err, qt.IsNil) node = node.(schema.TypedNode).Representation() nodeKind := node.Kind() for _, kind := range allKinds { err := useNodeAsKind(node, kind) comment := qt.Commentf("used as %v", kind) // Using the node should succed iff we used the right kind. if kind == nodeKind { qt.Assert(t, err, qt.IsNil, comment) } else { qt.Assert(t, err, qt.Not(qt.IsNil), comment) qt.Assert(t, err, qt.ErrorAs, new(datamodel.ErrWrongKind), comment) } } }) } } type verifyBadType struct { ptrType interface{} panicRegexp string } type ( namedBool bool namedInt64 int64 namedFloat64 float64 namedString string namedBytes []byte ) var verifyTests = []struct { name string schemaSrc string goodTypes []interface{} badTypes []verifyBadType }{ { name: "Bool", schemaSrc: `type Root bool`, goodTypes: []interface{}{ (*bool)(nil), (*namedBool)(nil), }, badTypes: []verifyBadType{ {(*string)(nil), `.*type Root .* type string: kind mismatch;.*`}, }, }, { name: "Int", schemaSrc: `type Root int`, goodTypes: []interface{}{ (*int)(nil), (*namedInt64)(nil), (*int8)(nil), (*int16)(nil), (*int32)(nil), (*int64)(nil), }, badTypes: []verifyBadType{ {(*string)(nil), `.*type Root .* type string: kind mismatch;.*`}, }, }, { name: "Float", schemaSrc: `type Root float`, goodTypes: []interface{}{ (*float64)(nil), (*namedFloat64)(nil), (*float32)(nil), }, badTypes: []verifyBadType{ {(*string)(nil), `.*type Root .* type string: kind mismatch;.*`}, }, }, { name: "String", schemaSrc: `type Root string`, goodTypes: []interface{}{ (*string)(nil), (*namedString)(nil), }, badTypes: []verifyBadType{ {(*int)(nil), `.*type Root .* type int: kind mismatch;.*`}, }, }, { name: "Bytes", schemaSrc: `type Root bytes`, goodTypes: []interface{}{ (*[]byte)(nil), (*namedBytes)(nil), (*[]uint8)(nil), // alias of byte }, badTypes: []verifyBadType{ {(*int)(nil), `.*type Root .* type int: kind mismatch;.*`}, {(*[]int)(nil), `.*type Root .* type \[\]int: kind mismatch;.*`}, }, }, { name: "List", schemaSrc: `type Root [String]`, goodTypes: []interface{}{ (*[]string)(nil), (*[]namedString)(nil), }, badTypes: []verifyBadType{ {(*string)(nil), `.*type Root .* type string: kind mismatch;.*`}, {(*[]int)(nil), `.*type String .* type int: kind mismatch;.*`}, {(*[3]string)(nil), `.*type Root .* type \[3\]string: kind mismatch;.*`}, }, }, { name: "Struct", schemaSrc: `type Root struct { int Int }`, goodTypes: []interface{}{ (*struct{ Int int })(nil), (*struct{ Int namedInt64 })(nil), }, badTypes: []verifyBadType{ {(*string)(nil), `.*type Root .* type string: kind mismatch;.*`}, {(*struct{ Int bool })(nil), `.*type Int .* type bool: kind mismatch;.*`}, {(*struct{ Int1, Int2 int })(nil), `.*type Root .* type struct {.*}: 2 vs 1 fields`}, }, }, { name: "Map", schemaSrc: `type Root {String:Int}`, goodTypes: []interface{}{ (*struct { Keys []string Values map[string]int })(nil), (*struct { Keys []namedString Values map[namedString]namedInt64 })(nil), }, badTypes: []verifyBadType{ {(*string)(nil), `.*type Root .* type string: kind mismatch;.*`}, {(*struct{ Keys []string })(nil), `.*type Root .*: 1 vs 2 fields`}, {(*struct{ Values map[string]int })(nil), `.*type Root .*: 1 vs 2 fields`}, {(*struct { Keys string Values map[string]int })(nil), `.*type Root .*: kind mismatch;.*`}, {(*struct { Keys []string Values string })(nil), `.*type Root .*: kind mismatch;.*`}, }, }, { name: "MapNullableAny", schemaSrc: `type Root {String:nullable Any}`, goodTypes: []interface{}{ (*struct { Keys []string Values map[string]*datamodel.Node })(nil), (*struct { Keys []string Values map[string]datamodel.Node })(nil), }, badTypes: []verifyBadType{ {(*string)(nil), `.*type Root .* type string: kind mismatch;.*`}, }, }, { name: "Union", schemaSrc: `type Root union { | List_String list | String string } representation kinded type List_String [String] `, goodTypes: []interface{}{ (*struct { List *[]string String *string })(nil), (*struct { List []string String *string })(nil), (*struct { List *[]namedString String *namedString })(nil), }, badTypes: []verifyBadType{ {(*string)(nil), `.*type Root .* type string: kind mismatch;.*`}, {(*struct{ List *[]string })(nil), `.*type Root .*: 1 vs 2 members`}, {(*struct { List []string String string })(nil), `.*type Root .*: union members must be nilable`}, {(*struct { List *[]string String *int })(nil), `.*type String .*: kind mismatch;.*`}, }, }, } func TestSchemaVerify(t *testing.T) { t.Parallel() for _, test := range verifyTests { test := test // don't reuse the range var t.Run(test.name, func(t *testing.T) { t.Parallel() ts, err := ipld.LoadSchemaBytes([]byte(test.schemaSrc)) qt.Assert(t, err, qt.IsNil) schemaType := ts.TypeByName("Root") qt.Assert(t, schemaType, qt.Not(qt.IsNil)) for _, ptrType := range test.goodTypes { proto := bindnode.Prototype(ptrType, schemaType) qt.Assert(t, proto, qt.Not(qt.IsNil)) ptrVal := reflect.New(reflect.TypeOf(ptrType).Elem()).Interface() node := bindnode.Wrap(ptrVal, schemaType) qt.Assert(t, node, qt.Not(qt.IsNil)) } for _, bad := range test.badTypes { qt.Check(t, func() { bindnode.Prototype(bad.ptrType, schemaType) }, qt.PanicMatches, bad.panicRegexp) ptrVal := reflect.New(reflect.TypeOf(bad.ptrType).Elem()).Interface() qt.Check(t, func() { bindnode.Wrap(ptrVal, schemaType) }, qt.PanicMatches, bad.panicRegexp) } }) } } func TestProduceGoTypes(t *testing.T) { t.Parallel() for _, test := range prototypeTests { test := test // don't reuse the range var t.Run(test.name, func(t *testing.T) { t.Parallel() ts, err := ipld.LoadSchemaBytes([]byte(test.schemaSrc)) qt.Assert(t, err, qt.IsNil) // Include a package line and the datamodel import. buf := new(bytes.Buffer) fmt.Fprintln(buf, `package p`) fmt.Fprintln(buf, `import "github.com/ipld/go-ipld-prime/datamodel"`) fmt.Fprintln(buf, `var _ datamodel.Link // always used`) err = bindnode.ProduceGoTypes(buf, ts) qt.Assert(t, err, qt.IsNil) // Ensure that the output builds, i.e. typechecks. genPath := filepath.Join(t.TempDir(), "gen.go") err = os.WriteFile(genPath, buf.Bytes(), 0o666) qt.Assert(t, err, qt.IsNil) out, err := exec.Command("go", "build", genPath).CombinedOutput() qt.Assert(t, err, qt.IsNil, qt.Commentf("output: %s", out)) // TODO: check that the generated types are compatible with the schema. }) } } func TestRenameAssignNode(t *testing.T) { type Foo struct{ I int } ts, _ := ipld.LoadSchemaBytes([]byte(` type Foo struct { I Int (rename "J") } `)) FooProto := bindnode.Prototype((*Foo)(nil), ts.TypeByName("Foo")) // Decode straight into bindnode typed builder nb := FooProto.Representation().NewBuilder() err := dagjson.Decode(nb, bytes.NewReader([]byte(`{"J":100}`))) qt.Assert(t, err, qt.IsNil) nb.Build() // decode into basicnode builder nb = basicnode.Prototype.Any.NewBuilder() err = dagjson.Decode(nb, bytes.NewReader([]byte(`{"J":100}`))) qt.Assert(t, err, qt.IsNil) node := nb.Build() // AssignNode from the basicnode form nb = FooProto.Representation().NewBuilder() err = nb.AssignNode(node) qt.Assert(t, err, qt.IsNil) nb.Build() } func TestEmptyTypedAssignNode(t *testing.T) { type Foo struct { I string J string K int } type Foo1Optional struct { I string J string K *int } type Foo2Optional struct { I string J *string K *int } tupleSchema := `type Foo struct { I String J String K Int } representation tuple` tuple1OptionalSchema := `type Foo struct { I String J String K optional Int } representation tuple` tuple2OptionalSchema := `type Foo struct { I String J optional String K optional Int } representation tuple` testCases := map[string]struct { schema string typ interface{} dagJson string err string }{ "tuple": { schema: tupleSchema, typ: (*Foo)(nil), dagJson: `["","",0]`, }, "tuple with 2 absents": { schema: tupleSchema, typ: (*Foo)(nil), dagJson: `[""]`, err: "missing required fields: J,K", }, "tuple with 1 optional": { schema: tuple1OptionalSchema, typ: (*Foo1Optional)(nil), dagJson: `["","",0]`, }, "tuple with 1 optional and absent": { schema: tuple1OptionalSchema, typ: (*Foo1Optional)(nil), dagJson: `["",""]`, }, "tuple with 1 optional and 2 absents": { schema: tuple1OptionalSchema, typ: (*Foo1Optional)(nil), dagJson: `[""]`, err: "missing required fields: J", }, "tuple with 2 optional": { schema: tuple2OptionalSchema, typ: (*Foo2Optional)(nil), dagJson: `["","",0]`, }, "tuple with 2 optional and 1 absent": { schema: tuple2OptionalSchema, typ: (*Foo2Optional)(nil), dagJson: `["",""]`, }, "tuple with 2 optional and 2 absent": { schema: tuple2OptionalSchema, typ: (*Foo2Optional)(nil), dagJson: `[""]`, }, "tuple with 2 optional and 3 absent": { schema: tuple2OptionalSchema, typ: (*Foo2Optional)(nil), dagJson: `[]`, err: "missing required fields: I", }, "map": { schema: `type Foo struct { I String J String K Int } representation map `, typ: (*Foo)(nil), dagJson: `{"I":"","J":"","K":0}`, }, } for testCase, data := range testCases { t.Run(testCase, func(t *testing.T) { ts, _ := ipld.LoadSchemaBytes([]byte(data.schema)) FooProto := bindnode.Prototype(data.typ, ts.TypeByName("Foo")) // decode an "empty" object into Foo, these are all default values nb := basicnode.Prototype.Any.NewBuilder() err := dagjson.Decode(nb, bytes.NewReader([]byte(data.dagJson))) qt.Assert(t, err, qt.IsNil) node := nb.Build() // AssignNode from the basicnode form nb = FooProto.Representation().NewBuilder() err = nb.AssignNode(node) if data.err == "" { qt.Assert(t, err, qt.IsNil) } else { qt.Assert(t, err, qt.ErrorMatches, data.err) } nb.Build() // make an "empty" form, although none of the fields are optional so we should end up with defaults nb = FooProto.Representation().NewBuilder() empty := nb.Build() // AssignNode from the representation of the "empty" form, which should pass through default values nb = FooProto.Representation().NewBuilder() err = nb.AssignNode(empty.(schema.TypedNode).Representation()) qt.Assert(t, err, qt.IsNil) }) } } func TestInferLinksAndAny(t *testing.T) { link, err := cid.Decode("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi") qt.Assert(t, err, qt.IsNil) type Nd struct { A cid.Cid B cidlink.Link C datamodel.Link D datamodel.Node } proto := bindnode.Prototype(&Nd{}, nil) expected := &Nd{ A: link, B: cidlink.Link{Cid: link}, C: cidlink.Link{Cid: link}, D: basicnode.NewString("Any Here"), } node := bindnode.Wrap(expected, proto.Type()) byts, err := ipld.Encode(node, dagjson.Encode) qt.Assert(t, err, qt.IsNil) qt.Assert(t, string(byts), qt.Equals, `{"A":{"/":"bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi"},"B":{"/":"bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi"},"C":{"/":"bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi"},"D":"Any Here"}`) nodeRt, err := ipld.DecodeUsingPrototype(byts, dagjson.Decode, proto) qt.Assert(t, err, qt.IsNil) if actual, ok := bindnode.Unwrap(nodeRt).(*Nd); ok { qt.Assert(t, actual, qt.CmpEquals(cmp.Comparer(func(x, y cid.Cid) bool { return x.Equals(y) })), expected) } else { t.Error("expected *Nd from Unwrap") } } ================================================ FILE: node/bindnode/node.go ================================================ package bindnode import ( "fmt" "math" "reflect" "runtime" "strings" "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime/datamodel" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/node/mixins" "github.com/ipld/go-ipld-prime/schema" ) // Assert that we implement all the interfaces as expected. // Grouped by the interfaces to implement, roughly. var ( _ datamodel.NodePrototype = (*_prototype)(nil) _ schema.TypedPrototype = (*_prototype)(nil) _ datamodel.NodePrototype = (*_prototypeRepr)(nil) _ datamodel.Node = (*_node)(nil) _ schema.TypedNode = (*_node)(nil) _ datamodel.Node = (*_nodeRepr)(nil) _ datamodel.Node = (*_uintNode)(nil) _ schema.TypedNode = (*_uintNode)(nil) _ datamodel.UintNode = (*_uintNode)(nil) _ datamodel.Node = (*_uintNodeRepr)(nil) _ datamodel.UintNode = (*_uintNodeRepr)(nil) _ datamodel.NodeBuilder = (*_builder)(nil) _ datamodel.NodeBuilder = (*_builderRepr)(nil) _ datamodel.NodeAssembler = (*_assembler)(nil) _ datamodel.NodeAssembler = (*_assemblerRepr)(nil) _ datamodel.NodeAssembler = (*_errorAssembler)(nil) _ datamodel.NodeAssembler = (*_listpairsFieldAssemblerRepr)(nil) _ datamodel.MapAssembler = (*_structAssembler)(nil) _ datamodel.MapAssembler = (*_structAssemblerRepr)(nil) _ datamodel.MapIterator = (*_structIterator)(nil) _ datamodel.MapIterator = (*_structIteratorRepr)(nil) _ datamodel.ListAssembler = (*_listAssembler)(nil) _ datamodel.ListAssembler = (*_listAssemblerRepr)(nil) _ datamodel.ListAssembler = (*_listStructAssemblerRepr)(nil) _ datamodel.ListAssembler = (*_listpairsFieldListAssemblerRepr)(nil) _ datamodel.ListIterator = (*_listIterator)(nil) _ datamodel.ListIterator = (*_tupleIteratorRepr)(nil) _ datamodel.ListIterator = (*_listpairsIteratorRepr)(nil) _ datamodel.MapAssembler = (*_unionAssembler)(nil) _ datamodel.MapAssembler = (*_unionAssemblerRepr)(nil) _ datamodel.MapIterator = (*_unionIterator)(nil) _ datamodel.MapIterator = (*_unionIteratorRepr)(nil) ) type _prototype struct { cfg *config schemaType schema.Type goType reflect.Type // non-pointer } func (w *_prototype) NewBuilder() datamodel.NodeBuilder { return &_builder{_assembler{ cfg: w.cfg, schemaType: w.schemaType, val: reflect.New(w.goType).Elem(), }} } func (w *_prototype) Type() schema.Type { return w.schemaType } func (w *_prototype) Representation() datamodel.NodePrototype { return (*_prototypeRepr)(w) } type _node struct { cfg *config schemaType schema.Type val reflect.Value // non-pointer } // TODO: only expose TypedNode methods if the schema was explicit. // type _typedNode struct { // _node // } func newNode(cfg *config, schemaType schema.Type, val reflect.Value) schema.TypedNode { if schemaType.TypeKind() == schema.TypeKind_Int && nonPtrVal(val).Kind() == reflect.Uint64 { // special case for uint64 values so we can handle the >int64 range // we give this treatment to all uint64s, regardless of current value // because we have no guarantees the value won't change underneath us return &_uintNode{ cfg: cfg, schemaType: schemaType, val: val, } } return &_node{cfg, schemaType, val} } func (w *_node) Type() schema.Type { return w.schemaType } func (w *_node) Representation() datamodel.Node { return (*_nodeRepr)(w) } func (w *_node) Kind() datamodel.Kind { return actualKind(w.schemaType) } // matching schema level types to data model kinds, since our Node and Builder // interfaces operate on kinds func compatibleKind(schemaType schema.Type, kind datamodel.Kind) error { switch sch := schemaType.(type) { case *schema.TypeAny: return nil default: actual := actualKind(sch) // ActsLike data model if actual == kind { return nil } // Error methodName := "" if pc, _, _, ok := runtime.Caller(1); ok { if fn := runtime.FuncForPC(pc); fn != nil { methodName = fn.Name() // Go from "pkg/path.Type.Method" to just "Method". methodName = methodName[strings.LastIndexByte(methodName, '.')+1:] } } return datamodel.ErrWrongKind{ TypeName: schemaType.Name(), MethodName: methodName, AppropriateKind: datamodel.KindSet{kind}, ActualKind: actual, } } } func actualKind(schemaType schema.Type) datamodel.Kind { return schemaType.TypeKind().ActsLike() } func nonPtrVal(val reflect.Value) reflect.Value { // TODO: support **T as well as *T? if val.Kind() == reflect.Ptr { if val.IsNil() { // TODO: error in this case? return reflect.Value{} } val = val.Elem() } return val } func ptrVal(val reflect.Value) reflect.Value { if val.Kind() == reflect.Ptr { return val } return val.Addr() } func nonPtrType(val reflect.Value) reflect.Type { typ := val.Type() if typ.Kind() == reflect.Ptr { return typ.Elem() } return typ } // where we need to cal Set(), ensure the Value we're setting is a pointer or // not, depending on the field we're setting into. func matchSettable(val interface{}, to reflect.Value) reflect.Value { setVal := nonPtrVal(reflect.ValueOf(val)) if !setVal.Type().AssignableTo(to.Type()) && setVal.Type().ConvertibleTo(to.Type()) { setVal = setVal.Convert(to.Type()) } return setVal } func (w *_node) LookupByString(key string) (datamodel.Node, error) { switch typ := w.schemaType.(type) { case *schema.TypeStruct: field := typ.Field(key) if field == nil { return nil, schema.ErrInvalidKey{ TypeName: typ.Name(), Key: basicnode.NewString(key), } } fval := nonPtrVal(w.val).FieldByName(fieldNameFromSchema(key)) if !fval.IsValid() { return nil, fmt.Errorf("bindnode TODO: go-schema mismatch") } if field.IsOptional() { if fval.IsNil() { return datamodel.Absent, nil } if fval.Kind() == reflect.Ptr { fval = fval.Elem() } } if field.IsNullable() { if fval.IsNil() { return datamodel.Null, nil } if fval.Kind() == reflect.Ptr { fval = fval.Elem() } } if _, ok := field.Type().(*schema.TypeAny); ok { if customConverter := w.cfg.converterFor(field.Type().Name(), fval); customConverter != nil { // field is an Any and we have a custom type converter for the type return customConverter.customToAny(ptrVal(fval).Interface()) } // field is an Any, safely assume a Node in fval return nonPtrVal(fval).Interface().(datamodel.Node), nil } return newNode(w.cfg, field.Type(), fval), nil case *schema.TypeMap: // maps can only be structs with a Values map var kval reflect.Value valuesVal := nonPtrVal(w.val).FieldByName("Values") switch ktyp := typ.KeyType().(type) { case *schema.TypeString: // plain String keys, so safely use the map key as is kval = reflect.ValueOf(key) default: // key is something other than a string that we need to assemble via // the string representation form, use _assemblerRepr to reverse from // string to the type that indexes the map asm := &_assembler{ cfg: w.cfg, schemaType: ktyp, val: reflect.New(valuesVal.Type().Key()).Elem(), } if err := (*_assemblerRepr)(asm).AssignString(key); err != nil { return nil, err } kval = asm.val } fval := valuesVal.MapIndex(kval) if !fval.IsValid() { // not found return nil, datamodel.ErrNotExists{Segment: datamodel.PathSegmentOfString(key)} } // TODO: Error/panic if fval.IsNil() && !typ.ValueIsNullable()? // Otherwise we could have two non-equal Go values (nil map, // non-nil-but-empty map) which represent the exact same IPLD // node when the field is not nullable. if typ.ValueIsNullable() { if fval.IsNil() { return datamodel.Null, nil } fval = fval.Elem() } if _, ok := typ.ValueType().(*schema.TypeAny); ok { if customConverter := w.cfg.converterFor(typ.ValueType().Name(), fval); customConverter != nil { // value is an Any and we have a custom type converter for the type return customConverter.customToAny(ptrVal(fval).Interface()) } // value is an Any, safely assume a Node in fval return nonPtrVal(fval).Interface().(datamodel.Node), nil } return newNode(w.cfg, typ.ValueType(), fval), nil case *schema.TypeUnion: // treat a union similar to a struct, but we have the member names more // easily accessible to match to 'key' var idx int var mtyp schema.Type for i, member := range typ.Members() { if member.Name() == key { idx = i mtyp = member break } } if mtyp == nil { // not found return nil, datamodel.ErrNotExists{Segment: datamodel.PathSegmentOfString(key)} } // TODO: we could look up the right Go field straight away via idx. haveIdx, mval := unionMember(nonPtrVal(w.val)) if haveIdx != idx { // mismatching type return nil, datamodel.ErrNotExists{Segment: datamodel.PathSegmentOfString(key)} } return newNode(w.cfg, mtyp, mval), nil } return nil, datamodel.ErrWrongKind{ TypeName: w.schemaType.Name(), MethodName: "LookupByString", AppropriateKind: datamodel.KindSet_JustMap, ActualKind: w.Kind(), } } var invalidValue reflect.Value // unionMember finds which union member is set in the corresponding Go struct. func unionMember(val reflect.Value) (int, reflect.Value) { // The first non-nil field is a match. for i := 0; i < val.NumField(); i++ { elemVal := val.Field(i) if elemVal.Kind() != reflect.Ptr { panic("bindnode bug: found unexpected non-pointer in a union field") } if elemVal.IsNil() { continue } return i, elemVal.Elem() } return -1, invalidValue } func unionSetMember(val reflect.Value, memberIdx int, memberPtr reflect.Value) { // Reset the entire union struct to zero, to clear any non-nil pointers. val.Set(reflect.Zero(val.Type())) // Set the index pointer to the given value. val.Field(memberIdx).Set(memberPtr) } func (w *_node) LookupByIndex(idx int64) (datamodel.Node, error) { switch typ := w.schemaType.(type) { case *schema.TypeList: val := nonPtrVal(w.val) // we should be able assume that val is something we can Len() and Index() if idx < 0 || int(idx) >= val.Len() { return nil, datamodel.ErrNotExists{Segment: datamodel.PathSegmentOfInt(idx)} } val = val.Index(int(idx)) _, isAny := typ.ValueType().(*schema.TypeAny) if isAny { if customConverter := w.cfg.converterFor(typ.ValueType().Name(), val); customConverter != nil { // values are Any and we have a converter for this type that will give us // a datamodel.Node return customConverter.customToAny(ptrVal(val).Interface()) } } if typ.ValueIsNullable() { if val.IsNil() { return datamodel.Null, nil } // nullable elements are assumed to be pointers val = val.Elem() } if isAny { // Any always yields a plain datamodel.Node return nonPtrVal(val).Interface().(datamodel.Node), nil } return newNode(w.cfg, typ.ValueType(), val), nil } return nil, datamodel.ErrWrongKind{ TypeName: w.schemaType.Name(), MethodName: "LookupByIndex", AppropriateKind: datamodel.KindSet_JustList, ActualKind: w.Kind(), } } func (w *_node) LookupBySegment(seg datamodel.PathSegment) (datamodel.Node, error) { switch w.Kind() { case datamodel.Kind_Map: return w.LookupByString(seg.String()) case datamodel.Kind_List: idx, err := seg.Index() if err != nil { return nil, err } return w.LookupByIndex(idx) } return nil, datamodel.ErrWrongKind{ TypeName: w.schemaType.Name(), MethodName: "LookupBySegment", AppropriateKind: datamodel.KindSet_Recursive, ActualKind: w.Kind(), } } func (w *_node) LookupByNode(key datamodel.Node) (datamodel.Node, error) { switch w.Kind() { case datamodel.Kind_Map: s, err := key.AsString() if err != nil { return nil, err } return w.LookupByString(s) case datamodel.Kind_List: i, err := key.AsInt() if err != nil { return nil, err } return w.LookupByIndex(i) } return nil, datamodel.ErrWrongKind{ TypeName: w.schemaType.Name(), MethodName: "LookupByNode", AppropriateKind: datamodel.KindSet_Recursive, ActualKind: w.Kind(), } } func (w *_node) MapIterator() datamodel.MapIterator { val := nonPtrVal(w.val) // structs, unions and maps can all iterate but they each have different // access semantics for the underlying type, so we need a different iterator // for each switch typ := w.schemaType.(type) { case *schema.TypeStruct: return &_structIterator{ cfg: w.cfg, schemaType: typ, fields: typ.Fields(), val: val, } case *schema.TypeUnion: return &_unionIterator{ cfg: w.cfg, schemaType: typ, members: typ.Members(), val: val, } case *schema.TypeMap: // we can assume a: struct{Keys []string, Values map[x]y} return &_mapIterator{ cfg: w.cfg, schemaType: typ, keysVal: val.FieldByName("Keys"), valuesVal: val.FieldByName("Values"), } } return nil } func (w *_node) ListIterator() datamodel.ListIterator { val := nonPtrVal(w.val) switch typ := w.schemaType.(type) { case *schema.TypeList: return &_listIterator{cfg: w.cfg, schemaType: typ, val: val} } return nil } func (w *_node) Length() int64 { val := nonPtrVal(w.val) switch w.Kind() { case datamodel.Kind_Map: switch typ := w.schemaType.(type) { case *schema.TypeStruct: return int64(len(typ.Fields())) case *schema.TypeUnion: return 1 } return int64(val.FieldByName("Keys").Len()) case datamodel.Kind_List: return int64(val.Len()) } return -1 } // TODO: better story around pointers and absent/null func (w *_node) IsAbsent() bool { return false } func (w *_node) IsNull() bool { return false } // The AsX methods are matter of fetching the non-pointer form of the underlying // value and returning the appropriate Go type. The user may have registered // custom converters for the kind being converted, in which case the underlying // type may not be the type we need, but the converter will supply it for us. func (w *_node) AsBool() (bool, error) { if err := compatibleKind(w.schemaType, datamodel.Kind_Bool); err != nil { return false, err } if customConverter := w.cfg.converterFor(w.schemaType.Name(), w.val); customConverter != nil { // user has registered a converter that takes the underlying type and returns a bool return customConverter.customToBool(ptrVal(w.val).Interface()) } return nonPtrVal(w.val).Bool(), nil } func (w *_node) AsInt() (int64, error) { if err := compatibleKind(w.schemaType, datamodel.Kind_Int); err != nil { return 0, err } if customConverter := w.cfg.converterFor(w.schemaType.Name(), w.val); customConverter != nil { // user has registered a converter that takes the underlying type and returns an int return customConverter.customToInt(ptrVal(w.val).Interface()) } val := nonPtrVal(w.val) if kindUint[val.Kind()] { u := val.Uint() if u > math.MaxInt64 { return 0, fmt.Errorf("bindnode: integer overflow, %d is too large for an int64", u) } return int64(u), nil } return val.Int(), nil } func (w *_node) AsFloat() (float64, error) { if err := compatibleKind(w.schemaType, datamodel.Kind_Float); err != nil { return 0, err } if customConverter := w.cfg.converterFor(w.schemaType.Name(), w.val); customConverter != nil { // user has registered a converter that takes the underlying type and returns a float return customConverter.customToFloat(ptrVal(w.val).Interface()) } return nonPtrVal(w.val).Float(), nil } func (w *_node) AsString() (string, error) { if err := compatibleKind(w.schemaType, datamodel.Kind_String); err != nil { return "", err } if customConverter := w.cfg.converterFor(w.schemaType.Name(), w.val); customConverter != nil { // user has registered a converter that takes the underlying type and returns a string return customConverter.customToString(ptrVal(w.val).Interface()) } return nonPtrVal(w.val).String(), nil } func (w *_node) AsBytes() ([]byte, error) { if err := compatibleKind(w.schemaType, datamodel.Kind_Bytes); err != nil { return nil, err } if customConverter := w.cfg.converterFor(w.schemaType.Name(), w.val); customConverter != nil { // user has registered a converter that takes the underlying type and returns a []byte return customConverter.customToBytes(ptrVal(w.val).Interface()) } return nonPtrVal(w.val).Bytes(), nil } func (w *_node) AsLink() (datamodel.Link, error) { if err := compatibleKind(w.schemaType, datamodel.Kind_Link); err != nil { return nil, err } if customConverter := w.cfg.converterFor(w.schemaType.Name(), w.val); customConverter != nil { // user has registered a converter that takes the underlying type and returns a cid.Cid cid, err := customConverter.customToLink(ptrVal(w.val).Interface()) if err != nil { return nil, err } return cidlink.Link{Cid: cid}, nil } switch val := nonPtrVal(w.val).Interface().(type) { case datamodel.Link: return val, nil case cid.Cid: return cidlink.Link{Cid: val}, nil default: return nil, fmt.Errorf("bindnode: unexpected link type %T", val) } } func (w *_node) Prototype() datamodel.NodePrototype { return &_prototype{cfg: w.cfg, schemaType: w.schemaType, goType: w.val.Type()} } type _builder struct { _assembler } func (w *_builder) Build() datamodel.Node { // TODO: should we panic if no Assign call was made, just like codegen? return newNode(w.cfg, w.schemaType, w.val) } func (w *_builder) Reset() { panic("bindnode TODO: Reset") } type _assembler struct { cfg *config schemaType schema.Type val reflect.Value // non-pointer // finish is used as an optional post-assemble step. // For example, assigning to a kinded union uses a finish func // to set the right union member in the Go union struct, // which isn't known before the assemble has finished. finish func() error nullable bool // true if field or map value is nullable } // createNonPtrVal is used for Set() operations on the underlying value func (w *_assembler) createNonPtrVal() reflect.Value { val := w.val // TODO: if val is not a pointer, we reuse its value. // If it is a pointer, we allocate a new one and replace it. // We should probably never reuse the existing value. // TODO: support **T as well as *T? if val.Kind() == reflect.Ptr { // TODO: Sometimes we call createNonPtrVal before an assignment actually // happens. Does that matter? // If it matters and we only want to modify the destination value on // success, then we should make use of the "finish" func. val.Set(reflect.New(val.Type().Elem())) val = val.Elem() } return val } func (w *_assembler) Representation() datamodel.NodeAssembler { return (*_assemblerRepr)(w) } // basicMapAssembler is for assembling basicnode values, it's only use is for // Any fields that end up needing a BeginMap() type basicMapAssembler struct { datamodel.MapAssembler builder datamodel.NodeBuilder parent *_assembler converter *converter } func (w *basicMapAssembler) Finish() error { if err := w.MapAssembler.Finish(); err != nil { return err } basicNode := w.builder.Build() if w.converter != nil { // we can assume an Any converter because basicMapAssembler is only for Any // the user has registered the ability to convert a datamodel.Node to the // underlying Go type which may not be a datamodel.Node typ, err := w.converter.customFromAny(basicNode) if err != nil { return err } w.parent.createNonPtrVal().Set(matchSettable(typ, reflect.ValueOf(basicNode))) } else { w.parent.createNonPtrVal().Set(reflect.ValueOf(basicNode)) } if w.parent.finish != nil { if err := w.parent.finish(); err != nil { return err } } return nil } func (w *_assembler) BeginMap(sizeHint int64) (datamodel.MapAssembler, error) { switch typ := w.schemaType.(type) { case *schema.TypeAny: basicBuilder := basicnode.Prototype.Any.NewBuilder() mapAsm, err := basicBuilder.BeginMap(sizeHint) if err != nil { return nil, err } converter := w.cfg.converterFor(w.schemaType.Name(), w.val) return &basicMapAssembler{MapAssembler: mapAsm, builder: basicBuilder, parent: w, converter: converter}, nil case *schema.TypeStruct: val := w.createNonPtrVal() // _structAssembler walks through the fields in order as the entries are // assembled, verifyCompatibility() should mean it's safe to assume that // they match the schema, but we need to keep track of the fields that are // set in case of premature Finish() doneFields := make([]bool, val.NumField()) return &_structAssembler{ cfg: w.cfg, schemaType: typ, val: val, doneFields: doneFields, finish: w.finish, }, nil case *schema.TypeMap: // assume a struct{Keys []string, Values map[x]y} that we can fill with // _mapAssembler val := w.createNonPtrVal() keysVal := val.FieldByName("Keys") valuesVal := val.FieldByName("Values") if valuesVal.IsNil() { valuesVal.Set(reflect.MakeMap(valuesVal.Type())) } return &_mapAssembler{ cfg: w.cfg, schemaType: typ, keysVal: keysVal, valuesVal: valuesVal, finish: w.finish, }, nil case *schema.TypeUnion: // we can use _unionAssembler to assemble a union as if it were a map with // a single entry val := w.createNonPtrVal() return &_unionAssembler{ cfg: w.cfg, schemaType: typ, val: val, finish: w.finish, }, nil } return nil, datamodel.ErrWrongKind{ TypeName: w.schemaType.Name(), MethodName: "BeginMap", AppropriateKind: datamodel.KindSet_JustMap, ActualKind: actualKind(w.schemaType), } } // basicListAssembler is for assembling basicnode values, it's only use is for // Any fields that end up needing a BeginList() type basicListAssembler struct { datamodel.ListAssembler builder datamodel.NodeBuilder parent *_assembler converter *converter } func (w *basicListAssembler) Finish() error { if err := w.ListAssembler.Finish(); err != nil { return err } basicNode := w.builder.Build() if w.converter != nil { // we can assume an Any converter because basicListAssembler is only for Any // the user has registered the ability to convert a datamodel.Node to the // underlying Go type which may not be a datamodel.Node typ, err := w.converter.customFromAny(basicNode) if err != nil { return err } w.parent.createNonPtrVal().Set(matchSettable(typ, reflect.ValueOf(basicNode))) } else { w.parent.createNonPtrVal().Set(reflect.ValueOf(basicNode)) } if w.parent.finish != nil { if err := w.parent.finish(); err != nil { return err } } return nil } func (w *_assembler) BeginList(sizeHint int64) (datamodel.ListAssembler, error) { switch typ := w.schemaType.(type) { case *schema.TypeAny: basicBuilder := basicnode.Prototype.Any.NewBuilder() listAsm, err := basicBuilder.BeginList(sizeHint) if err != nil { return nil, err } converter := w.cfg.converterFor(w.schemaType.Name(), w.val) return &basicListAssembler{ListAssembler: listAsm, builder: basicBuilder, parent: w, converter: converter}, nil case *schema.TypeList: // we should be able to safely assume we're dealing with a Go slice here, // so _listAssembler can append to that val := w.createNonPtrVal() return &_listAssembler{ cfg: w.cfg, schemaType: typ, val: val, finish: w.finish, }, nil } return nil, datamodel.ErrWrongKind{ TypeName: w.schemaType.Name(), MethodName: "BeginList", AppropriateKind: datamodel.KindSet_JustList, ActualKind: actualKind(w.schemaType), } } func (w *_assembler) AssignNull() error { _, isAny := w.schemaType.(*schema.TypeAny) if customConverter := w.cfg.converterFor(w.schemaType.Name(), w.val); customConverter != nil && isAny { // an Any field that is being assigned a Null, we pass the Null directly to // the converter, regardless of whether this field is nullable or not typ, err := customConverter.customFromAny(datamodel.Null) if err != nil { return err } w.createNonPtrVal().Set(matchSettable(typ, w.val)) } else { if !w.nullable { return datamodel.ErrWrongKind{ TypeName: w.schemaType.Name(), MethodName: "AssignNull", // TODO } } // set the zero value for the underlying type as a stand-in for Null w.val.Set(reflect.Zero(w.val.Type())) } if w.finish != nil { if err := w.finish(); err != nil { return err } } return nil } func (w *_assembler) AssignBool(b bool) error { if err := compatibleKind(w.schemaType, datamodel.Kind_Bool); err != nil { return err } customConverter := w.cfg.converterFor(w.schemaType.Name(), w.val) _, isAny := w.schemaType.(*schema.TypeAny) if customConverter != nil { var typ interface{} var err error if isAny { // field is an Any, so the converter will be an Any converter that wants // a datamodel.Node to convert to whatever the underlying Go type is if typ, err = customConverter.customFromAny(basicnode.NewBool(b)); err != nil { return err } } else { // field is a Bool, but the user has registered a converter from a bool to // whatever the underlying Go type is if typ, err = customConverter.customFromBool(b); err != nil { return err } } w.createNonPtrVal().Set(matchSettable(typ, w.val)) } else { if isAny { // Any means the Go type must receive a datamodel.Node w.createNonPtrVal().Set(reflect.ValueOf(basicnode.NewBool(b))) } else { w.createNonPtrVal().SetBool(b) } } if w.finish != nil { if err := w.finish(); err != nil { return err } } return nil } func (w *_assembler) assignUInt(uin datamodel.UintNode) error { if err := compatibleKind(w.schemaType, datamodel.Kind_Int); err != nil { return err } _, isAny := w.schemaType.(*schema.TypeAny) // TODO: customConverter for uint?? if isAny { // Any means the Go type must receive a datamodel.Node w.createNonPtrVal().Set(reflect.ValueOf(uin)) } else { i, err := uin.AsUint() if err != nil { return err } if kindUint[w.val.Kind()] { w.createNonPtrVal().SetUint(i) } else { // TODO: check for overflow w.createNonPtrVal().SetInt(int64(i)) } } if w.finish != nil { if err := w.finish(); err != nil { return err } } return nil } func (w *_assembler) AssignInt(i int64) error { if err := compatibleKind(w.schemaType, datamodel.Kind_Int); err != nil { return err } // TODO: check for overflow customConverter := w.cfg.converterFor(w.schemaType.Name(), w.val) _, isAny := w.schemaType.(*schema.TypeAny) if customConverter != nil { var typ interface{} var err error if isAny { // field is an Any, so the converter will be an Any converter that wants // a datamodel.Node to convert to whatever the underlying Go type is if typ, err = customConverter.customFromAny(basicnode.NewInt(i)); err != nil { return err } } else { // field is an Int, but the user has registered a converter from an int to // whatever the underlying Go type is if typ, err = customConverter.customFromInt(i); err != nil { return err } } w.createNonPtrVal().Set(matchSettable(typ, w.val)) } else { if isAny { // Any means the Go type must receive a datamodel.Node w.createNonPtrVal().Set(reflect.ValueOf(basicnode.NewInt(i))) } else if kindUint[w.val.Kind()] { if i < 0 { // TODO: write a test return fmt.Errorf("bindnode: cannot assign negative integer to %s", w.val.Type()) } w.createNonPtrVal().SetUint(uint64(i)) } else { w.createNonPtrVal().SetInt(i) } } if w.finish != nil { if err := w.finish(); err != nil { return err } } return nil } func (w *_assembler) AssignFloat(f float64) error { if err := compatibleKind(w.schemaType, datamodel.Kind_Float); err != nil { return err } customConverter := w.cfg.converterFor(w.schemaType.Name(), w.val) _, isAny := w.schemaType.(*schema.TypeAny) if customConverter != nil { var typ interface{} var err error if isAny { // field is an Any, so the converter will be an Any converter that wants // a datamodel.Node to convert to whatever the underlying Go type is if typ, err = customConverter.customFromAny(basicnode.NewFloat(f)); err != nil { return err } } else { // field is a Float, but the user has registered a converter from a float // to whatever the underlying Go type is if typ, err = customConverter.customFromFloat(f); err != nil { return err } } w.createNonPtrVal().Set(matchSettable(typ, w.val)) } else { if isAny { // Any means the Go type must receive a datamodel.Node w.createNonPtrVal().Set(reflect.ValueOf(basicnode.NewFloat(f))) } else { w.createNonPtrVal().SetFloat(f) } } if w.finish != nil { if err := w.finish(); err != nil { return err } } return nil } func (w *_assembler) AssignString(s string) error { if err := compatibleKind(w.schemaType, datamodel.Kind_String); err != nil { return err } customConverter := w.cfg.converterFor(w.schemaType.Name(), w.val) _, isAny := w.schemaType.(*schema.TypeAny) if customConverter != nil { var typ interface{} var err error if isAny { // field is an Any, so the converter will be an Any converter that wants // a datamodel.Node to convert to whatever the underlying Go type is if typ, err = customConverter.customFromAny(basicnode.NewString(s)); err != nil { return err } } else { // field is a String, but the user has registered a converter from a // string to whatever the underlying Go type is if typ, err = customConverter.customFromString(s); err != nil { return err } } w.createNonPtrVal().Set(matchSettable(typ, w.val)) } else { if isAny { // Any means the Go type must receive a datamodel.Node w.createNonPtrVal().Set(reflect.ValueOf(basicnode.NewString(s))) } else { w.createNonPtrVal().SetString(s) } } if w.finish != nil { if err := w.finish(); err != nil { return err } } return nil } func (w *_assembler) AssignBytes(p []byte) error { if err := compatibleKind(w.schemaType, datamodel.Kind_Bytes); err != nil { return err } customConverter := w.cfg.converterFor(w.schemaType.Name(), w.val) _, isAny := w.schemaType.(*schema.TypeAny) if customConverter != nil { var typ interface{} var err error if isAny { // field is an Any, so the converter will be an Any converter that wants // a datamodel.Node to convert to whatever the underlying Go type is if typ, err = customConverter.customFromAny(basicnode.NewBytes(p)); err != nil { return err } } else { // field is a Bytes, but the user has registered a converter from a []byte // to whatever the underlying Go type is if typ, err = customConverter.customFromBytes(p); err != nil { return err } } w.createNonPtrVal().Set(matchSettable(typ, w.val)) } else { if isAny { // Any means the Go type must receive a datamodel.Node w.createNonPtrVal().Set(reflect.ValueOf(basicnode.NewBytes(p))) } else { w.createNonPtrVal().SetBytes(p) } } if w.finish != nil { if err := w.finish(); err != nil { return err } } return nil } func (w *_assembler) AssignLink(link datamodel.Link) error { val := w.createNonPtrVal() // TODO: newVal.Type() panics if link==nil; add a test and fix. customConverter := w.cfg.converterFor(w.schemaType.Name(), w.val) if _, ok := w.schemaType.(*schema.TypeAny); ok { if customConverter != nil { // field is an Any, so the converter will be an Any converter that wants // a datamodel.Node to convert to whatever the underlying Go type is typ, err := customConverter.customFromAny(basicnode.NewLink(link)) if err != nil { return err } w.createNonPtrVal().Set(matchSettable(typ, w.val)) } else { // Any means the Go type must receive a datamodel.Node val.Set(reflect.ValueOf(basicnode.NewLink(link))) } } else if customConverter != nil { if cl, ok := link.(cidlink.Link); ok { // field is a Link, but the user has registered a converter from a cid.Cid // to whatever the underlying Go type is typ, err := customConverter.customFromLink(cl.Cid) if err != nil { return err } w.createNonPtrVal().Set(matchSettable(typ, w.val)) } else { return fmt.Errorf("bindnode: custom converter can only receive a cidlink.Link through AssignLink") } } else if newVal := reflect.ValueOf(link); newVal.Type().AssignableTo(val.Type()) { // Directly assignable. val.Set(newVal) } else if newVal.Type() == goTypeCidLink && goTypeCid.AssignableTo(val.Type()) { // Unbox a cidlink.Link to assign to a go-cid.Cid value. newVal = newVal.FieldByName("Cid") val.Set(newVal) } else if actual := actualKind(w.schemaType); actual != datamodel.Kind_Link { // We're assigning a Link to a schema type that isn't a Link. return datamodel.ErrWrongKind{ TypeName: w.schemaType.Name(), MethodName: "AssignLink", AppropriateKind: datamodel.KindSet_JustLink, ActualKind: actualKind(w.schemaType), } } else { // The schema type is a Link, but we somehow can't assign to the Go value. // Almost certainly a bug; we should have verified for compatibility upfront. return fmt.Errorf("bindnode bug: AssignLink with %s argument can't be used on Go type %s", newVal.Type(), val.Type()) } if w.finish != nil { if err := w.finish(); err != nil { return err } } return nil } func (w *_assembler) AssignNode(node datamodel.Node) error { // TODO: does this ever trigger? // newVal := reflect.ValueOf(node) // if newVal.Type().AssignableTo(w.val.Type()) { // w.val.Set(newVal) // return nil // } if uintNode, ok := node.(datamodel.UintNode); ok { return w.assignUInt(uintNode) } return datamodel.Copy(node, w) } func (w *_assembler) Prototype() datamodel.NodePrototype { return &_prototype{cfg: w.cfg, schemaType: w.schemaType, goType: w.val.Type()} } // _structAssembler is used for Struct assembling via BeginMap() type _structAssembler struct { // TODO: embed _assembler? cfg *config schemaType *schema.TypeStruct val reflect.Value // non-pointer finish func() error // TODO: more state checks // TODO: Consider if we could do this in a cheaper way, // such as looking at the reflect.Value directly. // If not, at least avoid an extra alloc. doneFields []bool // TODO: optimize for structs curKey _assembler nextIndex int // only used by repr.go } func (w *_structAssembler) AssembleKey() datamodel.NodeAssembler { w.curKey = _assembler{ cfg: w.cfg, schemaType: schemaTypeString, val: reflect.New(goTypeString).Elem(), } return &w.curKey } func (w *_structAssembler) AssembleValue() datamodel.NodeAssembler { // TODO: optimize this to do one lookup by name name := w.curKey.val.String() field := w.schemaType.Field(name) if field == nil { // TODO: should've been raised when the key was submitted instead. // TODO: should make well-typed errors for this. return _errorAssembler{fmt.Errorf("bindnode TODO: invalid key: %q is not a field in type %s", name, w.schemaType.Name())} // panic(schema.ErrInvalidKey{ // TypeName: w.schemaType.Name(), // Key: basicnode.NewString(name), // }) } ftyp, ok := w.val.Type().FieldByName(fieldNameFromSchema(name)) if !ok { // It is unfortunate this is not detected proactively earlier during bind. return _errorAssembler{fmt.Errorf("schema type %q has field %q, we expect go struct to have field %q", w.schemaType.Name(), field.Name(), fieldNameFromSchema(name))} } if len(ftyp.Index) > 1 { return _errorAssembler{fmt.Errorf("bindnode TODO: embedded fields")} } w.doneFields[ftyp.Index[0]] = true fval := w.val.FieldByIndex(ftyp.Index) if field.IsOptional() { if fval.Kind() == reflect.Ptr { // ptrVal = new(T); val = *ptrVal fval.Set(reflect.New(fval.Type().Elem())) fval = fval.Elem() } else { // val = *new(T) fval.Set(reflect.New(fval.Type()).Elem()) } } // TODO: reuse same assembler for perf? return &_assembler{ cfg: w.cfg, schemaType: field.Type(), val: fval, nullable: field.IsNullable(), } } func (w *_structAssembler) AssembleEntry(k string) (datamodel.NodeAssembler, error) { if err := w.AssembleKey().AssignString(k); err != nil { return nil, err } am := w.AssembleValue() return am, nil } func (w *_structAssembler) Finish() error { fields := w.schemaType.Fields() var missing []string for i, field := range fields { if !field.IsOptional() && !w.doneFields[i] { missing = append(missing, field.Name()) } } if len(missing) > 0 { return schema.ErrMissingRequiredField{Missing: missing} } if w.finish != nil { if err := w.finish(); err != nil { return err } } return nil } func (w *_structAssembler) KeyPrototype() datamodel.NodePrototype { // TODO: if the user provided their own schema with their own typesystem, // the schemaTypeString here may be using the wrong typesystem. return &_prototype{cfg: w.cfg, schemaType: schemaTypeString, goType: goTypeString} } func (w *_structAssembler) ValuePrototype(k string) datamodel.NodePrototype { panic("bindnode TODO: struct ValuePrototype") } type _errorAssembler struct { err error } func (w _errorAssembler) BeginMap(int64) (datamodel.MapAssembler, error) { return nil, w.err } func (w _errorAssembler) BeginList(int64) (datamodel.ListAssembler, error) { return nil, w.err } func (w _errorAssembler) AssignNull() error { return w.err } func (w _errorAssembler) AssignBool(bool) error { return w.err } func (w _errorAssembler) AssignInt(int64) error { return w.err } func (w _errorAssembler) AssignFloat(float64) error { return w.err } func (w _errorAssembler) AssignString(string) error { return w.err } func (w _errorAssembler) AssignBytes([]byte) error { return w.err } func (w _errorAssembler) AssignLink(datamodel.Link) error { return w.err } func (w _errorAssembler) AssignNode(datamodel.Node) error { return w.err } func (w _errorAssembler) Prototype() datamodel.NodePrototype { return nil } // used for Maps which we can assume are of type: struct{Keys []string, Values map[x]y}, // where we have Keys in keysVal and Values in valuesVal type _mapAssembler struct { cfg *config schemaType *schema.TypeMap keysVal reflect.Value // non-pointer valuesVal reflect.Value // non-pointer finish func() error // TODO: more state checks curKey _assembler } func (w *_mapAssembler) AssembleKey() datamodel.NodeAssembler { w.curKey = _assembler{ cfg: w.cfg, schemaType: w.schemaType.KeyType(), val: reflect.New(w.valuesVal.Type().Key()).Elem(), } return &w.curKey } func (w *_mapAssembler) AssembleValue() datamodel.NodeAssembler { kval := w.curKey.val val := reflect.New(w.valuesVal.Type().Elem()).Elem() finish := func() error { // TODO: check for duplicates in keysVal w.keysVal.Set(reflect.Append(w.keysVal, kval)) w.valuesVal.SetMapIndex(kval, val) return nil } return &_assembler{ cfg: w.cfg, schemaType: w.schemaType.ValueType(), val: val, nullable: w.schemaType.ValueIsNullable(), finish: finish, } } func (w *_mapAssembler) AssembleEntry(k string) (datamodel.NodeAssembler, error) { if err := w.AssembleKey().AssignString(k); err != nil { return nil, err } am := w.AssembleValue() return am, nil } func (w *_mapAssembler) Finish() error { if w.finish != nil { if err := w.finish(); err != nil { return err } } return nil } func (w *_mapAssembler) KeyPrototype() datamodel.NodePrototype { return &_prototype{cfg: w.cfg, schemaType: w.schemaType.KeyType(), goType: w.valuesVal.Type().Key()} } func (w *_mapAssembler) ValuePrototype(k string) datamodel.NodePrototype { return &_prototype{cfg: w.cfg, schemaType: w.schemaType.ValueType(), goType: w.valuesVal.Type().Elem()} } // _listAssembler is for operating directly on slices, which we have in val type _listAssembler struct { cfg *config schemaType *schema.TypeList val reflect.Value // non-pointer finish func() error } func (w *_listAssembler) AssembleValue() datamodel.NodeAssembler { goType := w.val.Type().Elem() // TODO: use a finish func to append w.val.Set(reflect.Append(w.val, reflect.New(goType).Elem())) return &_assembler{ cfg: w.cfg, schemaType: w.schemaType.ValueType(), val: w.val.Index(w.val.Len() - 1), nullable: w.schemaType.ValueIsNullable(), } } func (w *_listAssembler) Finish() error { if w.finish != nil { if err := w.finish(); err != nil { return err } } return nil } func (w *_listAssembler) ValuePrototype(idx int64) datamodel.NodePrototype { return &_prototype{cfg: w.cfg, schemaType: w.schemaType.ValueType(), goType: w.val.Type().Elem()} } // when assembling as a Map but we anticipate a single value, which we need to // look up in the union members type _unionAssembler struct { cfg *config schemaType *schema.TypeUnion val reflect.Value // non-pointer finish func() error // TODO: more state checks curKey _assembler } func (w *_unionAssembler) AssembleKey() datamodel.NodeAssembler { w.curKey = _assembler{ cfg: w.cfg, schemaType: schemaTypeString, val: reflect.New(goTypeString).Elem(), } return &w.curKey } func (w *_unionAssembler) AssembleValue() datamodel.NodeAssembler { name := w.curKey.val.String() var idx int var mtyp schema.Type for i, member := range w.schemaType.Members() { if member.Name() == name { idx = i mtyp = member break } } if mtyp == nil { return _errorAssembler{ schema.ErrNotUnionStructure{ TypeName: w.schemaType.Name(), Detail: fmt.Sprintf("no member named %q", name), }, } } goType := w.val.Field(idx).Type().Elem() valPtr := reflect.New(goType) finish := func() error { unionSetMember(w.val, idx, valPtr) return nil } return &_assembler{ cfg: w.cfg, schemaType: mtyp, val: valPtr.Elem(), finish: finish, } } func (w *_unionAssembler) AssembleEntry(k string) (datamodel.NodeAssembler, error) { if err := w.AssembleKey().AssignString(k); err != nil { return nil, err } am := w.AssembleValue() return am, nil } func (w *_unionAssembler) Finish() error { // TODO(rvagg): I think this might allow setting multiple members of the union // we need a test for this. haveIdx, _ := unionMember(w.val) if haveIdx < 0 { return schema.ErrNotUnionStructure{TypeName: w.schemaType.Name(), Detail: "a union must have exactly one entry"} } if w.finish != nil { if err := w.finish(); err != nil { return err } } return nil } func (w *_unionAssembler) KeyPrototype() datamodel.NodePrototype { return &_prototype{cfg: w.cfg, schemaType: schemaTypeString, goType: goTypeString} } func (w *_unionAssembler) ValuePrototype(k string) datamodel.NodePrototype { panic("bindnode TODO: union ValuePrototype") } // _structIterator is for iterating over Struct types which operate over Go // structs. The iteration order is dictated by Go field declaration order which // should match the schema for this type. type _structIterator struct { // TODO: support embedded fields? cfg *config schemaType *schema.TypeStruct fields []schema.StructField val reflect.Value // non-pointer nextIndex int // these are only used in repr.go reprEnd int } func (w *_structIterator) Next() (key, value datamodel.Node, _ error) { if w.Done() { return nil, nil, datamodel.ErrIteratorOverread{} } field := w.fields[w.nextIndex] val := w.val.Field(w.nextIndex) w.nextIndex++ key = basicnode.NewString(field.Name()) if field.IsOptional() { if val.IsNil() { return key, datamodel.Absent, nil } if val.Kind() == reflect.Ptr { val = val.Elem() } } _, isAny := field.Type().(*schema.TypeAny) if isAny { if customConverter := w.cfg.converterFor(field.Type().Name(), val); customConverter != nil { // field is an Any and we have an Any converter which takes the underlying // struct field value and returns a datamodel.Node v, err := customConverter.customToAny(ptrVal(val).Interface()) if err != nil { return nil, nil, err } return key, v, nil } } if field.IsNullable() { if val.IsNil() { return key, datamodel.Null, nil } if val.Kind() == reflect.Ptr { val = val.Elem() } } if isAny { // field holds a datamodel.Node return key, nonPtrVal(val).Interface().(datamodel.Node), nil } return key, newNode(w.cfg, field.Type(), val), nil } func (w *_structIterator) Done() bool { return w.nextIndex >= len(w.fields) } // _mapIterator is for iterating over a struct{Keys []string, Values map[x]y}, // where we have the Keys in keysVal and Values in valuesVal type _mapIterator struct { cfg *config schemaType *schema.TypeMap keysVal reflect.Value // non-pointer valuesVal reflect.Value // non-pointer nextIndex int } func (w *_mapIterator) Next() (key, value datamodel.Node, _ error) { if w.Done() { return nil, nil, datamodel.ErrIteratorOverread{} } goKey := w.keysVal.Index(w.nextIndex) val := w.valuesVal.MapIndex(goKey) w.nextIndex++ key = newNode(w.cfg, w.schemaType.KeyType(), goKey) _, isAny := w.schemaType.ValueType().(*schema.TypeAny) if isAny { if customConverter := w.cfg.converterFor(w.schemaType.ValueType().Name(), val); customConverter != nil { // values of this map are Any and we have an Any converter which takes the // underlying map value and returns a datamodel.Node // TODO(rvagg): can't call ptrVal on a map value that's not a pointer // so only map[string]*foo will work for the Values map and an Any // converter. Should we check in infer.go? val, err := customConverter.customToAny(ptrVal(val).Interface()) return key, val, err } } if w.schemaType.ValueIsNullable() { if val.IsNil() { return key, datamodel.Null, nil } val = val.Elem() // nullable entries are pointers } if isAny { // Values holds datamodel.Nodes return key, nonPtrVal(val).Interface().(datamodel.Node), nil } return key, newNode(w.cfg, w.schemaType.ValueType(), val), nil } func (w *_mapIterator) Done() bool { return w.nextIndex >= w.keysVal.Len() } // _listIterator is for iterating over slices, which is held in val type _listIterator struct { cfg *config schemaType *schema.TypeList val reflect.Value // non-pointer nextIndex int } func (w *_listIterator) Next() (index int64, value datamodel.Node, _ error) { if w.Done() { return 0, nil, datamodel.ErrIteratorOverread{} } idx := int64(w.nextIndex) val := w.val.Index(w.nextIndex) w.nextIndex++ if w.schemaType.ValueIsNullable() { if val.IsNil() { return idx, datamodel.Null, nil } val = val.Elem() // nullable values are pointers } if _, ok := w.schemaType.ValueType().(*schema.TypeAny); ok { if customConverter := w.cfg.converterFor(w.schemaType.ValueType().Name(), val); customConverter != nil { // values are Any and we have an Any converter which can take whatever // the underlying Go type in this slice is and return a datamodel.Node val, err := customConverter.customToAny(ptrVal(val).Interface()) return idx, val, err } // values are Any, assume that they are datamodel.Nodes return idx, nonPtrVal(val).Interface().(datamodel.Node), nil } return idx, newNode(w.cfg, w.schemaType.ValueType(), val), nil } func (w *_listIterator) Done() bool { return w.nextIndex >= w.val.Len() } type _unionIterator struct { // TODO: support embedded fields? cfg *config schemaType *schema.TypeUnion members []schema.Type val reflect.Value // non-pointer done bool } func (w *_unionIterator) Next() (key, value datamodel.Node, _ error) { // we can only call this once for a union since a union can only have one // entry even though it behaves like a Map if w.Done() { return nil, nil, datamodel.ErrIteratorOverread{} } w.done = true haveIdx, mval := unionMember(w.val) if haveIdx < 0 { return nil, nil, fmt.Errorf("bindnode: union %s has no member", w.val.Type()) } mtyp := w.members[haveIdx] node := newNode(w.cfg, mtyp, mval) key = basicnode.NewString(mtyp.Name()) return key, node, nil } func (w *_unionIterator) Done() bool { return w.done } // --- uint64 special case handling type _uintNode struct { cfg *config schemaType schema.Type val reflect.Value // non-pointer } func (tu *_uintNode) Type() schema.Type { return tu.schemaType } func (tu *_uintNode) Representation() datamodel.Node { return (*_uintNodeRepr)(tu) } func (_uintNode) Kind() datamodel.Kind { return datamodel.Kind_Int } func (_uintNode) LookupByString(string) (datamodel.Node, error) { return mixins.Int{TypeName: "int"}.LookupByString("") } func (_uintNode) LookupByNode(key datamodel.Node) (datamodel.Node, error) { return mixins.Int{TypeName: "int"}.LookupByNode(nil) } func (_uintNode) LookupByIndex(idx int64) (datamodel.Node, error) { return mixins.Int{TypeName: "int"}.LookupByIndex(0) } func (_uintNode) LookupBySegment(seg datamodel.PathSegment) (datamodel.Node, error) { return mixins.Int{TypeName: "int"}.LookupBySegment(seg) } func (_uintNode) MapIterator() datamodel.MapIterator { return nil } func (_uintNode) ListIterator() datamodel.ListIterator { return nil } func (_uintNode) Length() int64 { return -1 } func (_uintNode) IsAbsent() bool { return false } func (_uintNode) IsNull() bool { return false } func (_uintNode) AsBool() (bool, error) { return mixins.Int{TypeName: "int"}.AsBool() } func (tu *_uintNode) AsInt() (int64, error) { return (*_uintNodeRepr)(tu).AsInt() } func (tu *_uintNode) AsUint() (uint64, error) { return (*_uintNodeRepr)(tu).AsUint() } func (_uintNode) AsFloat() (float64, error) { return mixins.Int{TypeName: "int"}.AsFloat() } func (_uintNode) AsString() (string, error) { return mixins.Int{TypeName: "int"}.AsString() } func (_uintNode) AsBytes() ([]byte, error) { return mixins.Int{TypeName: "int"}.AsBytes() } func (_uintNode) AsLink() (datamodel.Link, error) { return mixins.Int{TypeName: "int"}.AsLink() } func (_uintNode) Prototype() datamodel.NodePrototype { return basicnode.Prototype__Int{} } // we need this for _uintNode#Representation() so we don't return a TypeNode type _uintNodeRepr _uintNode func (_uintNodeRepr) Kind() datamodel.Kind { return datamodel.Kind_Int } func (_uintNodeRepr) LookupByString(string) (datamodel.Node, error) { return mixins.Int{TypeName: "int"}.LookupByString("") } func (_uintNodeRepr) LookupByNode(key datamodel.Node) (datamodel.Node, error) { return mixins.Int{TypeName: "int"}.LookupByNode(nil) } func (_uintNodeRepr) LookupByIndex(idx int64) (datamodel.Node, error) { return mixins.Int{TypeName: "int"}.LookupByIndex(0) } func (_uintNodeRepr) LookupBySegment(seg datamodel.PathSegment) (datamodel.Node, error) { return mixins.Int{TypeName: "int"}.LookupBySegment(seg) } func (_uintNodeRepr) MapIterator() datamodel.MapIterator { return nil } func (_uintNodeRepr) ListIterator() datamodel.ListIterator { return nil } func (_uintNodeRepr) Length() int64 { return -1 } func (_uintNodeRepr) IsAbsent() bool { return false } func (_uintNodeRepr) IsNull() bool { return false } func (_uintNodeRepr) AsBool() (bool, error) { return mixins.Int{TypeName: "int"}.AsBool() } func (tu *_uintNodeRepr) AsInt() (int64, error) { if err := compatibleKind(tu.schemaType, datamodel.Kind_Int); err != nil { return 0, err } if customConverter := tu.cfg.converterFor(tu.schemaType.Name(), tu.val); customConverter != nil { // user has registered a converter that takes the underlying type and returns an int return customConverter.customToInt(ptrVal(tu.val).Interface()) } val := nonPtrVal(tu.val) // we can assume it's a uint64 at this point u := val.Uint() if u > math.MaxInt64 { return 0, fmt.Errorf("bindnode: integer overflow, %d is too large for an int64", u) } return int64(u), nil } func (tu *_uintNodeRepr) AsUint() (uint64, error) { if err := compatibleKind(tu.schemaType, datamodel.Kind_Int); err != nil { return 0, err } // TODO(rvagg): do we want a converter option for uint values? do we combine it // with int converters? // we can assume it's a uint64 at this point return nonPtrVal(tu.val).Uint(), nil } func (_uintNodeRepr) AsFloat() (float64, error) { return mixins.Int{TypeName: "int"}.AsFloat() } func (_uintNodeRepr) AsString() (string, error) { return mixins.Int{TypeName: "int"}.AsString() } func (_uintNodeRepr) AsBytes() ([]byte, error) { return mixins.Int{TypeName: "int"}.AsBytes() } func (_uintNodeRepr) AsLink() (datamodel.Link, error) { return mixins.Int{TypeName: "int"}.AsLink() } func (_uintNodeRepr) Prototype() datamodel.NodePrototype { return basicnode.Prototype__Int{} } ================================================ FILE: node/bindnode/registry/registry.go ================================================ package registry import ( "errors" "fmt" "io" "reflect" "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/codec" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/node/bindnode" "github.com/ipld/go-ipld-prime/schema" ) type prototypeData struct { proto schema.TypedPrototype options []bindnode.Option } // BindnodeRegistry holds TypedPrototype and bindnode options for Go types and // will use that data for conversion operations. type BindnodeRegistry map[reflect.Type]prototypeData // NewRegistry creates a new BindnodeRegistry func NewRegistry() BindnodeRegistry { return make(BindnodeRegistry) } func typeOf(ptrValue interface{}) reflect.Type { val := reflect.ValueOf(ptrValue).Type() for val.Kind() == reflect.Ptr { val = val.Elem() } return val } func (br BindnodeRegistry) prototypeDataFor(ptrType interface{}) prototypeData { typ := typeOf(ptrType) proto, ok := br[typ] if !ok { panic(fmt.Sprintf("bindnode utils: type has not been registered: %s", typ.Name())) } return proto } // RegisterType registers ptrType with schema such that it can be wrapped and // unwrapped without needing the schema, Type, or TypedPrototype. // Typically the typeName will match the Go type name, but it can be whatever // is defined in the schema for the type being registered. // Registering the same type twice on this registry will cause an error. // This call may also error if the schema is invalid or the type doesn't match // the schema. Additionally, panics from within bindnode's initial prototype // checks will be captured and returned as errors from this function. func (br BindnodeRegistry) RegisterType(ptrType interface{}, schema string, typeName string, options ...bindnode.Option) (err error) { typ := typeOf(ptrType) if _, ok := br[typ]; ok { return fmt.Errorf("bindnode utils: type already registered: %s", typ.Name()) } typeSystem, err := ipld.LoadSchemaBytes([]byte(schema)) if err != nil { return fmt.Errorf("bindnode utils: failed to load schema: %s", err.Error()) } schemaType := typeSystem.TypeByName(typeName) if schemaType == nil { return fmt.Errorf("bindnode utils: schema for [%T] does not contain that named type [%s]", ptrType, typ.Name()) } // focusing on bindnode setup panics defer func() { if rec := recover(); rec != nil { switch v := rec.(type) { case string: err = errors.New(v) case error: err = v default: panic(rec) } } }() proto := bindnode.Prototype(ptrType, schemaType, options...) br[typ] = prototypeData{ proto, options, } return err } // IsRegistered can be used to determine if the type has already been registered // within this registry. // Using RegisterType on an already registered type will cause a panic, so where // this may be the case, IsRegistered can be used to check. func (br BindnodeRegistry) IsRegistered(ptrType interface{}) bool { _, ok := br[typeOf(ptrType)] return ok } // TypeFromReader deserializes bytes using the given codec from a Reader and // instantiates the Go type that's provided as a pointer via the ptrValue // argument. func (br BindnodeRegistry) TypeFromReader(r io.Reader, ptrValue interface{}, decoder codec.Decoder) (interface{}, error) { protoData := br.prototypeDataFor(ptrValue) node, err := ipld.DecodeStreamingUsingPrototype(r, decoder, protoData.proto) if err != nil { return nil, err } typ := bindnode.Unwrap(node) return typ, nil } // TypeFromBytes deserializes bytes using the given codec from its bytes and // instantiates the Go type that's provided as a pointer via the ptrValue // argument. func (br BindnodeRegistry) TypeFromBytes(byts []byte, ptrValue interface{}, decoder codec.Decoder) (interface{}, error) { protoData := br.prototypeDataFor(ptrValue) node, err := ipld.DecodeUsingPrototype(byts, decoder, protoData.proto) if err != nil { return nil, err } typ := bindnode.Unwrap(node) return typ, nil } // TypeFromNode converts an datamodel.Node into an appropriate Go type that's // provided as a pointer via the ptrValue argument. func (br BindnodeRegistry) TypeFromNode(node datamodel.Node, ptrValue interface{}) (interface{}, error) { protoData := br.prototypeDataFor(ptrValue) if tn, ok := node.(schema.TypedNode); ok { node = tn.Representation() } builder := protoData.proto.Representation().NewBuilder() err := builder.AssignNode(node) if err != nil { return nil, err } typ := bindnode.Unwrap(builder.Build()) return typ, nil } // TypeToNode converts a Go type that's provided as a pointer via the ptrValue // argument to an schema.TypedNode. func (br BindnodeRegistry) TypeToNode(ptrValue interface{}) schema.TypedNode { protoData := br.prototypeDataFor(ptrValue) return bindnode.Wrap(ptrValue, protoData.proto.Type(), protoData.options...) } // TypeToWriter is a utility method that serializes a Go type that's provided as // a pointer via the ptrValue argument through the given codec to a Writer. func (br BindnodeRegistry) TypeToWriter(ptrValue interface{}, w io.Writer, encoder codec.Encoder) error { return ipld.EncodeStreaming(w, br.TypeToNode(ptrValue), encoder) } // TypeToBytes is a utility method that serializes a Go type that's provided as // a pointer via the ptrValue argument through the given codec and returns the // bytes. func (br BindnodeRegistry) TypeToBytes(ptrValue interface{}, encoder codec.Encoder) ([]byte, error) { return ipld.Encode(br.TypeToNode(ptrValue), encoder) } ================================================ FILE: node/bindnode/registry/registry_test.go ================================================ package registry_test import ( "bytes" "encoding/hex" "math" "testing" "github.com/ipld/go-ipld-prime/codec/dagcbor" "github.com/ipld/go-ipld-prime/codec/dagjson" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/node/bindnode" "github.com/ipld/go-ipld-prime/node/bindnode/registry" qt "github.com/frankban/quicktest" ) type HexString string type Foo struct { Int int Bool bool } func TestRegistry(t *testing.T) { reg := registry.NewRegistry() qt.Assert(t, reg.IsRegistered((*Foo)(nil)), qt.IsFalse) qt.Assert(t, reg.IsRegistered((*HexString)(nil)), qt.IsFalse) err := reg.RegisterType((*Foo)(nil), `type Foo struct { Int Int Bool Bool }`, "Foo") qt.Assert(t, err, qt.IsNil) err = reg.RegisterType((*HexString)(nil), "type HS bytes", "HS", bindnode.TypedBytesConverter( (*HexString)(nil), func(b []byte) (interface{}, error) { return HexString(hex.EncodeToString(b)), nil }, func(i interface{}) ([]byte, error) { s, _ := i.(*HexString) return hex.DecodeString(string(*s)) })) qt.Assert(t, err, qt.IsNil) qt.Assert(t, reg.IsRegistered((*Foo)(nil)), qt.IsTrue) qt.Assert(t, reg.IsRegistered((*HexString)(nil)), qt.IsTrue) hsi, err := reg.TypeFromNode(basicnode.NewBytes([]byte{0, 1, 2, 3, 4}), (*HexString)(nil)) qt.Assert(t, err, qt.IsNil) hs, ok := hsi.(*HexString) qt.Assert(t, ok, qt.IsTrue) qt.Assert(t, string(*hs), qt.Equals, "0001020304") byts, _ := hex.DecodeString("a263496e74386364426f6f6cf4") fooi, err := reg.TypeFromBytes(byts, (*Foo)(nil), dagcbor.Decode) qt.Assert(t, err, qt.IsNil) foo, ok := fooi.(*Foo) qt.Assert(t, ok, qt.IsTrue) qt.Assert(t, *foo, qt.Equals, Foo{Int: -100, Bool: false}) byts, err = reg.TypeToBytes(&Foo{Int: -100, Bool: false}, dagjson.Encode) qt.Assert(t, err, qt.IsNil) qt.Assert(t, string(byts), qt.Equals, `{"Bool":false,"Int":-100}`) byts, _ = hex.DecodeString("a263496e741a7fffffff64426f6f6cf5") fooi, err = reg.TypeFromReader(bytes.NewReader(byts), (*Foo)(nil), dagcbor.Decode) qt.Assert(t, err, qt.IsNil) foo, ok = fooi.(*Foo) qt.Assert(t, ok, qt.IsTrue) qt.Assert(t, *foo, qt.Equals, Foo{Int: math.MaxInt32, Bool: true}) w := bytes.Buffer{} err = reg.TypeToWriter(&Foo{Int: math.MaxInt32, Bool: true}, &w, dagjson.Encode) qt.Assert(t, err, qt.IsNil) qt.Assert(t, w.String(), qt.Equals, `{"Bool":true,"Int":2147483647}`) } func TestRegistryErrors(t *testing.T) { reg := registry.NewRegistry() err := reg.RegisterType((*Foo)(nil), `type Nope nope {}`, "Foo") qt.Assert(t, err, qt.ErrorMatches, `.*unknown type keyword: "nope".*`) err = reg.RegisterType((*HexString)(nil), "type HS string", "HS") qt.Assert(t, err, qt.IsNil) err = reg.RegisterType((*HexString)(nil), "type HS2 string", "HS2") qt.Assert(t, err, qt.ErrorMatches, `.*type already registered: HexString`) err = reg.RegisterType((*Foo)(nil), "type NotFoo string", "Foo") qt.Assert(t, err, qt.ErrorMatches, `.*does not contain that named type.*`) err = reg.RegisterType((*Foo)(nil), `type Foo struct { NotInt String NotBool Float }`, "Foo") qt.Assert(t, err, qt.ErrorMatches, `.*kind mismatch.*`) } ================================================ FILE: node/bindnode/repr.go ================================================ package bindnode import ( "fmt" "reflect" "strings" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/schema" ) func reprNode(node datamodel.Node) datamodel.Node { if node, ok := node.(schema.TypedNode); ok { return node.Representation() } // datamodel.Absent and datamodel.Null are not typed. // TODO: is this a problem? surely a typed struct's fields are always // typed, even when absent or null. return node } func reprStrategy(typ schema.Type) interface{} { // Can't use an interface check, as each method has a different result type. // TODO: consider inlining this type switch at each call site, // as the call sites need the underlying schema.Type too. switch typ := typ.(type) { case *schema.TypeStruct: return typ.RepresentationStrategy() case *schema.TypeUnion: return typ.RepresentationStrategy() case *schema.TypeEnum: return typ.RepresentationStrategy() } return nil } type _prototypeRepr _prototype func (w *_prototypeRepr) NewBuilder() datamodel.NodeBuilder { return &_builderRepr{_assemblerRepr{ cfg: w.cfg, schemaType: w.schemaType, val: reflect.New(w.goType).Elem(), }} } type _nodeRepr _node func (w *_nodeRepr) Kind() datamodel.Kind { switch reprStrategy(w.schemaType).(type) { case schema.StructRepresentation_Stringjoin: return datamodel.Kind_String case schema.StructRepresentation_Map: return datamodel.Kind_Map case schema.StructRepresentation_Tuple, schema.StructRepresentation_ListPairs: return datamodel.Kind_List case schema.UnionRepresentation_Keyed: return datamodel.Kind_Map case schema.UnionRepresentation_Kinded: haveIdx, _ := unionMember(w.val) if haveIdx < 0 { panic(fmt.Sprintf("bindnode: kinded union %s has no member", w.val.Type())) } mtyp := w.schemaType.(*schema.TypeUnion).Members()[haveIdx] return mtyp.RepresentationBehavior() case schema.UnionRepresentation_Stringprefix: return datamodel.Kind_String case schema.EnumRepresentation_Int: return datamodel.Kind_Int case schema.EnumRepresentation_String: return datamodel.Kind_String default: return (*_node)(w).Kind() } } func outboundMappedKey(stg schema.StructRepresentation_Map, key string) string { // TODO: why doesn't stg just allow us to "get" by the key string? field := schema.SpawnStructField(key, "", false, false) mappedKey := stg.GetFieldKey(field) return mappedKey } func inboundMappedKey(typ *schema.TypeStruct, stg schema.StructRepresentation_Map, key string) string { // TODO: can't do a "reverse" lookup... needs better API probably. fields := typ.Fields() for _, field := range fields { mappedKey := stg.GetFieldKey(field) if key == mappedKey { return field.Name() } } return key // fallback to the same key } func outboundMappedType(stg schema.UnionRepresentation_Keyed, key string) string { // TODO: why doesn't stg just allow us to "get" by the key string? typ := schema.SpawnBool(key) mappedKey := stg.GetDiscriminant(typ) return mappedKey } func inboundMappedType(typ *schema.TypeUnion, stg schema.UnionRepresentation_Keyed, key string) string { // TODO: can't do a "reverse" lookup... needs better API probably. for _, member := range typ.Members() { mappedKey := stg.GetDiscriminant(member) if key == mappedKey { // println(key, "rev-mapped to", field.Name()) return member.Name() } } // println(key, "had no mapping") return key // fallback to the same key } // asKinded can be called on a kinded union node to obtain a node // representing one of its members, identified by kind. func (w *_nodeRepr) asKinded(stg schema.UnionRepresentation_Kinded, kind datamodel.Kind) *_nodeRepr { name := stg.GetMember(kind) members := w.schemaType.(*schema.TypeUnion).Members() for i, member := range members { if member.Name() != name { continue } w2 := *w w2.val = w.val.Field(i).Elem() w2.schemaType = member return &w2 } panic("bindnode TODO: GetMember result is missing?") } func (w *_nodeRepr) LookupByString(key string) (datamodel.Node, error) { if stg, ok := reprStrategy(w.schemaType).(schema.UnionRepresentation_Kinded); ok { w = w.asKinded(stg, datamodel.Kind_Map) } switch stg := reprStrategy(w.schemaType).(type) { case schema.StructRepresentation_Map: revKey := inboundMappedKey(w.schemaType.(*schema.TypeStruct), stg, key) v, err := (*_node)(w).LookupByString(revKey) if err != nil { return nil, err } return reprNode(v), nil case schema.UnionRepresentation_Keyed: revKey := inboundMappedType(w.schemaType.(*schema.TypeUnion), stg, key) v, err := (*_node)(w).LookupByString(revKey) if err != nil { return nil, err } return reprNode(v), nil default: v, err := (*_node)(w).LookupByString(key) if err != nil { return nil, err } return reprNode(v), nil } } func (w *_nodeRepr) LookupByIndex(idx int64) (datamodel.Node, error) { switch stg := reprStrategy(w.schemaType).(type) { case schema.UnionRepresentation_Kinded: return w.asKinded(stg, datamodel.Kind_List).LookupByIndex(idx) case schema.StructRepresentation_Tuple: fields := w.schemaType.(*schema.TypeStruct).Fields() if idx < 0 || int(idx) >= len(fields) { return nil, datamodel.ErrNotExists{Segment: datamodel.PathSegmentOfInt(idx)} } field := fields[idx] v, err := (*_node)(w).LookupByString(field.Name()) if err != nil { return nil, err } return reprNode(v), nil case schema.StructRepresentation_ListPairs: fields := w.schemaType.(*schema.TypeStruct).Fields() if idx < 0 || int(idx) >= len(fields) { return nil, datamodel.ErrNotExists{Segment: datamodel.PathSegmentOfInt(idx)} } var curField int64 for _, field := range fields { value, err := (*_node)(w).LookupByString(field.Name()) if err != nil { return nil, err } if value.IsAbsent() { continue } if curField == idx { return buildListpairsField(basicnode.NewString(field.Name()), reprNode(value)) } curField++ } return nil, datamodel.ErrNotExists{Segment: datamodel.PathSegmentOfInt(idx)} default: v, err := (*_node)(w).LookupByIndex(idx) if err != nil { return nil, err } return reprNode(v), nil } } func (w *_nodeRepr) LookupBySegment(seg datamodel.PathSegment) (datamodel.Node, error) { switch w.Kind() { case datamodel.Kind_Map: return w.LookupByString(seg.String()) case datamodel.Kind_List: idx, err := seg.Index() if err != nil { return nil, err } return w.LookupByIndex(idx) } return nil, datamodel.ErrWrongKind{ TypeName: w.schemaType.Name(), MethodName: "LookupBySegment", AppropriateKind: datamodel.KindSet_Recursive, ActualKind: w.Kind(), } } func (w *_nodeRepr) LookupByNode(key datamodel.Node) (datamodel.Node, error) { switch w.Kind() { case datamodel.Kind_Map: s, err := key.AsString() if err != nil { return nil, err } return w.LookupByString(s) case datamodel.Kind_List: i, err := key.AsInt() if err != nil { return nil, err } return w.LookupByIndex(i) } return nil, datamodel.ErrWrongKind{ TypeName: w.schemaType.Name(), MethodName: "LookupByNode", AppropriateKind: datamodel.KindSet_Recursive, ActualKind: w.Kind(), } } func (w *_nodeRepr) MapIterator() datamodel.MapIterator { // TODO: we can try to reuse reprStrategy here and elsewhere if stg, ok := reprStrategy(w.schemaType).(schema.UnionRepresentation_Kinded); ok { w = w.asKinded(stg, datamodel.Kind_Map) } switch reprStrategy(w.schemaType).(type) { case schema.StructRepresentation_Map: itr := (*_node)(w).MapIterator().(*_structIterator) // When we reach the last non-absent field, we should stop. itr.reprEnd = int(w.lengthMinusTrailingAbsents()) return (*_structIteratorRepr)(itr) case schema.UnionRepresentation_Keyed: itr := (*_node)(w).MapIterator().(*_unionIterator) return (*_unionIteratorRepr)(itr) default: iter, _ := (*_node)(w).MapIterator().(*_mapIterator) if iter == nil { return nil } return (*_mapIteratorRepr)(iter) } } type _mapIteratorRepr _mapIterator func (w *_mapIteratorRepr) Next() (key, value datamodel.Node, _ error) { k, v, err := (*_mapIterator)(w).Next() if err != nil { return nil, nil, err } return reprNode(k), reprNode(v), nil } func (w *_mapIteratorRepr) Done() bool { return w.nextIndex >= w.keysVal.Len() } func (w *_nodeRepr) ListIterator() datamodel.ListIterator { if stg, ok := reprStrategy(w.schemaType).(schema.UnionRepresentation_Kinded); ok { w = w.asKinded(stg, datamodel.Kind_List) } switch reprStrategy(w.schemaType).(type) { case schema.StructRepresentation_Tuple: typ := w.schemaType.(*schema.TypeStruct) iter := _tupleIteratorRepr{cfg: w.cfg, schemaType: typ, fields: typ.Fields(), val: w.val} iter.reprEnd = int(w.lengthMinusTrailingAbsents()) return &iter case schema.StructRepresentation_ListPairs: typ := w.schemaType.(*schema.TypeStruct) iter := _listpairsIteratorRepr{cfg: w.cfg, schemaType: typ, fields: typ.Fields(), val: w.val} iter.reprEnd = int(w.lengthMinusTrailingAbsents()) return &iter default: iter, _ := (*_node)(w).ListIterator().(*_listIterator) if iter == nil { return nil } return (*_listIteratorRepr)(iter) } } type _listIteratorRepr _listIterator func (w *_listIteratorRepr) Next() (index int64, value datamodel.Node, _ error) { idx, v, err := (*_listIterator)(w).Next() if err != nil { return idx, nil, err } return idx, reprNode(v), nil } func (w *_listIteratorRepr) Done() bool { return w.nextIndex >= w.val.Len() } func (w *_nodeRepr) lengthMinusAbsents() int64 { fields := w.schemaType.(*schema.TypeStruct).Fields() n := int64(len(fields)) for i, field := range fields { if field.IsOptional() && w.val.Field(i).IsNil() { n-- } } return n } type _tupleIteratorRepr struct { // TODO: support embedded fields? cfg *config schemaType *schema.TypeStruct fields []schema.StructField val reflect.Value // non-pointer nextIndex int // these are only used in repr.go reprEnd int } func (w *_tupleIteratorRepr) Next() (index int64, value datamodel.Node, _ error) { for { idx := w.nextIndex _, value, err := (*_structIterator)(w).Next() if err != nil { return 0, nil, err } if w.nextIndex <= w.reprEnd { return int64(idx), reprNode(value), nil } } } func (w *_tupleIteratorRepr) Done() bool { return w.nextIndex >= w.reprEnd } type _listpairsIteratorRepr struct { cfg *config schemaType *schema.TypeStruct fields []schema.StructField val reflect.Value // non-pointer nextIndex int // these are only used in repr.go reprEnd int } func (w *_listpairsIteratorRepr) Next() (index int64, value datamodel.Node, _ error) { for { if w.Done() { return 0, nil, datamodel.ErrIteratorOverread{} } idx := w.nextIndex key, value, err := (*_structIterator)(w).Next() if err != nil { return 0, nil, err } if value.IsAbsent() || w.nextIndex > w.reprEnd { continue } field, err := buildListpairsField(key, reprNode(value)) if err != nil { return 0, nil, err } return int64(idx), field, nil } } func (w *_listpairsIteratorRepr) Done() bool { return w.nextIndex >= w.reprEnd } func buildListpairsField(key, value datamodel.Node) (datamodel.Node, error) { nb := basicnode.Prototype.List.NewBuilder() la, err := nb.BeginList(2) if err != nil { return nil, err } if err := la.AssembleValue().AssignNode(key); err != nil { return nil, err } if err := la.AssembleValue().AssignNode(value); err != nil { return nil, err } if err := la.Finish(); err != nil { return nil, err } return nb.Build(), nil } func (w *_nodeRepr) lengthMinusTrailingAbsents() int64 { fields := w.schemaType.(*schema.TypeStruct).Fields() for i := len(fields) - 1; i >= 0; i-- { field := fields[i] if !field.IsOptional() || !w.val.Field(i).IsNil() { return int64(i + 1) } } return 0 } func (w *_nodeRepr) Length() int64 { switch stg := reprStrategy(w.schemaType).(type) { case schema.StructRepresentation_Stringjoin: return -1 case schema.StructRepresentation_Map: return w.lengthMinusAbsents() case schema.StructRepresentation_Tuple: return w.lengthMinusTrailingAbsents() case schema.StructRepresentation_ListPairs: return w.lengthMinusAbsents() case schema.UnionRepresentation_Keyed: return (*_node)(w).Length() case schema.UnionRepresentation_Kinded: w = w.asKinded(stg, w.Kind()) return (*_node)(w).Length() default: return (*_node)(w).Length() } } func (w *_nodeRepr) IsAbsent() bool { if reprStrategy(w.schemaType) == nil { return (*_node)(w).IsAbsent() } return false } func (w *_nodeRepr) IsNull() bool { if reprStrategy(w.schemaType) == nil { return (*_node)(w).IsNull() } return false } func (w *_nodeRepr) AsBool() (bool, error) { switch stg := reprStrategy(w.schemaType).(type) { case schema.UnionRepresentation_Kinded: return w.asKinded(stg, datamodel.Kind_Bool).AsBool() default: return (*_node)(w).AsBool() } } func (w *_nodeRepr) AsInt() (int64, error) { switch stg := reprStrategy(w.schemaType).(type) { case schema.UnionRepresentation_Kinded: return w.asKinded(stg, datamodel.Kind_Int).AsInt() case schema.EnumRepresentation_Int: kind := w.val.Kind() if kind == reflect.String { s, err := (*_node)(w).AsString() if err != nil { return 0, err } mapped, ok := stg[s] if !ok { // We assume that the schema strategy is correct, // so we can only fail if the stored string isn't a valid member. return 0, fmt.Errorf("AsInt: %q is not a valid member of enum %s", s, w.schemaType.Name()) } // TODO: the strategy type should probably use int64 rather than int return int64(mapped), nil } var i int // TODO: check for overflows if kindInt[kind] { i = int(w.val.Int()) } else if kindUint[kind] { i = int(w.val.Uint()) } else { return 0, fmt.Errorf("AsInt: unexpected kind: %s", kind) } for _, reprInt := range stg { if reprInt == i { return int64(i), nil } } // We assume that the schema strategy is correct, // so we can only fail if the stored string isn't a valid member. return 0, fmt.Errorf("AsInt: %d is not a valid member of enum %s", i, w.schemaType.Name()) default: return (*_node)(w).AsInt() } } func (w *_nodeRepr) AsFloat() (float64, error) { switch stg := reprStrategy(w.schemaType).(type) { case schema.UnionRepresentation_Kinded: return w.asKinded(stg, datamodel.Kind_Float).AsFloat() default: return (*_node)(w).AsFloat() } } func (w *_nodeRepr) AsString() (string, error) { switch stg := reprStrategy(w.schemaType).(type) { case schema.StructRepresentation_Stringjoin: var b strings.Builder itr := (*_node)(w).MapIterator() first := true for !itr.Done() { _, v, err := itr.Next() if err != nil { return "", err } s, err := reprNode(v).AsString() if err != nil { return "", err } if first { first = false } else { b.WriteString(stg.GetDelim()) } b.WriteString(s) } return b.String(), nil case schema.UnionRepresentation_Stringprefix: haveIdx, mval := unionMember(w.val) mtyp := w.schemaType.(*schema.TypeUnion).Members()[haveIdx] w2 := *w w2.val = mval w2.schemaType = mtyp s, err := w2.AsString() if err != nil { return "", err } name := stg.GetDiscriminant(mtyp) return name + stg.GetDelim() + s, nil case schema.UnionRepresentation_Kinded: return w.asKinded(stg, datamodel.Kind_String).AsString() case schema.EnumRepresentation_String: s, err := (*_node)(w).AsString() if err != nil { return "", err } if mapped := stg[s]; mapped != "" { return mapped, nil } members := w.schemaType.(*schema.TypeEnum).Members() for _, member := range members { if s == member { return s, nil } } for k, v := range stg { // a programming error? we may have the enum string value rather than the type if v == s { return "", fmt.Errorf("AsString: %q is not a valid member of enum %s (bindnode works at the type level; did you mean %q?)", s, w.schemaType.Name(), k) } } return "", fmt.Errorf("AsString: %q is not a valid member of enum %s", s, w.schemaType.Name()) default: return (*_node)(w).AsString() } } func (w *_nodeRepr) AsBytes() ([]byte, error) { switch stg := reprStrategy(w.schemaType).(type) { case schema.UnionRepresentation_Kinded: return w.asKinded(stg, datamodel.Kind_Bytes).AsBytes() default: return (*_node)(w).AsBytes() } } func (w *_nodeRepr) AsLink() (datamodel.Link, error) { switch stg := reprStrategy(w.schemaType).(type) { case schema.UnionRepresentation_Kinded: return w.asKinded(stg, datamodel.Kind_Link).AsLink() default: return (*_node)(w).AsLink() } } func (w *_nodeRepr) Prototype() datamodel.NodePrototype { return (*_prototypeRepr)((*_node)(w).Prototype().(*_prototype)) } type _builderRepr struct { _assemblerRepr } // TODO: returning a repr node here is probably good, but there's a gotcha: one // can go from a typed node to a repr node via the Representation method, but // not the other way. That's probably why codegen returns a typed node here. // The solution might be to add a way to go from the repr node to its parent // typed node. func (w *_builderRepr) Build() datamodel.Node { // TODO: see the notes above. // return &_nodeRepr{schemaType: w.schemaType, val: w.val} return &_node{cfg: w.cfg, schemaType: w.schemaType, val: w.val} } func (w *_builderRepr) Reset() { panic("bindnode TODO: Reset") } type _assemblerRepr struct { cfg *config schemaType schema.Type val reflect.Value // non-pointer finish func() error nullable bool } func assemblerRepr(am datamodel.NodeAssembler) datamodel.NodeAssembler { switch am := am.(type) { case *_assembler: return (*_assemblerRepr)(am) case _errorAssembler: return am default: panic(fmt.Sprintf("unexpected NodeAssembler type: %T", am)) } } func (w *_assemblerRepr) asKinded(stg schema.UnionRepresentation_Kinded, kind datamodel.Kind) datamodel.NodeAssembler { name := stg.GetMember(kind) members := w.schemaType.(*schema.TypeUnion).Members() kindSet := make([]datamodel.Kind, 0, len(members)) for idx, member := range members { if member.Name() != name { kindSet = append(kindSet, member.RepresentationBehavior()) continue } w2 := *w goType := w.val.Field(idx).Type().Elem() valPtr := reflect.New(goType) w2.val = valPtr.Elem() w2.schemaType = member // Layer a new finish func on top, to set Index/Value. w2.finish = func() error { unionSetMember(w.val, idx, valPtr) if w.finish != nil { if err := w.finish(); err != nil { return err } } return nil } return &w2 } return _errorAssembler{datamodel.ErrWrongKind{ TypeName: w.schemaType.Name() + ".Repr", MethodName: "", // TODO: we could fill it via runtime.Callers AppropriateKind: datamodel.KindSet(kindSet), ActualKind: kind, }} } func (w *_assemblerRepr) BeginMap(sizeHint int64) (datamodel.MapAssembler, error) { if stg, ok := reprStrategy(w.schemaType).(schema.UnionRepresentation_Kinded); ok { return w.asKinded(stg, datamodel.Kind_Map).BeginMap(sizeHint) } asm, err := (*_assembler)(w).BeginMap(sizeHint) if err != nil { return nil, err } switch asm := asm.(type) { case *_structAssembler: return (*_structAssemblerRepr)(asm), nil case *_mapAssembler: return (*_mapAssemblerRepr)(asm), nil case *_unionAssembler: return (*_unionAssemblerRepr)(asm), nil case *basicMapAssembler: return asm, nil default: return nil, fmt.Errorf("bindnode BeginMap TODO: %T", asm) } } func (w *_assemblerRepr) BeginList(sizeHint int64) (datamodel.ListAssembler, error) { switch stg := reprStrategy(w.schemaType).(type) { case schema.UnionRepresentation_Kinded: return w.asKinded(stg, datamodel.Kind_List).BeginList(sizeHint) case schema.StructRepresentation_Tuple, schema.StructRepresentation_ListPairs: asm, err := (*_assembler)(w).BeginMap(sizeHint) if err != nil { return nil, err } return (*_listStructAssemblerRepr)(asm.(*_structAssembler)), nil default: asm, err := (*_assembler)(w).BeginList(sizeHint) if err != nil { return nil, err } if _, ok := asm.(*basicListAssembler); ok { return asm, nil } return (*_listAssemblerRepr)(asm.(*_listAssembler)), nil } } func (w *_assemblerRepr) AssignNull() error { return (*_assembler)(w).AssignNull() } func (w *_assemblerRepr) AssignBool(b bool) error { switch stg := reprStrategy(w.schemaType).(type) { case schema.UnionRepresentation_Kinded: return w.asKinded(stg, datamodel.Kind_Bool).AssignBool(b) default: return (*_assembler)(w).AssignBool(b) } } func (w *_assemblerRepr) assignUInt(uin datamodel.UintNode) error { switch stg := reprStrategy(w.schemaType).(type) { case schema.UnionRepresentation_Kinded: return w.asKinded(stg, datamodel.Kind_Int).(*_assemblerRepr).assignUInt(uin) case schema.EnumRepresentation_Int: uin, err := uin.AsUint() if err != nil { return err } return fmt.Errorf("AssignInt: %d is not a valid member of enum %s", uin, w.schemaType.Name()) default: return (*_assembler)(w).assignUInt(uin) } } func (w *_assemblerRepr) AssignInt(i int64) error { switch stg := reprStrategy(w.schemaType).(type) { case schema.UnionRepresentation_Kinded: return w.asKinded(stg, datamodel.Kind_Int).AssignInt(i) case schema.EnumRepresentation_Int: for member, reprInt := range stg { if int64(reprInt) != i { continue } val := (*_assembler)(w).createNonPtrVal() kind := val.Kind() if kind == reflect.String { // Reuse AssignString so we don't have to repeat ten lines. return (*_assembler)(w).AssignString(member) } // Short-cut to storing the repr int directly, akin to node.go's AssignInt. if kindInt[kind] { val.SetInt(i) } else if kindUint[kind] { if i < 0 { // TODO: write a test return fmt.Errorf("bindnode: cannot assign negative integer to %s", w.val.Type()) } val.SetUint(uint64(i)) } else { return fmt.Errorf("AsInt: unexpected kind: %s", val.Kind()) } if w.finish != nil { if err := w.finish(); err != nil { return err } } return nil } return fmt.Errorf("AssignInt: %d is not a valid member of enum %s", i, w.schemaType.Name()) default: return (*_assembler)(w).AssignInt(i) } } func (w *_assemblerRepr) AssignFloat(f float64) error { switch stg := reprStrategy(w.schemaType).(type) { case schema.UnionRepresentation_Kinded: return w.asKinded(stg, datamodel.Kind_Float).AssignFloat(f) default: return (*_assembler)(w).AssignFloat(f) } } func (w *_assemblerRepr) AssignString(s string) error { switch stg := reprStrategy(w.schemaType).(type) { case schema.StructRepresentation_Stringjoin: fields := w.schemaType.(*schema.TypeStruct).Fields() parts := strings.Split(s, stg.GetDelim()) if len(parts) != len(fields) { return fmt.Errorf("bindnode TODO: len mismatch") } mapAsm, err := (*_assembler)(w).BeginMap(-1) if err != nil { return err } for i, field := range fields { entryAsm, err := mapAsm.AssembleEntry(field.Name()) if err != nil { return err } entryAsm = assemblerRepr(entryAsm) if err := entryAsm.AssignString(parts[i]); err != nil { return err } } return mapAsm.Finish() case schema.UnionRepresentation_Kinded: return w.asKinded(stg, datamodel.Kind_String).AssignString(s) case schema.UnionRepresentation_Stringprefix: hasDelim := stg.GetDelim() != "" var prefix, remainder string if hasDelim { parts := strings.SplitN(s, stg.GetDelim(), 2) if len(parts) != 2 { return fmt.Errorf("schema rejects data: the union type %s expects delimiter %q, and it was not found in the data %q", w.schemaType.Name(), stg.GetDelim(), s) } prefix, remainder = parts[0], parts[1] } members := w.schemaType.(*schema.TypeUnion).Members() for idx, member := range members { descrm := stg.GetDiscriminant(member) if hasDelim { if stg.GetDiscriminant(member) != prefix { continue } } else { if !strings.HasPrefix(s, descrm) { continue } remainder = s[len(descrm):] } // TODO: DRY: this has much in common with the asKinded method; it differs only in that we picked idx already in a different way. w2 := *w goType := w.val.Field(idx).Type().Elem() valPtr := reflect.New(goType) w2.val = valPtr.Elem() w2.schemaType = member w2.finish = func() error { unionSetMember(w.val, idx, valPtr) if w.finish != nil { if err := w.finish(); err != nil { return err } } return nil } return w2.AssignString(remainder) } return fmt.Errorf("schema rejects data: the union type %s requires a known prefix, and it was not found in the data %q", w.schemaType.Name(), s) case schema.EnumRepresentation_String: // Note that we need to do a reverse lookup. for member, mapped := range stg { if mapped == s { return (*_assembler)(w).AssignString(member) } } members := w.schemaType.(*schema.TypeEnum).Members() for _, member := range members { if s == member { return (*_assembler)(w).AssignString(member) } } return fmt.Errorf("AssignString: %q is not a valid member of enum %s", s, w.schemaType.Name()) case schema.EnumRepresentation_Int: return datamodel.ErrWrongKind{ TypeName: w.schemaType.Name(), MethodName: "AssignString", AppropriateKind: datamodel.KindSet_JustInt, ActualKind: datamodel.Kind_String, } default: return (*_assembler)(w).AssignString(s) } } func (w *_assemblerRepr) AssignBytes(p []byte) error { switch stg := reprStrategy(w.schemaType).(type) { case schema.UnionRepresentation_Kinded: return w.asKinded(stg, datamodel.Kind_Bytes).AssignBytes(p) default: return (*_assembler)(w).AssignBytes(p) } } func (w *_assemblerRepr) AssignLink(link datamodel.Link) error { switch stg := reprStrategy(w.schemaType).(type) { case schema.UnionRepresentation_Kinded: return w.asKinded(stg, datamodel.Kind_Link).AssignLink(link) default: return (*_assembler)(w).AssignLink(link) } } func (w *_assemblerRepr) AssignNode(node datamodel.Node) error { // TODO: attempt to take a shortcut, like assembler.AssignNode if uintNode, ok := node.(datamodel.UintNode); ok { return w.assignUInt(uintNode) } return datamodel.Copy(node, w) } func (w *_assemblerRepr) Prototype() datamodel.NodePrototype { panic("bindnode TODO: Assembler.Prototype") } type _structAssemblerRepr _structAssembler func (w *_structAssemblerRepr) AssembleKey() datamodel.NodeAssembler { switch stg := reprStrategy(w.schemaType).(type) { case schema.StructRepresentation_Map: return (*_structAssembler)(w).AssembleKey() case schema.StructRepresentation_Stringjoin, schema.StructRepresentation_StringPairs: // TODO: perhaps the ErrorWrongKind type should also be extended to explicitly describe whether the method was applied on bare DM, type-level, or repr-level. return _errorAssembler{datamodel.ErrWrongKind{ TypeName: w.schemaType.Name() + ".Repr", MethodName: "AssembleKey", AppropriateKind: datamodel.KindSet_JustMap, ActualKind: datamodel.Kind_String, }} case schema.StructRepresentation_Tuple, schema.StructRepresentation_ListPairs: return _errorAssembler{datamodel.ErrWrongKind{ TypeName: w.schemaType.Name() + ".Repr", MethodName: "AssembleKey", AppropriateKind: datamodel.KindSet_JustMap, ActualKind: datamodel.Kind_List, }} default: return _errorAssembler{fmt.Errorf("bindnode AssembleKey TODO: %T", stg)} } } func (w *_structAssemblerRepr) AssembleValue() datamodel.NodeAssembler { switch stg := reprStrategy(w.schemaType).(type) { case schema.StructRepresentation_Map: key := w.curKey.val.String() revKey := inboundMappedKey(w.schemaType, stg, key) w.curKey.val.SetString(revKey) valAsm := (*_structAssembler)(w).AssembleValue() valAsm = assemblerRepr(valAsm) return valAsm default: return _errorAssembler{fmt.Errorf("bindnode AssembleValue TODO: %T", stg)} } } func (w *_structAssemblerRepr) AssembleEntry(k string) (datamodel.NodeAssembler, error) { if err := w.AssembleKey().AssignString(k); err != nil { return nil, err } am := w.AssembleValue() return am, nil } func (w *_structAssemblerRepr) Finish() error { switch stg := reprStrategy(w.schemaType).(type) { case schema.StructRepresentation_Map: err := (*_structAssembler)(w).Finish() if err, ok := err.(schema.ErrMissingRequiredField); ok { for i, name := range err.Missing { serial := outboundMappedKey(stg, name) if serial != name { err.Missing[i] += fmt.Sprintf(" (serial:%q)", serial) } } } return err default: return fmt.Errorf("bindnode Finish TODO: %T", stg) } } func (w *_structAssemblerRepr) KeyPrototype() datamodel.NodePrototype { panic("bindnode TODO") } func (w *_structAssemblerRepr) ValuePrototype(k string) datamodel.NodePrototype { panic("bindnode TODO: struct ValuePrototype") } type _mapAssemblerRepr _mapAssembler func (w *_mapAssemblerRepr) AssembleKey() datamodel.NodeAssembler { asm := (*_mapAssembler)(w).AssembleKey() return (*_assemblerRepr)(asm.(*_assembler)) } func (w *_mapAssemblerRepr) AssembleValue() datamodel.NodeAssembler { asm := (*_mapAssembler)(w).AssembleValue() return (*_assemblerRepr)(asm.(*_assembler)) } func (w *_mapAssemblerRepr) AssembleEntry(k string) (datamodel.NodeAssembler, error) { if err := w.AssembleKey().AssignString(k); err != nil { return nil, err } am := w.AssembleValue() return am, nil } func (w *_mapAssemblerRepr) Finish() error { return (*_mapAssembler)(w).Finish() } func (w *_mapAssemblerRepr) KeyPrototype() datamodel.NodePrototype { panic("bindnode TODO") } func (w *_mapAssemblerRepr) ValuePrototype(k string) datamodel.NodePrototype { panic("bindnode TODO: struct ValuePrototype") } type _listStructAssemblerRepr _structAssembler func (w *_listStructAssemblerRepr) AssembleValue() datamodel.NodeAssembler { switch stg := reprStrategy(w.schemaType).(type) { case schema.StructRepresentation_Tuple: fields := w.schemaType.Fields() if w.nextIndex >= len(fields) { return _errorAssembler{datamodel.ErrNotExists{ Segment: datamodel.PathSegmentOfInt(int64(w.nextIndex)), }} } field := fields[w.nextIndex] w.doneFields[w.nextIndex] = true w.nextIndex++ entryAsm, err := (*_structAssembler)(w).AssembleEntry(field.Name()) if err != nil { return _errorAssembler{err} } entryAsm = assemblerRepr(entryAsm) return entryAsm case schema.StructRepresentation_ListPairs: return &_listpairsFieldAssemblerRepr{parent: (*_structAssembler)(w)} default: return _errorAssembler{fmt.Errorf("bindnode AssembleValue TODO: %T", stg)} } } func (w *_listStructAssemblerRepr) Finish() error { switch stg := reprStrategy(w.schemaType).(type) { case schema.StructRepresentation_Tuple, schema.StructRepresentation_ListPairs: return (*_structAssembler)(w).Finish() default: return fmt.Errorf("bindnode Finish TODO: %T", stg) } } func (w *_listStructAssemblerRepr) ValuePrototype(idx int64) datamodel.NodePrototype { panic("bindnode TODO: list ValuePrototype") } type _listpairsFieldAssemblerRepr struct { parent *_structAssembler } func (w _listpairsFieldAssemblerRepr) BeginMap(int64) (datamodel.MapAssembler, error) { return nil, datamodel.ErrWrongKind{ TypeName: w.parent.schemaType.Name(), MethodName: "BeginMap", AppropriateKind: datamodel.KindSet_JustList, ActualKind: datamodel.Kind_Map, } } func (w *_listpairsFieldAssemblerRepr) BeginList(int64) (datamodel.ListAssembler, error) { return &_listpairsFieldListAssemblerRepr{parent: w.parent}, nil } func (w _listpairsFieldAssemblerRepr) AssignNull() error { return datamodel.ErrWrongKind{ TypeName: w.parent.schemaType.Name(), MethodName: "AssignNull", AppropriateKind: datamodel.KindSet_JustList, ActualKind: datamodel.Kind_Map, } } func (w _listpairsFieldAssemblerRepr) AssignBool(bool) error { return datamodel.ErrWrongKind{ TypeName: w.parent.schemaType.Name(), MethodName: "AssignBool", AppropriateKind: datamodel.KindSet_JustList, ActualKind: datamodel.Kind_Map, } } func (w _listpairsFieldAssemblerRepr) AssignInt(int64) error { return datamodel.ErrWrongKind{ TypeName: w.parent.schemaType.Name(), MethodName: "AssignInt", AppropriateKind: datamodel.KindSet_JustList, ActualKind: datamodel.Kind_Map, } } func (w _listpairsFieldAssemblerRepr) AssignFloat(float64) error { return datamodel.ErrWrongKind{ TypeName: w.parent.schemaType.Name(), MethodName: "AssignFloat", AppropriateKind: datamodel.KindSet_JustList, ActualKind: datamodel.Kind_Map, } } func (w _listpairsFieldAssemblerRepr) AssignString(string) error { return datamodel.ErrWrongKind{ TypeName: w.parent.schemaType.Name(), MethodName: "AssignString", AppropriateKind: datamodel.KindSet_JustList, ActualKind: datamodel.Kind_Map, } } func (w _listpairsFieldAssemblerRepr) AssignBytes([]byte) error { return datamodel.ErrWrongKind{ TypeName: w.parent.schemaType.Name(), MethodName: "AssignBytes", AppropriateKind: datamodel.KindSet_JustList, ActualKind: datamodel.Kind_Map, } } func (w _listpairsFieldAssemblerRepr) AssignLink(datamodel.Link) error { return datamodel.ErrWrongKind{ TypeName: w.parent.schemaType.Name(), MethodName: "AssignLink", AppropriateKind: datamodel.KindSet_JustList, ActualKind: datamodel.Kind_Map, } } func (w *_listpairsFieldAssemblerRepr) AssignNode(n datamodel.Node) error { return datamodel.Copy(n, w) } func (w _listpairsFieldAssemblerRepr) Prototype() datamodel.NodePrototype { panic("bindnode TODO: listpairs field Prototype") } type _listpairsFieldListAssemblerRepr struct { parent *_structAssembler idx int } func (w *_listpairsFieldListAssemblerRepr) AssembleValue() datamodel.NodeAssembler { w.idx++ switch w.idx { case 1: return w.parent.AssembleKey() case 2: asm := w.parent.AssembleValue() return assemblerRepr(asm.(*_assembler)) default: return _errorAssembler{fmt.Errorf("bindnode: too many values in listpairs field")} } } func (w *_listpairsFieldListAssemblerRepr) Finish() error { return nil } func (w *_listpairsFieldListAssemblerRepr) ValuePrototype(idx int64) datamodel.NodePrototype { panic("bindnode TODO: listpairs field ValuePrototype") } // Note that lists do not have any representation strategy right now. type _listAssemblerRepr _listAssembler func (w *_listAssemblerRepr) AssembleValue() datamodel.NodeAssembler { asm := (*_listAssembler)(w).AssembleValue() return (*_assemblerRepr)(asm.(*_assembler)) } func (w *_listAssemblerRepr) Finish() error { return (*_listAssembler)(w).Finish() } func (w *_listAssemblerRepr) ValuePrototype(idx int64) datamodel.NodePrototype { panic("bindnode TODO: list ValuePrototype") } type _unionAssemblerRepr _unionAssembler func (w *_unionAssemblerRepr) AssembleKey() datamodel.NodeAssembler { switch stg := reprStrategy(w.schemaType).(type) { case schema.UnionRepresentation_Keyed: return (*_unionAssembler)(w).AssembleKey() default: return _errorAssembler{fmt.Errorf("bindnode AssembleKey TODO: %T", stg)} } } func (w *_unionAssemblerRepr) AssembleValue() datamodel.NodeAssembler { switch stg := reprStrategy(w.schemaType).(type) { case schema.UnionRepresentation_Keyed: key := w.curKey.val.String() revKey := inboundMappedType(w.schemaType, stg, key) w.curKey.val.SetString(revKey) valAsm := (*_unionAssembler)(w).AssembleValue() valAsm = assemblerRepr(valAsm) return valAsm default: return _errorAssembler{fmt.Errorf("bindnode AssembleValue TODO: %T", stg)} } } func (w *_unionAssemblerRepr) AssembleEntry(k string) (datamodel.NodeAssembler, error) { if err := w.AssembleKey().AssignString(k); err != nil { return nil, err } am := w.AssembleValue() return am, nil } func (w *_unionAssemblerRepr) Finish() error { switch stg := reprStrategy(w.schemaType).(type) { case schema.UnionRepresentation_Keyed: return (*_unionAssembler)(w).Finish() default: return fmt.Errorf("bindnode Finish TODO: %T", stg) } } func (w *_unionAssemblerRepr) KeyPrototype() datamodel.NodePrototype { panic("bindnode TODO") } func (w *_unionAssemblerRepr) ValuePrototype(k string) datamodel.NodePrototype { panic("bindnode TODO: union ValuePrototype") } type _structIteratorRepr _structIterator func (w *_structIteratorRepr) Next() (key, value datamodel.Node, _ error) { switch stg := reprStrategy(w.schemaType).(type) { case schema.StructRepresentation_Map: for { key, value, err := (*_structIterator)(w).Next() if err != nil { return nil, nil, err } if value.IsAbsent() { continue } keyStr, _ := key.AsString() mappedKey := outboundMappedKey(stg, keyStr) if mappedKey != keyStr { key = basicnode.NewString(mappedKey) } return key, reprNode(value), nil } default: return nil, nil, fmt.Errorf("bindnode Next TODO: %T", stg) } } func (w *_structIteratorRepr) Done() bool { switch stg := reprStrategy(w.schemaType).(type) { case schema.StructRepresentation_Map: // TODO: the fact that repr map iterators skip absents should be // documented somewhere return w.nextIndex >= w.reprEnd default: panic(fmt.Sprintf("bindnode Done TODO: %T", stg)) } } type _unionIteratorRepr _unionIterator func (w *_unionIteratorRepr) Next() (key, value datamodel.Node, _ error) { switch stg := reprStrategy(w.schemaType).(type) { case schema.UnionRepresentation_Keyed: key, value, err := (*_unionIterator)(w).Next() if err != nil { return nil, nil, err } keyStr, _ := key.AsString() mappedKey := outboundMappedType(stg, keyStr) if mappedKey != keyStr { key = basicnode.NewString(mappedKey) } return key, reprNode(value), nil default: return nil, nil, fmt.Errorf("bindnode Next TODO: %T", stg) } } func (w *_unionIteratorRepr) Done() bool { switch stg := reprStrategy(w.schemaType).(type) { case schema.UnionRepresentation_Keyed: return (*_unionIterator)(w).Done() default: panic(fmt.Sprintf("bindnode Done TODO: %T", stg)) } } ================================================ FILE: node/bindnode/schema_test.go ================================================ package bindnode_test import ( "strings" "testing" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/node/bindnode" "github.com/ipld/go-ipld-prime/node/tests" "github.com/ipld/go-ipld-prime/schema" ) // For now, we simply run all schema tests with Prototype. // In the future, forSchemaTest might return multiple engines. func forSchemaTest(name string) []tests.EngineSubtest { if name == "Links" { // TODO(mvdan): support typed links; see https://github.com/ipld/go-ipld-prime/issues/272 return nil } if name == "UnionKeyedComplexChildren" { return nil // Specifically, 'InhabitantB/repr-create_with_AK+AV' borks, because it needs representation-level AssignNode to support more. } return []tests.EngineSubtest{{ Engine: &bindEngine{}, }} } func TestSchema(t *testing.T) { t.Parallel() tests.SchemaTestAll(t, forSchemaTest) } var _ tests.Engine = (*bindEngine)(nil) type bindEngine struct { ts schema.TypeSystem } func (e *bindEngine) Init(t *testing.T, ts schema.TypeSystem) { e.ts = ts } func (e *bindEngine) PrototypeByName(name string) datamodel.NodePrototype { wantRepr := strings.HasSuffix(name, ".Repr") if wantRepr { name = strings.TrimSuffix(name, ".Repr") } schemaType := e.ts.TypeByName(name) if wantRepr { return bindnode.Prototype(nil, schemaType).Representation() } return bindnode.Prototype(nil, schemaType) } ================================================ FILE: node/bindnode/testdata/fuzz/FuzzBindnodeViaDagCBOR/164dd28629e6a5637f02c6eaad32eee5bf8ea41ce6d1dae95334a7db7dd6188f ================================================ go test fuzz v1 []byte("\xa1etypes\xa0") []byte("\xf9\xfc\x00") ================================================ FILE: node/bindnode/testdata/fuzz/FuzzBindnodeViaDagCBOR/199afa754020c4587bc87633033b4e56ecdb5ecc2bb0dbac46b799d6c18c5201 ================================================ go test fuzz v1 []byte("\xa1etypes\xa1d0000\xa0") []byte("0") ================================================ FILE: node/bindnode/testdata/fuzz/FuzzBindnodeViaDagCBOR/2c17d42168478a0837ebba66f6aa98bdf4bb81b5196062d0e6b23c8b0325be6a ================================================ go test fuzz v1 []byte("\xa1etypes\xa1dRoot\xa1fstruct\xa2ffields\xa2b00\xa1dtypedBoolb00\xa1dtypeeBytesnrepresentation\xa1cmap\xa0") []byte("0") ================================================ FILE: node/bindnode/testdata/fuzz/FuzzBindnodeViaDagCBOR/425499b0c3693d87a0b11db10d21c540283dfb88610d114eb5b23485d5c2f342 ================================================ go test fuzz v1 []byte("\xa1etypes\xa1dRoot\xa1cmap\xa2gkeyTypefStringivalueTypecInt") []byte("\xa2ac0a00") ================================================ FILE: node/bindnode/testdata/fuzz/FuzzBindnodeViaDagCBOR/636f8e5cdaf52572b826e615a9bdf90f15f8acf2880b467b3949f5356c2ddded ================================================ go test fuzz v1 []byte("\xa1etypes\xa1dRoot\xa1fstruct\xa2ffields\xa2b00\xa1dtyped0000b00\xa1dtypee00000nrepresentation\xa1cmap\xa0") []byte("0") ================================================ FILE: node/bindnode/testdata/fuzz/FuzzBindnodeViaDagCBOR/669d57fbbe55b8ceb7e78e3ccd3b90e76c4b740d25284a66eac8aac0dbb2475d ================================================ go test fuzz v1 []byte("\xa1etypes\xa1dRoot\xa1eunion\xa2gmembers\x82cIntfStringnrepresentation\xa1ekeyed\xa2a0c000a0a000000") []byte("\xa1a00000") ================================================ FILE: node/bindnode/testdata/fuzz/FuzzBindnodeViaDagCBOR/7f9a6898dead41ba2f5fd4f07f2ddb44d54e7648d66e54982d09996bc720cf56 ================================================ go test fuzz v1 []byte("\xa1etypes\xa1dRoot\xa1dlist\xa1ivalueTypedRootcInt") []byte("\x810") ================================================ FILE: node/bindnode/testdata/fuzz/FuzzBindnodeViaDagCBOR/821c248529299bb6b68e443e4b00cc11c6605f766f25e619d85e6d6b40a33dac ================================================ go test fuzz v1 []byte("\xa1etypes\xa1dRoot\xa1denum\xa2gmembers\x82c000c000nrepresentation\xa1cint\xa2c0000c0000") []byte("a000") ================================================ FILE: node/bindnode/testdata/fuzz/FuzzBindnodeViaDagCBOR/bf7c410983f3e696a03e743df1bc6f606137871f7fb557af74836b7aa04e56ad ================================================ go test fuzz v1 []byte("\xa1etypes\xa0") []byte("\xf9|\x00") ================================================ FILE: node/bindnode/testdata/fuzz/FuzzBindnodeViaDagCBOR/dc95e9aad454ed9109e93b824f92d4bb00c9c778af29e6fcb38a6083c78d7dbb ================================================ go test fuzz v1 []byte("\xa1etypes\xa1d0000\xa1dlist\xa1ivalueTypea0") []byte("\xfa\u007f\xff00") ================================================ FILE: node/bindnode/testdata/fuzz/FuzzBindnodeViaDagCBOR/f63d4652cdac3208fc0a0d0b755615320443b60ef80c0ecdef99c9d895ae2124 ================================================ go test fuzz v1 []byte("\xa1etypes\xa1dRoot\xa1eunion\xa2gmembers\x82cIntfStringnrepresentation\xa1ekeyed\xa0") []byte("\xa1a00") ================================================ FILE: node/bindnode/testdata/fuzz/FuzzBindnodeViaDagCBOR/fe73a19655fff8b93193c3e90f9cd7b24b10ae0467175cf2264ff3ec135215a6 ================================================ go test fuzz v1 []byte("\xa1etypes\xa1dRoot\xa1denum\xa2gmembers\x82a1b00nrepresentation\xa1fstring\xa2a0a0a0`") []byte("\u007f\xff") ================================================ FILE: node/doc.go ================================================ /* The 'node' package gathers various general purpose Node implementations; the first one you should jump to is 'node/basicnode'. There's no code in this package itself; it's just for grouping. The `Node` interface itself is in the `ipld` package, which is the parent of this. The 'node/mixins' package contains reusable component code for building your own node implementations, should you desire to do so. This includes standardized behavioral tests (!), which are in the 'node/mixins/tests' package. Other planned subpackages include: a cbor-native Node implementation (which can optimize performance in some cases by lazily parsing serial data, and also retaining it as byte slice references for minimizing reserialization work for small mutations); a Node implementation which works over golang native types by use of reflection; a Node implementation which supports Schema type constraints and works without compile-time/codegen support by delegating storage to another Node implementation; etc. You can create your own Node implementations, too. There's nothing special about being in this package. Other Node implementations not found here may include those which are output from Schema-powered codegen! */ package node ================================================ FILE: node/gendemo/doc.go ================================================ // The gendemo package contains some what codegen output code, // so that it can demonstrate what schema-based codegen looks and acts like. // // The main purpose is to benchmark things, // and to provide an easy-to-look-at _thing_ for prospective users // who want to lay eyes on generated code without needing to get up-and-running with the generator themselves. // // This package is absolutely _not_ full of general purpose node implementations // that you should use in _any_ application. // // The input info for the code generation is in `gen.go` file. // (This is currently wired directly in code; in the future, the same instructions // will be extracted to an IPLD Schema file and standard tools will be used to process it.) // The code generation is triggered by `go:generate` comments in the `doc.go` file. //go:generate go run gen.go package gendemo ================================================ FILE: node/gendemo/gen.go ================================================ //go:build ignore package main import ( "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/schema" gengo "github.com/ipld/go-ipld-prime/schema/gen/go" ) func main() { pkgName := "gendemo" ts := schema.TypeSystem{} ts.Init() adjCfg := &gengo.AdjunctCfg{} ts.Accumulate(schema.SpawnInt("Int")) ts.Accumulate(schema.SpawnString("String")) ts.Accumulate(schema.SpawnStruct("Msg3", []schema.StructField{ schema.SpawnStructField("whee", "Int", false, false), schema.SpawnStructField("woot", "Int", false, false), schema.SpawnStructField("waga", "Int", false, false), }, schema.SpawnStructRepresentationMap(nil), )) ts.Accumulate(schema.SpawnMap("Map__String__Msg3", "String", "Msg3", false)) ts.Accumulate(schema.SpawnBool("Bar")) ts.Accumulate(schema.SpawnString("Baz")) ts.Accumulate(schema.SpawnInt("Foo")) ts.Accumulate(schema.SpawnUnion("UnionKinded", []schema.TypeName{ "Foo", "Bar", "Baz", }, schema.SpawnUnionRepresentationKinded( map[datamodel.Kind]schema.TypeName{ datamodel.Kind_Int: "Foo", datamodel.Kind_Bool: "Bar", datamodel.Kind_String: "Baz", }), )) gengo.Generate(".", pkgName, ts, adjCfg) } ================================================ FILE: node/gendemo/gendemo_test.go ================================================ package gendemo import ( "testing" "github.com/ipld/go-ipld-prime/node/tests" ) func BenchmarkMapStrInt_3n_AssembleStandard(b *testing.B) { tests.SpecBenchmarkMapStrInt_3n_AssembleStandard(b, _Msg3__Prototype{}) } func BenchmarkMapStrInt_3n_AssembleEntry(b *testing.B) { tests.SpecBenchmarkMapStrInt_3n_AssembleEntry(b, _Msg3__Prototype{}) } func BenchmarkMapStrInt_3n_Iteration(b *testing.B) { tests.SpecBenchmarkMapStrInt_3n_Iteration(b, _Msg3__Prototype{}) } func BenchmarkSpec_Marshal_Map3StrInt(b *testing.B) { tests.BenchmarkSpec_Marshal_Map3StrInt(b, _Msg3__Prototype{}) } func BenchmarkSpec_Unmarshal_Map3StrInt(b *testing.B) { tests.BenchmarkSpec_Unmarshal_Map3StrInt(b, _Msg3__Prototype{}) } func BenchmarkSpec_Marshal_MapNStrMap3StrInt(b *testing.B) { tests.BenchmarkSpec_Marshal_MapNStrMap3StrInt(b, _Map__String__Msg3__Prototype{}) } func BenchmarkSpec_Unmarshal_MapNStrMap3StrInt(b *testing.B) { tests.BenchmarkSpec_Unmarshal_MapNStrMap3StrInt(b, _Map__String__Msg3__Prototype{}) } // the standard 'walk' benchmarks don't work yet because those use selectors and use the prototype we give them for that, which... // does not fly: cramming selector keys into assemblers meant for struct types from our test corpus? nope. // this is a known shortcut-become-bug with the design of the 'walk' benchmarks; we'll have to fix soon. ================================================ FILE: node/gendemo/ipldsch_minima.go ================================================ package gendemo // Code generated by go-ipld-prime gengo. DO NOT EDIT. import ( "fmt" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/schema" ) const ( midvalue = schema.Maybe(4) allowNull = schema.Maybe(5) ) type maState uint8 const ( maState_initial maState = iota maState_midKey maState_expectValue maState_midValue maState_finished ) type laState uint8 const ( laState_initial laState = iota laState_midValue laState_finished ) type _ErrorThunkAssembler struct { e error } func (ea _ErrorThunkAssembler) BeginMap(_ int64) (datamodel.MapAssembler, error) { return nil, ea.e } func (ea _ErrorThunkAssembler) BeginList(_ int64) (datamodel.ListAssembler, error) { return nil, ea.e } func (ea _ErrorThunkAssembler) AssignNull() error { return ea.e } func (ea _ErrorThunkAssembler) AssignBool(bool) error { return ea.e } func (ea _ErrorThunkAssembler) AssignInt(int64) error { return ea.e } func (ea _ErrorThunkAssembler) AssignFloat(float64) error { return ea.e } func (ea _ErrorThunkAssembler) AssignString(string) error { return ea.e } func (ea _ErrorThunkAssembler) AssignBytes([]byte) error { return ea.e } func (ea _ErrorThunkAssembler) AssignLink(datamodel.Link) error { return ea.e } func (ea _ErrorThunkAssembler) AssignNode(datamodel.Node) error { return ea.e } func (ea _ErrorThunkAssembler) Prototype() datamodel.NodePrototype { panic(fmt.Errorf("cannot get prototype from error-carrying assembler: already derailed with error: %w", ea.e)) } ================================================ FILE: node/gendemo/ipldsch_satisfaction.go ================================================ package gendemo // Code generated by go-ipld-prime gengo. DO NOT EDIT. import ( "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/node/mixins" "github.com/ipld/go-ipld-prime/schema" ) func (n Bar) Bool() bool { return n.x } func (_Bar__Prototype) FromBool(v bool) (Bar, error) { n := _Bar{v} return &n, nil } type _Bar__Maybe struct { m schema.Maybe v _Bar } type MaybeBar = *_Bar__Maybe func (m MaybeBar) IsNull() bool { return m.m == schema.Maybe_Null } func (m MaybeBar) IsAbsent() bool { return m.m == schema.Maybe_Absent } func (m MaybeBar) Exists() bool { return m.m == schema.Maybe_Value } func (m MaybeBar) AsNode() datamodel.Node { switch m.m { case schema.Maybe_Absent: return datamodel.Absent case schema.Maybe_Null: return datamodel.Null case schema.Maybe_Value: return &m.v default: panic("unreachable") } } func (m MaybeBar) Must() Bar { if !m.Exists() { panic("unbox of a maybe rejected") } return &m.v } var _ datamodel.Node = (Bar)(&_Bar{}) var _ schema.TypedNode = (Bar)(&_Bar{}) func (Bar) Kind() datamodel.Kind { return datamodel.Kind_Bool } func (Bar) LookupByString(string) (datamodel.Node, error) { return mixins.Bool{TypeName: "gendemo.Bar"}.LookupByString("") } func (Bar) LookupByNode(datamodel.Node) (datamodel.Node, error) { return mixins.Bool{TypeName: "gendemo.Bar"}.LookupByNode(nil) } func (Bar) LookupByIndex(idx int64) (datamodel.Node, error) { return mixins.Bool{TypeName: "gendemo.Bar"}.LookupByIndex(0) } func (Bar) LookupBySegment(seg datamodel.PathSegment) (datamodel.Node, error) { return mixins.Bool{TypeName: "gendemo.Bar"}.LookupBySegment(seg) } func (Bar) MapIterator() datamodel.MapIterator { return nil } func (Bar) ListIterator() datamodel.ListIterator { return nil } func (Bar) Length() int64 { return -1 } func (Bar) IsAbsent() bool { return false } func (Bar) IsNull() bool { return false } func (n Bar) AsBool() (bool, error) { return n.x, nil } func (Bar) AsInt() (int64, error) { return mixins.Bool{TypeName: "gendemo.Bar"}.AsInt() } func (Bar) AsFloat() (float64, error) { return mixins.Bool{TypeName: "gendemo.Bar"}.AsFloat() } func (Bar) AsString() (string, error) { return mixins.Bool{TypeName: "gendemo.Bar"}.AsString() } func (Bar) AsBytes() ([]byte, error) { return mixins.Bool{TypeName: "gendemo.Bar"}.AsBytes() } func (Bar) AsLink() (datamodel.Link, error) { return mixins.Bool{TypeName: "gendemo.Bar"}.AsLink() } func (Bar) Prototype() datamodel.NodePrototype { return _Bar__Prototype{} } type _Bar__Prototype struct{} func (_Bar__Prototype) NewBuilder() datamodel.NodeBuilder { var nb _Bar__Builder nb.Reset() return &nb } type _Bar__Builder struct { _Bar__Assembler } func (nb *_Bar__Builder) Build() datamodel.Node { if *nb.m != schema.Maybe_Value { panic("invalid state: cannot call Build on an assembler that's not finished") } return nb.w } func (nb *_Bar__Builder) Reset() { var w _Bar var m schema.Maybe *nb = _Bar__Builder{_Bar__Assembler{w: &w, m: &m}} } type _Bar__Assembler struct { w *_Bar m *schema.Maybe } func (na *_Bar__Assembler) reset() {} func (_Bar__Assembler) BeginMap(sizeHint int64) (datamodel.MapAssembler, error) { return mixins.BoolAssembler{TypeName: "gendemo.Bar"}.BeginMap(0) } func (_Bar__Assembler) BeginList(sizeHint int64) (datamodel.ListAssembler, error) { return mixins.BoolAssembler{TypeName: "gendemo.Bar"}.BeginList(0) } func (na *_Bar__Assembler) AssignNull() error { switch *na.m { case allowNull: *na.m = schema.Maybe_Null return nil case schema.Maybe_Absent: return mixins.BoolAssembler{TypeName: "gendemo.Bar"}.AssignNull() case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") } panic("unreachable") } func (na *_Bar__Assembler) AssignBool(v bool) error { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") } na.w.x = v *na.m = schema.Maybe_Value return nil } func (_Bar__Assembler) AssignInt(int64) error { return mixins.BoolAssembler{TypeName: "gendemo.Bar"}.AssignInt(0) } func (_Bar__Assembler) AssignFloat(float64) error { return mixins.BoolAssembler{TypeName: "gendemo.Bar"}.AssignFloat(0) } func (_Bar__Assembler) AssignString(string) error { return mixins.BoolAssembler{TypeName: "gendemo.Bar"}.AssignString("") } func (_Bar__Assembler) AssignBytes([]byte) error { return mixins.BoolAssembler{TypeName: "gendemo.Bar"}.AssignBytes(nil) } func (_Bar__Assembler) AssignLink(datamodel.Link) error { return mixins.BoolAssembler{TypeName: "gendemo.Bar"}.AssignLink(nil) } func (na *_Bar__Assembler) AssignNode(v datamodel.Node) error { if v.IsNull() { return na.AssignNull() } if v2, ok := v.(*_Bar); ok { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") } *na.w = *v2 *na.m = schema.Maybe_Value return nil } if v2, err := v.AsBool(); err != nil { return err } else { return na.AssignBool(v2) } } func (_Bar__Assembler) Prototype() datamodel.NodePrototype { return _Bar__Prototype{} } func (Bar) Type() schema.Type { return nil /*TODO:typelit*/ } func (n Bar) Representation() datamodel.Node { return (*_Bar__Repr)(n) } type _Bar__Repr = _Bar var _ datamodel.Node = &_Bar__Repr{} type _Bar__ReprPrototype = _Bar__Prototype type _Bar__ReprAssembler = _Bar__Assembler func (n Baz) String() string { return n.x } func (_Baz__Prototype) fromString(w *_Baz, v string) error { *w = _Baz{v} return nil } func (_Baz__Prototype) FromString(v string) (Baz, error) { n := _Baz{v} return &n, nil } type _Baz__Maybe struct { m schema.Maybe v _Baz } type MaybeBaz = *_Baz__Maybe func (m MaybeBaz) IsNull() bool { return m.m == schema.Maybe_Null } func (m MaybeBaz) IsAbsent() bool { return m.m == schema.Maybe_Absent } func (m MaybeBaz) Exists() bool { return m.m == schema.Maybe_Value } func (m MaybeBaz) AsNode() datamodel.Node { switch m.m { case schema.Maybe_Absent: return datamodel.Absent case schema.Maybe_Null: return datamodel.Null case schema.Maybe_Value: return &m.v default: panic("unreachable") } } func (m MaybeBaz) Must() Baz { if !m.Exists() { panic("unbox of a maybe rejected") } return &m.v } var _ datamodel.Node = (Baz)(&_Baz{}) var _ schema.TypedNode = (Baz)(&_Baz{}) func (Baz) Kind() datamodel.Kind { return datamodel.Kind_String } func (Baz) LookupByString(string) (datamodel.Node, error) { return mixins.String{TypeName: "gendemo.Baz"}.LookupByString("") } func (Baz) LookupByNode(datamodel.Node) (datamodel.Node, error) { return mixins.String{TypeName: "gendemo.Baz"}.LookupByNode(nil) } func (Baz) LookupByIndex(idx int64) (datamodel.Node, error) { return mixins.String{TypeName: "gendemo.Baz"}.LookupByIndex(0) } func (Baz) LookupBySegment(seg datamodel.PathSegment) (datamodel.Node, error) { return mixins.String{TypeName: "gendemo.Baz"}.LookupBySegment(seg) } func (Baz) MapIterator() datamodel.MapIterator { return nil } func (Baz) ListIterator() datamodel.ListIterator { return nil } func (Baz) Length() int64 { return -1 } func (Baz) IsAbsent() bool { return false } func (Baz) IsNull() bool { return false } func (Baz) AsBool() (bool, error) { return mixins.String{TypeName: "gendemo.Baz"}.AsBool() } func (Baz) AsInt() (int64, error) { return mixins.String{TypeName: "gendemo.Baz"}.AsInt() } func (Baz) AsFloat() (float64, error) { return mixins.String{TypeName: "gendemo.Baz"}.AsFloat() } func (n Baz) AsString() (string, error) { return n.x, nil } func (Baz) AsBytes() ([]byte, error) { return mixins.String{TypeName: "gendemo.Baz"}.AsBytes() } func (Baz) AsLink() (datamodel.Link, error) { return mixins.String{TypeName: "gendemo.Baz"}.AsLink() } func (Baz) Prototype() datamodel.NodePrototype { return _Baz__Prototype{} } type _Baz__Prototype struct{} func (_Baz__Prototype) NewBuilder() datamodel.NodeBuilder { var nb _Baz__Builder nb.Reset() return &nb } type _Baz__Builder struct { _Baz__Assembler } func (nb *_Baz__Builder) Build() datamodel.Node { if *nb.m != schema.Maybe_Value { panic("invalid state: cannot call Build on an assembler that's not finished") } return nb.w } func (nb *_Baz__Builder) Reset() { var w _Baz var m schema.Maybe *nb = _Baz__Builder{_Baz__Assembler{w: &w, m: &m}} } type _Baz__Assembler struct { w *_Baz m *schema.Maybe } func (na *_Baz__Assembler) reset() {} func (_Baz__Assembler) BeginMap(sizeHint int64) (datamodel.MapAssembler, error) { return mixins.StringAssembler{TypeName: "gendemo.Baz"}.BeginMap(0) } func (_Baz__Assembler) BeginList(sizeHint int64) (datamodel.ListAssembler, error) { return mixins.StringAssembler{TypeName: "gendemo.Baz"}.BeginList(0) } func (na *_Baz__Assembler) AssignNull() error { switch *na.m { case allowNull: *na.m = schema.Maybe_Null return nil case schema.Maybe_Absent: return mixins.StringAssembler{TypeName: "gendemo.Baz"}.AssignNull() case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") } panic("unreachable") } func (_Baz__Assembler) AssignBool(bool) error { return mixins.StringAssembler{TypeName: "gendemo.Baz"}.AssignBool(false) } func (_Baz__Assembler) AssignInt(int64) error { return mixins.StringAssembler{TypeName: "gendemo.Baz"}.AssignInt(0) } func (_Baz__Assembler) AssignFloat(float64) error { return mixins.StringAssembler{TypeName: "gendemo.Baz"}.AssignFloat(0) } func (na *_Baz__Assembler) AssignString(v string) error { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") } na.w.x = v *na.m = schema.Maybe_Value return nil } func (_Baz__Assembler) AssignBytes([]byte) error { return mixins.StringAssembler{TypeName: "gendemo.Baz"}.AssignBytes(nil) } func (_Baz__Assembler) AssignLink(datamodel.Link) error { return mixins.StringAssembler{TypeName: "gendemo.Baz"}.AssignLink(nil) } func (na *_Baz__Assembler) AssignNode(v datamodel.Node) error { if v.IsNull() { return na.AssignNull() } if v2, ok := v.(*_Baz); ok { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") } *na.w = *v2 *na.m = schema.Maybe_Value return nil } if v2, err := v.AsString(); err != nil { return err } else { return na.AssignString(v2) } } func (_Baz__Assembler) Prototype() datamodel.NodePrototype { return _Baz__Prototype{} } func (Baz) Type() schema.Type { return nil /*TODO:typelit*/ } func (n Baz) Representation() datamodel.Node { return (*_Baz__Repr)(n) } type _Baz__Repr = _Baz var _ datamodel.Node = &_Baz__Repr{} type _Baz__ReprPrototype = _Baz__Prototype type _Baz__ReprAssembler = _Baz__Assembler func (n Foo) Int() int64 { return n.x } func (_Foo__Prototype) FromInt(v int64) (Foo, error) { n := _Foo{v} return &n, nil } type _Foo__Maybe struct { m schema.Maybe v _Foo } type MaybeFoo = *_Foo__Maybe func (m MaybeFoo) IsNull() bool { return m.m == schema.Maybe_Null } func (m MaybeFoo) IsAbsent() bool { return m.m == schema.Maybe_Absent } func (m MaybeFoo) Exists() bool { return m.m == schema.Maybe_Value } func (m MaybeFoo) AsNode() datamodel.Node { switch m.m { case schema.Maybe_Absent: return datamodel.Absent case schema.Maybe_Null: return datamodel.Null case schema.Maybe_Value: return &m.v default: panic("unreachable") } } func (m MaybeFoo) Must() Foo { if !m.Exists() { panic("unbox of a maybe rejected") } return &m.v } var _ datamodel.Node = (Foo)(&_Foo{}) var _ schema.TypedNode = (Foo)(&_Foo{}) func (Foo) Kind() datamodel.Kind { return datamodel.Kind_Int } func (Foo) LookupByString(string) (datamodel.Node, error) { return mixins.Int{TypeName: "gendemo.Foo"}.LookupByString("") } func (Foo) LookupByNode(datamodel.Node) (datamodel.Node, error) { return mixins.Int{TypeName: "gendemo.Foo"}.LookupByNode(nil) } func (Foo) LookupByIndex(idx int64) (datamodel.Node, error) { return mixins.Int{TypeName: "gendemo.Foo"}.LookupByIndex(0) } func (Foo) LookupBySegment(seg datamodel.PathSegment) (datamodel.Node, error) { return mixins.Int{TypeName: "gendemo.Foo"}.LookupBySegment(seg) } func (Foo) MapIterator() datamodel.MapIterator { return nil } func (Foo) ListIterator() datamodel.ListIterator { return nil } func (Foo) Length() int64 { return -1 } func (Foo) IsAbsent() bool { return false } func (Foo) IsNull() bool { return false } func (Foo) AsBool() (bool, error) { return mixins.Int{TypeName: "gendemo.Foo"}.AsBool() } func (n Foo) AsInt() (int64, error) { return n.x, nil } func (Foo) AsFloat() (float64, error) { return mixins.Int{TypeName: "gendemo.Foo"}.AsFloat() } func (Foo) AsString() (string, error) { return mixins.Int{TypeName: "gendemo.Foo"}.AsString() } func (Foo) AsBytes() ([]byte, error) { return mixins.Int{TypeName: "gendemo.Foo"}.AsBytes() } func (Foo) AsLink() (datamodel.Link, error) { return mixins.Int{TypeName: "gendemo.Foo"}.AsLink() } func (Foo) Prototype() datamodel.NodePrototype { return _Foo__Prototype{} } type _Foo__Prototype struct{} func (_Foo__Prototype) NewBuilder() datamodel.NodeBuilder { var nb _Foo__Builder nb.Reset() return &nb } type _Foo__Builder struct { _Foo__Assembler } func (nb *_Foo__Builder) Build() datamodel.Node { if *nb.m != schema.Maybe_Value { panic("invalid state: cannot call Build on an assembler that's not finished") } return nb.w } func (nb *_Foo__Builder) Reset() { var w _Foo var m schema.Maybe *nb = _Foo__Builder{_Foo__Assembler{w: &w, m: &m}} } type _Foo__Assembler struct { w *_Foo m *schema.Maybe } func (na *_Foo__Assembler) reset() {} func (_Foo__Assembler) BeginMap(sizeHint int64) (datamodel.MapAssembler, error) { return mixins.IntAssembler{TypeName: "gendemo.Foo"}.BeginMap(0) } func (_Foo__Assembler) BeginList(sizeHint int64) (datamodel.ListAssembler, error) { return mixins.IntAssembler{TypeName: "gendemo.Foo"}.BeginList(0) } func (na *_Foo__Assembler) AssignNull() error { switch *na.m { case allowNull: *na.m = schema.Maybe_Null return nil case schema.Maybe_Absent: return mixins.IntAssembler{TypeName: "gendemo.Foo"}.AssignNull() case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") } panic("unreachable") } func (_Foo__Assembler) AssignBool(bool) error { return mixins.IntAssembler{TypeName: "gendemo.Foo"}.AssignBool(false) } func (na *_Foo__Assembler) AssignInt(v int64) error { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") } na.w.x = v *na.m = schema.Maybe_Value return nil } func (_Foo__Assembler) AssignFloat(float64) error { return mixins.IntAssembler{TypeName: "gendemo.Foo"}.AssignFloat(0) } func (_Foo__Assembler) AssignString(string) error { return mixins.IntAssembler{TypeName: "gendemo.Foo"}.AssignString("") } func (_Foo__Assembler) AssignBytes([]byte) error { return mixins.IntAssembler{TypeName: "gendemo.Foo"}.AssignBytes(nil) } func (_Foo__Assembler) AssignLink(datamodel.Link) error { return mixins.IntAssembler{TypeName: "gendemo.Foo"}.AssignLink(nil) } func (na *_Foo__Assembler) AssignNode(v datamodel.Node) error { if v.IsNull() { return na.AssignNull() } if v2, ok := v.(*_Foo); ok { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") } *na.w = *v2 *na.m = schema.Maybe_Value return nil } if v2, err := v.AsInt(); err != nil { return err } else { return na.AssignInt(v2) } } func (_Foo__Assembler) Prototype() datamodel.NodePrototype { return _Foo__Prototype{} } func (Foo) Type() schema.Type { return nil /*TODO:typelit*/ } func (n Foo) Representation() datamodel.Node { return (*_Foo__Repr)(n) } type _Foo__Repr = _Foo var _ datamodel.Node = &_Foo__Repr{} type _Foo__ReprPrototype = _Foo__Prototype type _Foo__ReprAssembler = _Foo__Assembler func (n Int) Int() int64 { return n.x } func (_Int__Prototype) FromInt(v int64) (Int, error) { n := _Int{v} return &n, nil } type _Int__Maybe struct { m schema.Maybe v _Int } type MaybeInt = *_Int__Maybe func (m MaybeInt) IsNull() bool { return m.m == schema.Maybe_Null } func (m MaybeInt) IsAbsent() bool { return m.m == schema.Maybe_Absent } func (m MaybeInt) Exists() bool { return m.m == schema.Maybe_Value } func (m MaybeInt) AsNode() datamodel.Node { switch m.m { case schema.Maybe_Absent: return datamodel.Absent case schema.Maybe_Null: return datamodel.Null case schema.Maybe_Value: return &m.v default: panic("unreachable") } } func (m MaybeInt) Must() Int { if !m.Exists() { panic("unbox of a maybe rejected") } return &m.v } var _ datamodel.Node = (Int)(&_Int{}) var _ schema.TypedNode = (Int)(&_Int{}) func (Int) Kind() datamodel.Kind { return datamodel.Kind_Int } func (Int) LookupByString(string) (datamodel.Node, error) { return mixins.Int{TypeName: "gendemo.Int"}.LookupByString("") } func (Int) LookupByNode(datamodel.Node) (datamodel.Node, error) { return mixins.Int{TypeName: "gendemo.Int"}.LookupByNode(nil) } func (Int) LookupByIndex(idx int64) (datamodel.Node, error) { return mixins.Int{TypeName: "gendemo.Int"}.LookupByIndex(0) } func (Int) LookupBySegment(seg datamodel.PathSegment) (datamodel.Node, error) { return mixins.Int{TypeName: "gendemo.Int"}.LookupBySegment(seg) } func (Int) MapIterator() datamodel.MapIterator { return nil } func (Int) ListIterator() datamodel.ListIterator { return nil } func (Int) Length() int64 { return -1 } func (Int) IsAbsent() bool { return false } func (Int) IsNull() bool { return false } func (Int) AsBool() (bool, error) { return mixins.Int{TypeName: "gendemo.Int"}.AsBool() } func (n Int) AsInt() (int64, error) { return n.x, nil } func (Int) AsFloat() (float64, error) { return mixins.Int{TypeName: "gendemo.Int"}.AsFloat() } func (Int) AsString() (string, error) { return mixins.Int{TypeName: "gendemo.Int"}.AsString() } func (Int) AsBytes() ([]byte, error) { return mixins.Int{TypeName: "gendemo.Int"}.AsBytes() } func (Int) AsLink() (datamodel.Link, error) { return mixins.Int{TypeName: "gendemo.Int"}.AsLink() } func (Int) Prototype() datamodel.NodePrototype { return _Int__Prototype{} } type _Int__Prototype struct{} func (_Int__Prototype) NewBuilder() datamodel.NodeBuilder { var nb _Int__Builder nb.Reset() return &nb } type _Int__Builder struct { _Int__Assembler } func (nb *_Int__Builder) Build() datamodel.Node { if *nb.m != schema.Maybe_Value { panic("invalid state: cannot call Build on an assembler that's not finished") } return nb.w } func (nb *_Int__Builder) Reset() { var w _Int var m schema.Maybe *nb = _Int__Builder{_Int__Assembler{w: &w, m: &m}} } type _Int__Assembler struct { w *_Int m *schema.Maybe } func (na *_Int__Assembler) reset() {} func (_Int__Assembler) BeginMap(sizeHint int64) (datamodel.MapAssembler, error) { return mixins.IntAssembler{TypeName: "gendemo.Int"}.BeginMap(0) } func (_Int__Assembler) BeginList(sizeHint int64) (datamodel.ListAssembler, error) { return mixins.IntAssembler{TypeName: "gendemo.Int"}.BeginList(0) } func (na *_Int__Assembler) AssignNull() error { switch *na.m { case allowNull: *na.m = schema.Maybe_Null return nil case schema.Maybe_Absent: return mixins.IntAssembler{TypeName: "gendemo.Int"}.AssignNull() case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") } panic("unreachable") } func (_Int__Assembler) AssignBool(bool) error { return mixins.IntAssembler{TypeName: "gendemo.Int"}.AssignBool(false) } func (na *_Int__Assembler) AssignInt(v int64) error { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") } na.w.x = v *na.m = schema.Maybe_Value return nil } func (_Int__Assembler) AssignFloat(float64) error { return mixins.IntAssembler{TypeName: "gendemo.Int"}.AssignFloat(0) } func (_Int__Assembler) AssignString(string) error { return mixins.IntAssembler{TypeName: "gendemo.Int"}.AssignString("") } func (_Int__Assembler) AssignBytes([]byte) error { return mixins.IntAssembler{TypeName: "gendemo.Int"}.AssignBytes(nil) } func (_Int__Assembler) AssignLink(datamodel.Link) error { return mixins.IntAssembler{TypeName: "gendemo.Int"}.AssignLink(nil) } func (na *_Int__Assembler) AssignNode(v datamodel.Node) error { if v.IsNull() { return na.AssignNull() } if v2, ok := v.(*_Int); ok { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") } *na.w = *v2 *na.m = schema.Maybe_Value return nil } if v2, err := v.AsInt(); err != nil { return err } else { return na.AssignInt(v2) } } func (_Int__Assembler) Prototype() datamodel.NodePrototype { return _Int__Prototype{} } func (Int) Type() schema.Type { return nil /*TODO:typelit*/ } func (n Int) Representation() datamodel.Node { return (*_Int__Repr)(n) } type _Int__Repr = _Int var _ datamodel.Node = &_Int__Repr{} type _Int__ReprPrototype = _Int__Prototype type _Int__ReprAssembler = _Int__Assembler func (n *_Map__String__Msg3) Lookup(k String) Msg3 { v, exists := n.m[*k] if !exists { return nil } return v } func (n *_Map__String__Msg3) LookupMaybe(k String) MaybeMsg3 { v, exists := n.m[*k] if !exists { return &_Map__String__Msg3__valueAbsent } return &_Msg3__Maybe{ m: schema.Maybe_Value, v: v, } } var _Map__String__Msg3__valueAbsent = _Msg3__Maybe{m: schema.Maybe_Absent} func (n Map__String__Msg3) Iterator() *Map__String__Msg3__Itr { return &Map__String__Msg3__Itr{n, 0} } type Map__String__Msg3__Itr struct { n Map__String__Msg3 idx int } func (itr *Map__String__Msg3__Itr) Next() (k String, v Msg3) { if itr.idx >= len(itr.n.t) { return nil, nil } x := &itr.n.t[itr.idx] k = &x.k v = &x.v itr.idx++ return } func (itr *Map__String__Msg3__Itr) Done() bool { return itr.idx >= len(itr.n.t) } type _Map__String__Msg3__Maybe struct { m schema.Maybe v _Map__String__Msg3 } type MaybeMap__String__Msg3 = *_Map__String__Msg3__Maybe func (m MaybeMap__String__Msg3) IsNull() bool { return m.m == schema.Maybe_Null } func (m MaybeMap__String__Msg3) IsAbsent() bool { return m.m == schema.Maybe_Absent } func (m MaybeMap__String__Msg3) Exists() bool { return m.m == schema.Maybe_Value } func (m MaybeMap__String__Msg3) AsNode() datamodel.Node { switch m.m { case schema.Maybe_Absent: return datamodel.Absent case schema.Maybe_Null: return datamodel.Null case schema.Maybe_Value: return &m.v default: panic("unreachable") } } func (m MaybeMap__String__Msg3) Must() Map__String__Msg3 { if !m.Exists() { panic("unbox of a maybe rejected") } return &m.v } var _ datamodel.Node = (Map__String__Msg3)(&_Map__String__Msg3{}) var _ schema.TypedNode = (Map__String__Msg3)(&_Map__String__Msg3{}) func (Map__String__Msg3) Kind() datamodel.Kind { return datamodel.Kind_Map } func (n Map__String__Msg3) LookupByString(k string) (datamodel.Node, error) { var k2 _String if err := (_String__ReprPrototype{}).fromString(&k2, k); err != nil { return nil, err // TODO wrap in some kind of ErrInvalidKey } v, exists := n.m[k2] if !exists { return nil, datamodel.ErrNotExists{Segment: datamodel.PathSegmentOfString(k)} } return v, nil } func (n Map__String__Msg3) LookupByNode(k datamodel.Node) (datamodel.Node, error) { k2, ok := k.(String) if !ok { panic("todo invalid key type error") // 'schema.ErrInvalidKey{TypeName:"gendemo.Map__String__Msg3", Key:&_String{k}}' doesn't quite cut it: need room to explain the type, and it's not guaranteed k can be turned into a string at all } v, exists := n.m[*k2] if !exists { return nil, datamodel.ErrNotExists{Segment: datamodel.PathSegmentOfString(k2.String())} } return v, nil } func (Map__String__Msg3) LookupByIndex(idx int64) (datamodel.Node, error) { return mixins.Map{TypeName: "gendemo.Map__String__Msg3"}.LookupByIndex(0) } func (n Map__String__Msg3) LookupBySegment(seg datamodel.PathSegment) (datamodel.Node, error) { return n.LookupByString(seg.String()) } func (n Map__String__Msg3) MapIterator() datamodel.MapIterator { return &_Map__String__Msg3__MapItr{n, 0} } type _Map__String__Msg3__MapItr struct { n Map__String__Msg3 idx int } func (itr *_Map__String__Msg3__MapItr) Next() (k datamodel.Node, v datamodel.Node, _ error) { if itr.idx >= len(itr.n.t) { return nil, nil, datamodel.ErrIteratorOverread{} } x := &itr.n.t[itr.idx] k = &x.k v = &x.v itr.idx++ return } func (itr *_Map__String__Msg3__MapItr) Done() bool { return itr.idx >= len(itr.n.t) } func (Map__String__Msg3) ListIterator() datamodel.ListIterator { return nil } func (n Map__String__Msg3) Length() int64 { return int64(len(n.t)) } func (Map__String__Msg3) IsAbsent() bool { return false } func (Map__String__Msg3) IsNull() bool { return false } func (Map__String__Msg3) AsBool() (bool, error) { return mixins.Map{TypeName: "gendemo.Map__String__Msg3"}.AsBool() } func (Map__String__Msg3) AsInt() (int64, error) { return mixins.Map{TypeName: "gendemo.Map__String__Msg3"}.AsInt() } func (Map__String__Msg3) AsFloat() (float64, error) { return mixins.Map{TypeName: "gendemo.Map__String__Msg3"}.AsFloat() } func (Map__String__Msg3) AsString() (string, error) { return mixins.Map{TypeName: "gendemo.Map__String__Msg3"}.AsString() } func (Map__String__Msg3) AsBytes() ([]byte, error) { return mixins.Map{TypeName: "gendemo.Map__String__Msg3"}.AsBytes() } func (Map__String__Msg3) AsLink() (datamodel.Link, error) { return mixins.Map{TypeName: "gendemo.Map__String__Msg3"}.AsLink() } func (Map__String__Msg3) Prototype() datamodel.NodePrototype { return _Map__String__Msg3__Prototype{} } type _Map__String__Msg3__Prototype struct{} func (_Map__String__Msg3__Prototype) NewBuilder() datamodel.NodeBuilder { var nb _Map__String__Msg3__Builder nb.Reset() return &nb } type _Map__String__Msg3__Builder struct { _Map__String__Msg3__Assembler } func (nb *_Map__String__Msg3__Builder) Build() datamodel.Node { if *nb.m != schema.Maybe_Value { panic("invalid state: cannot call Build on an assembler that's not finished") } return nb.w } func (nb *_Map__String__Msg3__Builder) Reset() { var w _Map__String__Msg3 var m schema.Maybe *nb = _Map__String__Msg3__Builder{_Map__String__Msg3__Assembler{w: &w, m: &m}} } type _Map__String__Msg3__Assembler struct { w *_Map__String__Msg3 m *schema.Maybe state maState cm schema.Maybe ka _String__Assembler va _Msg3__Assembler } func (na *_Map__String__Msg3__Assembler) reset() { na.state = maState_initial na.ka.reset() na.va.reset() } func (na *_Map__String__Msg3__Assembler) BeginMap(sizeHint int64) (datamodel.MapAssembler, error) { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") case midvalue: panic("invalid state: it makes no sense to 'begin' twice on the same assembler!") } *na.m = midvalue if sizeHint < 0 { sizeHint = 0 } na.w.m = make(map[_String]*_Msg3, sizeHint) na.w.t = make([]_Map__String__Msg3__entry, 0, sizeHint) return na, nil } func (_Map__String__Msg3__Assembler) BeginList(sizeHint int64) (datamodel.ListAssembler, error) { return mixins.MapAssembler{TypeName: "gendemo.Map__String__Msg3"}.BeginList(0) } func (na *_Map__String__Msg3__Assembler) AssignNull() error { switch *na.m { case allowNull: *na.m = schema.Maybe_Null return nil case schema.Maybe_Absent: return mixins.MapAssembler{TypeName: "gendemo.Map__String__Msg3"}.AssignNull() case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") case midvalue: panic("invalid state: cannot assign null into an assembler that's already begun working on recursive structures!") } panic("unreachable") } func (_Map__String__Msg3__Assembler) AssignBool(bool) error { return mixins.MapAssembler{TypeName: "gendemo.Map__String__Msg3"}.AssignBool(false) } func (_Map__String__Msg3__Assembler) AssignInt(int64) error { return mixins.MapAssembler{TypeName: "gendemo.Map__String__Msg3"}.AssignInt(0) } func (_Map__String__Msg3__Assembler) AssignFloat(float64) error { return mixins.MapAssembler{TypeName: "gendemo.Map__String__Msg3"}.AssignFloat(0) } func (_Map__String__Msg3__Assembler) AssignString(string) error { return mixins.MapAssembler{TypeName: "gendemo.Map__String__Msg3"}.AssignString("") } func (_Map__String__Msg3__Assembler) AssignBytes([]byte) error { return mixins.MapAssembler{TypeName: "gendemo.Map__String__Msg3"}.AssignBytes(nil) } func (_Map__String__Msg3__Assembler) AssignLink(datamodel.Link) error { return mixins.MapAssembler{TypeName: "gendemo.Map__String__Msg3"}.AssignLink(nil) } func (na *_Map__String__Msg3__Assembler) AssignNode(v datamodel.Node) error { if v.IsNull() { return na.AssignNull() } if v2, ok := v.(*_Map__String__Msg3); ok { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") case midvalue: panic("invalid state: cannot assign null into an assembler that's already begun working on recursive structures!") } *na.w = *v2 *na.m = schema.Maybe_Value return nil } if v.Kind() != datamodel.Kind_Map { return datamodel.ErrWrongKind{TypeName: "gendemo.Map__String__Msg3", MethodName: "AssignNode", AppropriateKind: datamodel.KindSet_JustMap, ActualKind: v.Kind()} } itr := v.MapIterator() for !itr.Done() { k, v, err := itr.Next() if err != nil { return err } if err := na.AssembleKey().AssignNode(k); err != nil { return err } if err := na.AssembleValue().AssignNode(v); err != nil { return err } } return na.Finish() } func (_Map__String__Msg3__Assembler) Prototype() datamodel.NodePrototype { return _Map__String__Msg3__Prototype{} } func (ma *_Map__String__Msg3__Assembler) keyFinishTidy() bool { switch ma.cm { case schema.Maybe_Value: ma.ka.w = nil tz := &ma.w.t[len(ma.w.t)-1] ma.cm = schema.Maybe_Absent ma.state = maState_expectValue ma.w.m[tz.k] = &tz.v ma.va.w = &tz.v ma.va.m = &ma.cm ma.ka.reset() return true default: return false } } func (ma *_Map__String__Msg3__Assembler) valueFinishTidy() bool { switch ma.cm { case schema.Maybe_Value: ma.va.w = nil ma.cm = schema.Maybe_Absent ma.state = maState_initial ma.va.reset() return true default: return false } } func (ma *_Map__String__Msg3__Assembler) AssembleEntry(k string) (datamodel.NodeAssembler, error) { switch ma.state { case maState_initial: // carry on case maState_midKey: panic("invalid state: AssembleEntry cannot be called when in the middle of assembling another key") case maState_expectValue: panic("invalid state: AssembleEntry cannot be called when expecting start of value assembly") case maState_midValue: if !ma.valueFinishTidy() { panic("invalid state: AssembleEntry cannot be called when in the middle of assembling a value") } // if tidy success: carry on case maState_finished: panic("invalid state: AssembleEntry cannot be called on an assembler that's already finished") } var k2 _String if err := (_String__ReprPrototype{}).fromString(&k2, k); err != nil { return nil, err // TODO wrap in some kind of ErrInvalidKey } if _, exists := ma.w.m[k2]; exists { return nil, datamodel.ErrRepeatedMapKey{Key: &k2} } ma.w.t = append(ma.w.t, _Map__String__Msg3__entry{k: k2}) tz := &ma.w.t[len(ma.w.t)-1] ma.state = maState_midValue ma.w.m[k2] = &tz.v ma.va.w = &tz.v ma.va.m = &ma.cm return &ma.va, nil } func (ma *_Map__String__Msg3__Assembler) AssembleKey() datamodel.NodeAssembler { switch ma.state { case maState_initial: // carry on case maState_midKey: panic("invalid state: AssembleKey cannot be called when in the middle of assembling another key") case maState_expectValue: panic("invalid state: AssembleKey cannot be called when expecting start of value assembly") case maState_midValue: if !ma.valueFinishTidy() { panic("invalid state: AssembleKey cannot be called when in the middle of assembling a value") } // if tidy success: carry on case maState_finished: panic("invalid state: AssembleKey cannot be called on an assembler that's already finished") } ma.w.t = append(ma.w.t, _Map__String__Msg3__entry{}) ma.state = maState_midKey ma.ka.m = &ma.cm ma.ka.w = &ma.w.t[len(ma.w.t)-1].k return &ma.ka } func (ma *_Map__String__Msg3__Assembler) AssembleValue() datamodel.NodeAssembler { switch ma.state { case maState_initial: panic("invalid state: AssembleValue cannot be called when no key is primed") case maState_midKey: if !ma.keyFinishTidy() { panic("invalid state: AssembleValue cannot be called when in the middle of assembling a key") } // if tidy success: carry on case maState_expectValue: // carry on case maState_midValue: panic("invalid state: AssembleValue cannot be called when in the middle of assembling another value") case maState_finished: panic("invalid state: AssembleValue cannot be called on an assembler that's already finished") } ma.state = maState_midValue return &ma.va } func (ma *_Map__String__Msg3__Assembler) Finish() error { switch ma.state { case maState_initial: // carry on case maState_midKey: panic("invalid state: Finish cannot be called when in the middle of assembling a key") case maState_expectValue: panic("invalid state: Finish cannot be called when expecting start of value assembly") case maState_midValue: if !ma.valueFinishTidy() { panic("invalid state: Finish cannot be called when in the middle of assembling a value") } // if tidy success: carry on case maState_finished: panic("invalid state: Finish cannot be called on an assembler that's already finished") } ma.state = maState_finished *ma.m = schema.Maybe_Value return nil } func (ma *_Map__String__Msg3__Assembler) KeyPrototype() datamodel.NodePrototype { return _String__Prototype{} } func (ma *_Map__String__Msg3__Assembler) ValuePrototype(_ string) datamodel.NodePrototype { return _Msg3__Prototype{} } func (Map__String__Msg3) Type() schema.Type { return nil /*TODO:typelit*/ } func (n Map__String__Msg3) Representation() datamodel.Node { return (*_Map__String__Msg3__Repr)(n) } type _Map__String__Msg3__Repr _Map__String__Msg3 var _ datamodel.Node = &_Map__String__Msg3__Repr{} func (_Map__String__Msg3__Repr) Kind() datamodel.Kind { return datamodel.Kind_Map } func (nr *_Map__String__Msg3__Repr) LookupByString(k string) (datamodel.Node, error) { v, err := (Map__String__Msg3)(nr).LookupByString(k) if err != nil || v == datamodel.Null { return v, err } return v.(Msg3).Representation(), nil } func (nr *_Map__String__Msg3__Repr) LookupByNode(k datamodel.Node) (datamodel.Node, error) { v, err := (Map__String__Msg3)(nr).LookupByNode(k) if err != nil || v == datamodel.Null { return v, err } return v.(Msg3).Representation(), nil } func (_Map__String__Msg3__Repr) LookupByIndex(idx int64) (datamodel.Node, error) { return mixins.Map{TypeName: "gendemo.Map__String__Msg3.Repr"}.LookupByIndex(0) } func (n _Map__String__Msg3__Repr) LookupBySegment(seg datamodel.PathSegment) (datamodel.Node, error) { return n.LookupByString(seg.String()) } func (nr *_Map__String__Msg3__Repr) MapIterator() datamodel.MapIterator { return &_Map__String__Msg3__ReprMapItr{(Map__String__Msg3)(nr), 0} } type _Map__String__Msg3__ReprMapItr _Map__String__Msg3__MapItr func (itr *_Map__String__Msg3__ReprMapItr) Next() (k datamodel.Node, v datamodel.Node, err error) { k, v, err = (*_Map__String__Msg3__MapItr)(itr).Next() if err != nil || v == datamodel.Null { return } return k, v.(Msg3).Representation(), nil } func (itr *_Map__String__Msg3__ReprMapItr) Done() bool { return (*_Map__String__Msg3__MapItr)(itr).Done() } func (_Map__String__Msg3__Repr) ListIterator() datamodel.ListIterator { return nil } func (rn *_Map__String__Msg3__Repr) Length() int64 { return int64(len(rn.t)) } func (_Map__String__Msg3__Repr) IsAbsent() bool { return false } func (_Map__String__Msg3__Repr) IsNull() bool { return false } func (_Map__String__Msg3__Repr) AsBool() (bool, error) { return mixins.Map{TypeName: "gendemo.Map__String__Msg3.Repr"}.AsBool() } func (_Map__String__Msg3__Repr) AsInt() (int64, error) { return mixins.Map{TypeName: "gendemo.Map__String__Msg3.Repr"}.AsInt() } func (_Map__String__Msg3__Repr) AsFloat() (float64, error) { return mixins.Map{TypeName: "gendemo.Map__String__Msg3.Repr"}.AsFloat() } func (_Map__String__Msg3__Repr) AsString() (string, error) { return mixins.Map{TypeName: "gendemo.Map__String__Msg3.Repr"}.AsString() } func (_Map__String__Msg3__Repr) AsBytes() ([]byte, error) { return mixins.Map{TypeName: "gendemo.Map__String__Msg3.Repr"}.AsBytes() } func (_Map__String__Msg3__Repr) AsLink() (datamodel.Link, error) { return mixins.Map{TypeName: "gendemo.Map__String__Msg3.Repr"}.AsLink() } func (_Map__String__Msg3__Repr) Prototype() datamodel.NodePrototype { return _Map__String__Msg3__ReprPrototype{} } type _Map__String__Msg3__ReprPrototype struct{} func (_Map__String__Msg3__ReprPrototype) NewBuilder() datamodel.NodeBuilder { var nb _Map__String__Msg3__ReprBuilder nb.Reset() return &nb } type _Map__String__Msg3__ReprBuilder struct { _Map__String__Msg3__ReprAssembler } func (nb *_Map__String__Msg3__ReprBuilder) Build() datamodel.Node { if *nb.m != schema.Maybe_Value { panic("invalid state: cannot call Build on an assembler that's not finished") } return nb.w } func (nb *_Map__String__Msg3__ReprBuilder) Reset() { var w _Map__String__Msg3 var m schema.Maybe *nb = _Map__String__Msg3__ReprBuilder{_Map__String__Msg3__ReprAssembler{w: &w, m: &m}} } type _Map__String__Msg3__ReprAssembler struct { w *_Map__String__Msg3 m *schema.Maybe state maState cm schema.Maybe ka _String__ReprAssembler va _Msg3__ReprAssembler } func (na *_Map__String__Msg3__ReprAssembler) reset() { na.state = maState_initial na.ka.reset() na.va.reset() } func (na *_Map__String__Msg3__ReprAssembler) BeginMap(sizeHint int64) (datamodel.MapAssembler, error) { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") case midvalue: panic("invalid state: it makes no sense to 'begin' twice on the same assembler!") } *na.m = midvalue if sizeHint < 0 { sizeHint = 0 } na.w.m = make(map[_String]*_Msg3, sizeHint) na.w.t = make([]_Map__String__Msg3__entry, 0, sizeHint) return na, nil } func (_Map__String__Msg3__ReprAssembler) BeginList(sizeHint int64) (datamodel.ListAssembler, error) { return mixins.MapAssembler{TypeName: "gendemo.Map__String__Msg3.Repr"}.BeginList(0) } func (na *_Map__String__Msg3__ReprAssembler) AssignNull() error { switch *na.m { case allowNull: *na.m = schema.Maybe_Null return nil case schema.Maybe_Absent: return mixins.MapAssembler{TypeName: "gendemo.Map__String__Msg3.Repr.Repr"}.AssignNull() case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") case midvalue: panic("invalid state: cannot assign null into an assembler that's already begun working on recursive structures!") } panic("unreachable") } func (_Map__String__Msg3__ReprAssembler) AssignBool(bool) error { return mixins.MapAssembler{TypeName: "gendemo.Map__String__Msg3.Repr"}.AssignBool(false) } func (_Map__String__Msg3__ReprAssembler) AssignInt(int64) error { return mixins.MapAssembler{TypeName: "gendemo.Map__String__Msg3.Repr"}.AssignInt(0) } func (_Map__String__Msg3__ReprAssembler) AssignFloat(float64) error { return mixins.MapAssembler{TypeName: "gendemo.Map__String__Msg3.Repr"}.AssignFloat(0) } func (_Map__String__Msg3__ReprAssembler) AssignString(string) error { return mixins.MapAssembler{TypeName: "gendemo.Map__String__Msg3.Repr"}.AssignString("") } func (_Map__String__Msg3__ReprAssembler) AssignBytes([]byte) error { return mixins.MapAssembler{TypeName: "gendemo.Map__String__Msg3.Repr"}.AssignBytes(nil) } func (_Map__String__Msg3__ReprAssembler) AssignLink(datamodel.Link) error { return mixins.MapAssembler{TypeName: "gendemo.Map__String__Msg3.Repr"}.AssignLink(nil) } func (na *_Map__String__Msg3__ReprAssembler) AssignNode(v datamodel.Node) error { if v.IsNull() { return na.AssignNull() } if v2, ok := v.(*_Map__String__Msg3); ok { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") case midvalue: panic("invalid state: cannot assign null into an assembler that's already begun working on recursive structures!") } *na.w = *v2 *na.m = schema.Maybe_Value return nil } if v.Kind() != datamodel.Kind_Map { return datamodel.ErrWrongKind{TypeName: "gendemo.Map__String__Msg3.Repr", MethodName: "AssignNode", AppropriateKind: datamodel.KindSet_JustMap, ActualKind: v.Kind()} } itr := v.MapIterator() for !itr.Done() { k, v, err := itr.Next() if err != nil { return err } if err := na.AssembleKey().AssignNode(k); err != nil { return err } if err := na.AssembleValue().AssignNode(v); err != nil { return err } } return na.Finish() } func (_Map__String__Msg3__ReprAssembler) Prototype() datamodel.NodePrototype { return _Map__String__Msg3__ReprPrototype{} } func (ma *_Map__String__Msg3__ReprAssembler) keyFinishTidy() bool { switch ma.cm { case schema.Maybe_Value: ma.ka.w = nil tz := &ma.w.t[len(ma.w.t)-1] ma.cm = schema.Maybe_Absent ma.state = maState_expectValue ma.w.m[tz.k] = &tz.v ma.va.w = &tz.v ma.va.m = &ma.cm ma.ka.reset() return true default: return false } } func (ma *_Map__String__Msg3__ReprAssembler) valueFinishTidy() bool { switch ma.cm { case schema.Maybe_Value: ma.va.w = nil ma.cm = schema.Maybe_Absent ma.state = maState_initial ma.va.reset() return true default: return false } } func (ma *_Map__String__Msg3__ReprAssembler) AssembleEntry(k string) (datamodel.NodeAssembler, error) { switch ma.state { case maState_initial: // carry on case maState_midKey: panic("invalid state: AssembleEntry cannot be called when in the middle of assembling another key") case maState_expectValue: panic("invalid state: AssembleEntry cannot be called when expecting start of value assembly") case maState_midValue: if !ma.valueFinishTidy() { panic("invalid state: AssembleEntry cannot be called when in the middle of assembling a value") } // if tidy success: carry on case maState_finished: panic("invalid state: AssembleEntry cannot be called on an assembler that's already finished") } var k2 _String if err := (_String__ReprPrototype{}).fromString(&k2, k); err != nil { return nil, err // TODO wrap in some kind of ErrInvalidKey } if _, exists := ma.w.m[k2]; exists { return nil, datamodel.ErrRepeatedMapKey{Key: &k2} } ma.w.t = append(ma.w.t, _Map__String__Msg3__entry{k: k2}) tz := &ma.w.t[len(ma.w.t)-1] ma.state = maState_midValue ma.w.m[k2] = &tz.v ma.va.w = &tz.v ma.va.m = &ma.cm return &ma.va, nil } func (ma *_Map__String__Msg3__ReprAssembler) AssembleKey() datamodel.NodeAssembler { switch ma.state { case maState_initial: // carry on case maState_midKey: panic("invalid state: AssembleKey cannot be called when in the middle of assembling another key") case maState_expectValue: panic("invalid state: AssembleKey cannot be called when expecting start of value assembly") case maState_midValue: if !ma.valueFinishTidy() { panic("invalid state: AssembleKey cannot be called when in the middle of assembling a value") } // if tidy success: carry on case maState_finished: panic("invalid state: AssembleKey cannot be called on an assembler that's already finished") } ma.w.t = append(ma.w.t, _Map__String__Msg3__entry{}) ma.state = maState_midKey ma.ka.m = &ma.cm ma.ka.w = &ma.w.t[len(ma.w.t)-1].k return &ma.ka } func (ma *_Map__String__Msg3__ReprAssembler) AssembleValue() datamodel.NodeAssembler { switch ma.state { case maState_initial: panic("invalid state: AssembleValue cannot be called when no key is primed") case maState_midKey: if !ma.keyFinishTidy() { panic("invalid state: AssembleValue cannot be called when in the middle of assembling a key") } // if tidy success: carry on case maState_expectValue: // carry on case maState_midValue: panic("invalid state: AssembleValue cannot be called when in the middle of assembling another value") case maState_finished: panic("invalid state: AssembleValue cannot be called on an assembler that's already finished") } ma.state = maState_midValue return &ma.va } func (ma *_Map__String__Msg3__ReprAssembler) Finish() error { switch ma.state { case maState_initial: // carry on case maState_midKey: panic("invalid state: Finish cannot be called when in the middle of assembling a key") case maState_expectValue: panic("invalid state: Finish cannot be called when expecting start of value assembly") case maState_midValue: if !ma.valueFinishTidy() { panic("invalid state: Finish cannot be called when in the middle of assembling a value") } // if tidy success: carry on case maState_finished: panic("invalid state: Finish cannot be called on an assembler that's already finished") } ma.state = maState_finished *ma.m = schema.Maybe_Value return nil } func (ma *_Map__String__Msg3__ReprAssembler) KeyPrototype() datamodel.NodePrototype { return _String__ReprPrototype{} } func (ma *_Map__String__Msg3__ReprAssembler) ValuePrototype(_ string) datamodel.NodePrototype { return _Msg3__ReprPrototype{} } func (n _Msg3) FieldWhee() Int { return &n.whee } func (n _Msg3) FieldWoot() Int { return &n.woot } func (n _Msg3) FieldWaga() Int { return &n.waga } type _Msg3__Maybe struct { m schema.Maybe v Msg3 } type MaybeMsg3 = *_Msg3__Maybe func (m MaybeMsg3) IsNull() bool { return m.m == schema.Maybe_Null } func (m MaybeMsg3) IsAbsent() bool { return m.m == schema.Maybe_Absent } func (m MaybeMsg3) Exists() bool { return m.m == schema.Maybe_Value } func (m MaybeMsg3) AsNode() datamodel.Node { switch m.m { case schema.Maybe_Absent: return datamodel.Absent case schema.Maybe_Null: return datamodel.Null case schema.Maybe_Value: return m.v default: panic("unreachable") } } func (m MaybeMsg3) Must() Msg3 { if !m.Exists() { panic("unbox of a maybe rejected") } return m.v } var ( fieldName__Msg3_Whee = _String{"whee"} fieldName__Msg3_Woot = _String{"woot"} fieldName__Msg3_Waga = _String{"waga"} ) var _ datamodel.Node = (Msg3)(&_Msg3{}) var _ schema.TypedNode = (Msg3)(&_Msg3{}) func (Msg3) Kind() datamodel.Kind { return datamodel.Kind_Map } func (n Msg3) LookupByString(key string) (datamodel.Node, error) { switch key { case "whee": return &n.whee, nil case "woot": return &n.woot, nil case "waga": return &n.waga, nil default: return nil, schema.ErrNoSuchField{Type: nil /*TODO*/, Field: datamodel.PathSegmentOfString(key)} } } func (n Msg3) LookupByNode(key datamodel.Node) (datamodel.Node, error) { ks, err := key.AsString() if err != nil { return nil, err } return n.LookupByString(ks) } func (Msg3) LookupByIndex(idx int64) (datamodel.Node, error) { return mixins.Map{TypeName: "gendemo.Msg3"}.LookupByIndex(0) } func (n Msg3) LookupBySegment(seg datamodel.PathSegment) (datamodel.Node, error) { return n.LookupByString(seg.String()) } func (n Msg3) MapIterator() datamodel.MapIterator { return &_Msg3__MapItr{n, 0} } type _Msg3__MapItr struct { n Msg3 idx int } func (itr *_Msg3__MapItr) Next() (k datamodel.Node, v datamodel.Node, _ error) { if itr.idx >= 3 { return nil, nil, datamodel.ErrIteratorOverread{} } switch itr.idx { case 0: k = &fieldName__Msg3_Whee v = &itr.n.whee case 1: k = &fieldName__Msg3_Woot v = &itr.n.woot case 2: k = &fieldName__Msg3_Waga v = &itr.n.waga default: panic("unreachable") } itr.idx++ return } func (itr *_Msg3__MapItr) Done() bool { return itr.idx >= 3 } func (Msg3) ListIterator() datamodel.ListIterator { return nil } func (Msg3) Length() int64 { return 3 } func (Msg3) IsAbsent() bool { return false } func (Msg3) IsNull() bool { return false } func (Msg3) AsBool() (bool, error) { return mixins.Map{TypeName: "gendemo.Msg3"}.AsBool() } func (Msg3) AsInt() (int64, error) { return mixins.Map{TypeName: "gendemo.Msg3"}.AsInt() } func (Msg3) AsFloat() (float64, error) { return mixins.Map{TypeName: "gendemo.Msg3"}.AsFloat() } func (Msg3) AsString() (string, error) { return mixins.Map{TypeName: "gendemo.Msg3"}.AsString() } func (Msg3) AsBytes() ([]byte, error) { return mixins.Map{TypeName: "gendemo.Msg3"}.AsBytes() } func (Msg3) AsLink() (datamodel.Link, error) { return mixins.Map{TypeName: "gendemo.Msg3"}.AsLink() } func (Msg3) Prototype() datamodel.NodePrototype { return _Msg3__Prototype{} } type _Msg3__Prototype struct{} func (_Msg3__Prototype) NewBuilder() datamodel.NodeBuilder { var nb _Msg3__Builder nb.Reset() return &nb } type _Msg3__Builder struct { _Msg3__Assembler } func (nb *_Msg3__Builder) Build() datamodel.Node { if *nb.m != schema.Maybe_Value { panic("invalid state: cannot call Build on an assembler that's not finished") } return nb.w } func (nb *_Msg3__Builder) Reset() { var w _Msg3 var m schema.Maybe *nb = _Msg3__Builder{_Msg3__Assembler{w: &w, m: &m}} } type _Msg3__Assembler struct { w *_Msg3 m *schema.Maybe state maState s int f int cm schema.Maybe ca_whee _Int__Assembler ca_woot _Int__Assembler ca_waga _Int__Assembler } func (na *_Msg3__Assembler) reset() { na.state = maState_initial na.s = 0 na.ca_whee.reset() na.ca_woot.reset() na.ca_waga.reset() } var ( fieldBit__Msg3_Whee = 1 << 0 fieldBit__Msg3_Woot = 1 << 1 fieldBit__Msg3_Waga = 1 << 2 fieldBits__Msg3_sufficient = 0 + 1<<0 + 1<<1 + 1<<2 ) func (na *_Msg3__Assembler) BeginMap(int64) (datamodel.MapAssembler, error) { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") case midvalue: panic("invalid state: it makes no sense to 'begin' twice on the same assembler!") } *na.m = midvalue if na.w == nil { na.w = &_Msg3{} } return na, nil } func (_Msg3__Assembler) BeginList(sizeHint int64) (datamodel.ListAssembler, error) { return mixins.MapAssembler{TypeName: "gendemo.Msg3"}.BeginList(0) } func (na *_Msg3__Assembler) AssignNull() error { switch *na.m { case allowNull: *na.m = schema.Maybe_Null return nil case schema.Maybe_Absent: return mixins.MapAssembler{TypeName: "gendemo.Msg3"}.AssignNull() case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") case midvalue: panic("invalid state: cannot assign null into an assembler that's already begun working on recursive structures!") } panic("unreachable") } func (_Msg3__Assembler) AssignBool(bool) error { return mixins.MapAssembler{TypeName: "gendemo.Msg3"}.AssignBool(false) } func (_Msg3__Assembler) AssignInt(int64) error { return mixins.MapAssembler{TypeName: "gendemo.Msg3"}.AssignInt(0) } func (_Msg3__Assembler) AssignFloat(float64) error { return mixins.MapAssembler{TypeName: "gendemo.Msg3"}.AssignFloat(0) } func (_Msg3__Assembler) AssignString(string) error { return mixins.MapAssembler{TypeName: "gendemo.Msg3"}.AssignString("") } func (_Msg3__Assembler) AssignBytes([]byte) error { return mixins.MapAssembler{TypeName: "gendemo.Msg3"}.AssignBytes(nil) } func (_Msg3__Assembler) AssignLink(datamodel.Link) error { return mixins.MapAssembler{TypeName: "gendemo.Msg3"}.AssignLink(nil) } func (na *_Msg3__Assembler) AssignNode(v datamodel.Node) error { if v.IsNull() { return na.AssignNull() } if v2, ok := v.(*_Msg3); ok { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") case midvalue: panic("invalid state: cannot assign null into an assembler that's already begun working on recursive structures!") } if na.w == nil { na.w = v2 *na.m = schema.Maybe_Value return nil } *na.w = *v2 *na.m = schema.Maybe_Value return nil } if v.Kind() != datamodel.Kind_Map { return datamodel.ErrWrongKind{TypeName: "gendemo.Msg3", MethodName: "AssignNode", AppropriateKind: datamodel.KindSet_JustMap, ActualKind: v.Kind()} } itr := v.MapIterator() for !itr.Done() { k, v, err := itr.Next() if err != nil { return err } if err := na.AssembleKey().AssignNode(k); err != nil { return err } if err := na.AssembleValue().AssignNode(v); err != nil { return err } } return na.Finish() } func (_Msg3__Assembler) Prototype() datamodel.NodePrototype { return _Msg3__Prototype{} } func (ma *_Msg3__Assembler) valueFinishTidy() bool { switch ma.f { case 0: switch ma.cm { case schema.Maybe_Value: ma.ca_whee.w = nil ma.cm = schema.Maybe_Absent ma.state = maState_initial return true default: return false } case 1: switch ma.cm { case schema.Maybe_Value: ma.ca_woot.w = nil ma.cm = schema.Maybe_Absent ma.state = maState_initial return true default: return false } case 2: switch ma.cm { case schema.Maybe_Value: ma.ca_waga.w = nil ma.cm = schema.Maybe_Absent ma.state = maState_initial return true default: return false } default: panic("unreachable") } } func (ma *_Msg3__Assembler) AssembleEntry(k string) (datamodel.NodeAssembler, error) { switch ma.state { case maState_initial: // carry on case maState_midKey: panic("invalid state: AssembleEntry cannot be called when in the middle of assembling another key") case maState_expectValue: panic("invalid state: AssembleEntry cannot be called when expecting start of value assembly") case maState_midValue: if !ma.valueFinishTidy() { panic("invalid state: AssembleEntry cannot be called when in the middle of assembling a value") } // if tidy success: carry on case maState_finished: panic("invalid state: AssembleEntry cannot be called on an assembler that's already finished") } switch k { case "whee": if ma.s&fieldBit__Msg3_Whee != 0 { return nil, datamodel.ErrRepeatedMapKey{Key: &fieldName__Msg3_Whee} } ma.s += fieldBit__Msg3_Whee ma.state = maState_midValue ma.f = 0 ma.ca_whee.w = &ma.w.whee ma.ca_whee.m = &ma.cm return &ma.ca_whee, nil case "woot": if ma.s&fieldBit__Msg3_Woot != 0 { return nil, datamodel.ErrRepeatedMapKey{Key: &fieldName__Msg3_Woot} } ma.s += fieldBit__Msg3_Woot ma.state = maState_midValue ma.f = 1 ma.ca_woot.w = &ma.w.woot ma.ca_woot.m = &ma.cm return &ma.ca_woot, nil case "waga": if ma.s&fieldBit__Msg3_Waga != 0 { return nil, datamodel.ErrRepeatedMapKey{Key: &fieldName__Msg3_Waga} } ma.s += fieldBit__Msg3_Waga ma.state = maState_midValue ma.f = 2 ma.ca_waga.w = &ma.w.waga ma.ca_waga.m = &ma.cm return &ma.ca_waga, nil } return nil, schema.ErrInvalidKey{TypeName: "gendemo.Msg3", Key: &_String{k}} } func (ma *_Msg3__Assembler) AssembleKey() datamodel.NodeAssembler { switch ma.state { case maState_initial: // carry on case maState_midKey: panic("invalid state: AssembleKey cannot be called when in the middle of assembling another key") case maState_expectValue: panic("invalid state: AssembleKey cannot be called when expecting start of value assembly") case maState_midValue: if !ma.valueFinishTidy() { panic("invalid state: AssembleKey cannot be called when in the middle of assembling a value") } // if tidy success: carry on case maState_finished: panic("invalid state: AssembleKey cannot be called on an assembler that's already finished") } ma.state = maState_midKey return (*_Msg3__KeyAssembler)(ma) } func (ma *_Msg3__Assembler) AssembleValue() datamodel.NodeAssembler { switch ma.state { case maState_initial: panic("invalid state: AssembleValue cannot be called when no key is primed") case maState_midKey: panic("invalid state: AssembleValue cannot be called when in the middle of assembling a key") case maState_expectValue: // carry on case maState_midValue: panic("invalid state: AssembleValue cannot be called when in the middle of assembling another value") case maState_finished: panic("invalid state: AssembleValue cannot be called on an assembler that's already finished") } ma.state = maState_midValue switch ma.f { case 0: ma.ca_whee.w = &ma.w.whee ma.ca_whee.m = &ma.cm return &ma.ca_whee case 1: ma.ca_woot.w = &ma.w.woot ma.ca_woot.m = &ma.cm return &ma.ca_woot case 2: ma.ca_waga.w = &ma.w.waga ma.ca_waga.m = &ma.cm return &ma.ca_waga default: panic("unreachable") } } func (ma *_Msg3__Assembler) Finish() error { switch ma.state { case maState_initial: // carry on case maState_midKey: panic("invalid state: Finish cannot be called when in the middle of assembling a key") case maState_expectValue: panic("invalid state: Finish cannot be called when expecting start of value assembly") case maState_midValue: if !ma.valueFinishTidy() { panic("invalid state: Finish cannot be called when in the middle of assembling a value") } // if tidy success: carry on case maState_finished: panic("invalid state: Finish cannot be called on an assembler that's already finished") } if ma.s&fieldBits__Msg3_sufficient != fieldBits__Msg3_sufficient { err := schema.ErrMissingRequiredField{Missing: make([]string, 0)} if ma.s&fieldBit__Msg3_Whee == 0 { err.Missing = append(err.Missing, "whee") } if ma.s&fieldBit__Msg3_Woot == 0 { err.Missing = append(err.Missing, "woot") } if ma.s&fieldBit__Msg3_Waga == 0 { err.Missing = append(err.Missing, "waga") } return err } ma.state = maState_finished *ma.m = schema.Maybe_Value return nil } func (ma *_Msg3__Assembler) KeyPrototype() datamodel.NodePrototype { return _String__Prototype{} } func (ma *_Msg3__Assembler) ValuePrototype(k string) datamodel.NodePrototype { panic("todo structbuilder mapassembler valueprototype") } type _Msg3__KeyAssembler _Msg3__Assembler func (_Msg3__KeyAssembler) BeginMap(sizeHint int64) (datamodel.MapAssembler, error) { return mixins.StringAssembler{TypeName: "gendemo.Msg3.KeyAssembler"}.BeginMap(0) } func (_Msg3__KeyAssembler) BeginList(sizeHint int64) (datamodel.ListAssembler, error) { return mixins.StringAssembler{TypeName: "gendemo.Msg3.KeyAssembler"}.BeginList(0) } func (na *_Msg3__KeyAssembler) AssignNull() error { return mixins.StringAssembler{TypeName: "gendemo.Msg3.KeyAssembler"}.AssignNull() } func (_Msg3__KeyAssembler) AssignBool(bool) error { return mixins.StringAssembler{TypeName: "gendemo.Msg3.KeyAssembler"}.AssignBool(false) } func (_Msg3__KeyAssembler) AssignInt(int64) error { return mixins.StringAssembler{TypeName: "gendemo.Msg3.KeyAssembler"}.AssignInt(0) } func (_Msg3__KeyAssembler) AssignFloat(float64) error { return mixins.StringAssembler{TypeName: "gendemo.Msg3.KeyAssembler"}.AssignFloat(0) } func (ka *_Msg3__KeyAssembler) AssignString(k string) error { if ka.state != maState_midKey { panic("misuse: KeyAssembler held beyond its valid lifetime") } switch k { case "whee": if ka.s&fieldBit__Msg3_Whee != 0 { return datamodel.ErrRepeatedMapKey{Key: &fieldName__Msg3_Whee} } ka.s += fieldBit__Msg3_Whee ka.state = maState_expectValue ka.f = 0 return nil case "woot": if ka.s&fieldBit__Msg3_Woot != 0 { return datamodel.ErrRepeatedMapKey{Key: &fieldName__Msg3_Woot} } ka.s += fieldBit__Msg3_Woot ka.state = maState_expectValue ka.f = 1 return nil case "waga": if ka.s&fieldBit__Msg3_Waga != 0 { return datamodel.ErrRepeatedMapKey{Key: &fieldName__Msg3_Waga} } ka.s += fieldBit__Msg3_Waga ka.state = maState_expectValue ka.f = 2 return nil default: return schema.ErrInvalidKey{TypeName: "gendemo.Msg3", Key: &_String{k}} } } func (_Msg3__KeyAssembler) AssignBytes([]byte) error { return mixins.StringAssembler{TypeName: "gendemo.Msg3.KeyAssembler"}.AssignBytes(nil) } func (_Msg3__KeyAssembler) AssignLink(datamodel.Link) error { return mixins.StringAssembler{TypeName: "gendemo.Msg3.KeyAssembler"}.AssignLink(nil) } func (ka *_Msg3__KeyAssembler) AssignNode(v datamodel.Node) error { if v2, err := v.AsString(); err != nil { return err } else { return ka.AssignString(v2) } } func (_Msg3__KeyAssembler) Prototype() datamodel.NodePrototype { return _String__Prototype{} } func (Msg3) Type() schema.Type { return nil /*TODO:typelit*/ } func (n Msg3) Representation() datamodel.Node { return (*_Msg3__Repr)(n) } type _Msg3__Repr _Msg3 var ( fieldName__Msg3_Whee_serial = _String{"whee"} fieldName__Msg3_Woot_serial = _String{"woot"} fieldName__Msg3_Waga_serial = _String{"waga"} ) var _ datamodel.Node = &_Msg3__Repr{} func (_Msg3__Repr) Kind() datamodel.Kind { return datamodel.Kind_Map } func (n *_Msg3__Repr) LookupByString(key string) (datamodel.Node, error) { switch key { case "whee": return n.whee.Representation(), nil case "woot": return n.woot.Representation(), nil case "waga": return n.waga.Representation(), nil default: return nil, schema.ErrNoSuchField{Type: nil /*TODO*/, Field: datamodel.PathSegmentOfString(key)} } } func (n *_Msg3__Repr) LookupByNode(key datamodel.Node) (datamodel.Node, error) { ks, err := key.AsString() if err != nil { return nil, err } return n.LookupByString(ks) } func (_Msg3__Repr) LookupByIndex(idx int64) (datamodel.Node, error) { return mixins.Map{TypeName: "gendemo.Msg3.Repr"}.LookupByIndex(0) } func (n _Msg3__Repr) LookupBySegment(seg datamodel.PathSegment) (datamodel.Node, error) { return n.LookupByString(seg.String()) } func (n *_Msg3__Repr) MapIterator() datamodel.MapIterator { return &_Msg3__ReprMapItr{n, 0} } type _Msg3__ReprMapItr struct { n *_Msg3__Repr idx int } func (itr *_Msg3__ReprMapItr) Next() (k datamodel.Node, v datamodel.Node, _ error) { if itr.idx >= 3 { return nil, nil, datamodel.ErrIteratorOverread{} } switch itr.idx { case 0: k = &fieldName__Msg3_Whee_serial v = itr.n.whee.Representation() case 1: k = &fieldName__Msg3_Woot_serial v = itr.n.woot.Representation() case 2: k = &fieldName__Msg3_Waga_serial v = itr.n.waga.Representation() default: panic("unreachable") } itr.idx++ return } func (itr *_Msg3__ReprMapItr) Done() bool { return itr.idx >= 3 } func (_Msg3__Repr) ListIterator() datamodel.ListIterator { return nil } func (rn *_Msg3__Repr) Length() int64 { l := 3 return int64(l) } func (_Msg3__Repr) IsAbsent() bool { return false } func (_Msg3__Repr) IsNull() bool { return false } func (_Msg3__Repr) AsBool() (bool, error) { return mixins.Map{TypeName: "gendemo.Msg3.Repr"}.AsBool() } func (_Msg3__Repr) AsInt() (int64, error) { return mixins.Map{TypeName: "gendemo.Msg3.Repr"}.AsInt() } func (_Msg3__Repr) AsFloat() (float64, error) { return mixins.Map{TypeName: "gendemo.Msg3.Repr"}.AsFloat() } func (_Msg3__Repr) AsString() (string, error) { return mixins.Map{TypeName: "gendemo.Msg3.Repr"}.AsString() } func (_Msg3__Repr) AsBytes() ([]byte, error) { return mixins.Map{TypeName: "gendemo.Msg3.Repr"}.AsBytes() } func (_Msg3__Repr) AsLink() (datamodel.Link, error) { return mixins.Map{TypeName: "gendemo.Msg3.Repr"}.AsLink() } func (_Msg3__Repr) Prototype() datamodel.NodePrototype { return _Msg3__ReprPrototype{} } type _Msg3__ReprPrototype struct{} func (_Msg3__ReprPrototype) NewBuilder() datamodel.NodeBuilder { var nb _Msg3__ReprBuilder nb.Reset() return &nb } type _Msg3__ReprBuilder struct { _Msg3__ReprAssembler } func (nb *_Msg3__ReprBuilder) Build() datamodel.Node { if *nb.m != schema.Maybe_Value { panic("invalid state: cannot call Build on an assembler that's not finished") } return nb.w } func (nb *_Msg3__ReprBuilder) Reset() { var w _Msg3 var m schema.Maybe *nb = _Msg3__ReprBuilder{_Msg3__ReprAssembler{w: &w, m: &m}} } type _Msg3__ReprAssembler struct { w *_Msg3 m *schema.Maybe state maState s int f int cm schema.Maybe ca_whee _Int__ReprAssembler ca_woot _Int__ReprAssembler ca_waga _Int__ReprAssembler } func (na *_Msg3__ReprAssembler) reset() { na.state = maState_initial na.s = 0 na.ca_whee.reset() na.ca_woot.reset() na.ca_waga.reset() } func (na *_Msg3__ReprAssembler) BeginMap(int64) (datamodel.MapAssembler, error) { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") case midvalue: panic("invalid state: it makes no sense to 'begin' twice on the same assembler!") } *na.m = midvalue if na.w == nil { na.w = &_Msg3{} } return na, nil } func (_Msg3__ReprAssembler) BeginList(sizeHint int64) (datamodel.ListAssembler, error) { return mixins.MapAssembler{TypeName: "gendemo.Msg3.Repr"}.BeginList(0) } func (na *_Msg3__ReprAssembler) AssignNull() error { switch *na.m { case allowNull: *na.m = schema.Maybe_Null return nil case schema.Maybe_Absent: return mixins.MapAssembler{TypeName: "gendemo.Msg3.Repr.Repr"}.AssignNull() case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") case midvalue: panic("invalid state: cannot assign null into an assembler that's already begun working on recursive structures!") } panic("unreachable") } func (_Msg3__ReprAssembler) AssignBool(bool) error { return mixins.MapAssembler{TypeName: "gendemo.Msg3.Repr"}.AssignBool(false) } func (_Msg3__ReprAssembler) AssignInt(int64) error { return mixins.MapAssembler{TypeName: "gendemo.Msg3.Repr"}.AssignInt(0) } func (_Msg3__ReprAssembler) AssignFloat(float64) error { return mixins.MapAssembler{TypeName: "gendemo.Msg3.Repr"}.AssignFloat(0) } func (_Msg3__ReprAssembler) AssignString(string) error { return mixins.MapAssembler{TypeName: "gendemo.Msg3.Repr"}.AssignString("") } func (_Msg3__ReprAssembler) AssignBytes([]byte) error { return mixins.MapAssembler{TypeName: "gendemo.Msg3.Repr"}.AssignBytes(nil) } func (_Msg3__ReprAssembler) AssignLink(datamodel.Link) error { return mixins.MapAssembler{TypeName: "gendemo.Msg3.Repr"}.AssignLink(nil) } func (na *_Msg3__ReprAssembler) AssignNode(v datamodel.Node) error { if v.IsNull() { return na.AssignNull() } if v2, ok := v.(*_Msg3); ok { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") case midvalue: panic("invalid state: cannot assign null into an assembler that's already begun working on recursive structures!") } if na.w == nil { na.w = v2 *na.m = schema.Maybe_Value return nil } *na.w = *v2 *na.m = schema.Maybe_Value return nil } if v.Kind() != datamodel.Kind_Map { return datamodel.ErrWrongKind{TypeName: "gendemo.Msg3.Repr", MethodName: "AssignNode", AppropriateKind: datamodel.KindSet_JustMap, ActualKind: v.Kind()} } itr := v.MapIterator() for !itr.Done() { k, v, err := itr.Next() if err != nil { return err } if err := na.AssembleKey().AssignNode(k); err != nil { return err } if err := na.AssembleValue().AssignNode(v); err != nil { return err } } return na.Finish() } func (_Msg3__ReprAssembler) Prototype() datamodel.NodePrototype { return _Msg3__ReprPrototype{} } func (ma *_Msg3__ReprAssembler) valueFinishTidy() bool { switch ma.f { case 0: switch ma.cm { case schema.Maybe_Value: ma.cm = schema.Maybe_Absent ma.state = maState_initial return true default: return false } case 1: switch ma.cm { case schema.Maybe_Value: ma.cm = schema.Maybe_Absent ma.state = maState_initial return true default: return false } case 2: switch ma.cm { case schema.Maybe_Value: ma.cm = schema.Maybe_Absent ma.state = maState_initial return true default: return false } default: panic("unreachable") } } func (ma *_Msg3__ReprAssembler) AssembleEntry(k string) (datamodel.NodeAssembler, error) { switch ma.state { case maState_initial: // carry on case maState_midKey: panic("invalid state: AssembleEntry cannot be called when in the middle of assembling another key") case maState_expectValue: panic("invalid state: AssembleEntry cannot be called when expecting start of value assembly") case maState_midValue: if !ma.valueFinishTidy() { panic("invalid state: AssembleEntry cannot be called when in the middle of assembling a value") } // if tidy success: carry on case maState_finished: panic("invalid state: AssembleEntry cannot be called on an assembler that's already finished") } switch k { case "whee": if ma.s&fieldBit__Msg3_Whee != 0 { return nil, datamodel.ErrRepeatedMapKey{Key: &fieldName__Msg3_Whee_serial} } ma.s += fieldBit__Msg3_Whee ma.state = maState_midValue ma.f = 0 ma.ca_whee.w = &ma.w.whee ma.ca_whee.m = &ma.cm return &ma.ca_whee, nil case "woot": if ma.s&fieldBit__Msg3_Woot != 0 { return nil, datamodel.ErrRepeatedMapKey{Key: &fieldName__Msg3_Woot_serial} } ma.s += fieldBit__Msg3_Woot ma.state = maState_midValue ma.f = 1 ma.ca_woot.w = &ma.w.woot ma.ca_woot.m = &ma.cm return &ma.ca_woot, nil case "waga": if ma.s&fieldBit__Msg3_Waga != 0 { return nil, datamodel.ErrRepeatedMapKey{Key: &fieldName__Msg3_Waga_serial} } ma.s += fieldBit__Msg3_Waga ma.state = maState_midValue ma.f = 2 ma.ca_waga.w = &ma.w.waga ma.ca_waga.m = &ma.cm return &ma.ca_waga, nil default: } return nil, schema.ErrInvalidKey{TypeName: "gendemo.Msg3.Repr", Key: &_String{k}} } func (ma *_Msg3__ReprAssembler) AssembleKey() datamodel.NodeAssembler { switch ma.state { case maState_initial: // carry on case maState_midKey: panic("invalid state: AssembleKey cannot be called when in the middle of assembling another key") case maState_expectValue: panic("invalid state: AssembleKey cannot be called when expecting start of value assembly") case maState_midValue: if !ma.valueFinishTidy() { panic("invalid state: AssembleKey cannot be called when in the middle of assembling a value") } // if tidy success: carry on case maState_finished: panic("invalid state: AssembleKey cannot be called on an assembler that's already finished") } ma.state = maState_midKey return (*_Msg3__ReprKeyAssembler)(ma) } func (ma *_Msg3__ReprAssembler) AssembleValue() datamodel.NodeAssembler { switch ma.state { case maState_initial: panic("invalid state: AssembleValue cannot be called when no key is primed") case maState_midKey: panic("invalid state: AssembleValue cannot be called when in the middle of assembling a key") case maState_expectValue: // carry on case maState_midValue: panic("invalid state: AssembleValue cannot be called when in the middle of assembling another value") case maState_finished: panic("invalid state: AssembleValue cannot be called on an assembler that's already finished") } ma.state = maState_midValue switch ma.f { case 0: ma.ca_whee.w = &ma.w.whee ma.ca_whee.m = &ma.cm return &ma.ca_whee case 1: ma.ca_woot.w = &ma.w.woot ma.ca_woot.m = &ma.cm return &ma.ca_woot case 2: ma.ca_waga.w = &ma.w.waga ma.ca_waga.m = &ma.cm return &ma.ca_waga default: panic("unreachable") } } func (ma *_Msg3__ReprAssembler) Finish() error { switch ma.state { case maState_initial: // carry on case maState_midKey: panic("invalid state: Finish cannot be called when in the middle of assembling a key") case maState_expectValue: panic("invalid state: Finish cannot be called when expecting start of value assembly") case maState_midValue: if !ma.valueFinishTidy() { panic("invalid state: Finish cannot be called when in the middle of assembling a value") } // if tidy success: carry on case maState_finished: panic("invalid state: Finish cannot be called on an assembler that's already finished") } if ma.s&fieldBits__Msg3_sufficient != fieldBits__Msg3_sufficient { err := schema.ErrMissingRequiredField{Missing: make([]string, 0)} if ma.s&fieldBit__Msg3_Whee == 0 { err.Missing = append(err.Missing, "whee") } if ma.s&fieldBit__Msg3_Woot == 0 { err.Missing = append(err.Missing, "woot") } if ma.s&fieldBit__Msg3_Waga == 0 { err.Missing = append(err.Missing, "waga") } return err } ma.state = maState_finished *ma.m = schema.Maybe_Value return nil } func (ma *_Msg3__ReprAssembler) KeyPrototype() datamodel.NodePrototype { return _String__Prototype{} } func (ma *_Msg3__ReprAssembler) ValuePrototype(k string) datamodel.NodePrototype { panic("todo structbuilder mapassembler repr valueprototype") } type _Msg3__ReprKeyAssembler _Msg3__ReprAssembler func (_Msg3__ReprKeyAssembler) BeginMap(sizeHint int64) (datamodel.MapAssembler, error) { return mixins.StringAssembler{TypeName: "gendemo.Msg3.Repr.KeyAssembler"}.BeginMap(0) } func (_Msg3__ReprKeyAssembler) BeginList(sizeHint int64) (datamodel.ListAssembler, error) { return mixins.StringAssembler{TypeName: "gendemo.Msg3.Repr.KeyAssembler"}.BeginList(0) } func (na *_Msg3__ReprKeyAssembler) AssignNull() error { return mixins.StringAssembler{TypeName: "gendemo.Msg3.Repr.KeyAssembler"}.AssignNull() } func (_Msg3__ReprKeyAssembler) AssignBool(bool) error { return mixins.StringAssembler{TypeName: "gendemo.Msg3.Repr.KeyAssembler"}.AssignBool(false) } func (_Msg3__ReprKeyAssembler) AssignInt(int64) error { return mixins.StringAssembler{TypeName: "gendemo.Msg3.Repr.KeyAssembler"}.AssignInt(0) } func (_Msg3__ReprKeyAssembler) AssignFloat(float64) error { return mixins.StringAssembler{TypeName: "gendemo.Msg3.Repr.KeyAssembler"}.AssignFloat(0) } func (ka *_Msg3__ReprKeyAssembler) AssignString(k string) error { if ka.state != maState_midKey { panic("misuse: KeyAssembler held beyond its valid lifetime") } switch k { case "whee": if ka.s&fieldBit__Msg3_Whee != 0 { return datamodel.ErrRepeatedMapKey{Key: &fieldName__Msg3_Whee_serial} } ka.s += fieldBit__Msg3_Whee ka.state = maState_expectValue ka.f = 0 return nil case "woot": if ka.s&fieldBit__Msg3_Woot != 0 { return datamodel.ErrRepeatedMapKey{Key: &fieldName__Msg3_Woot_serial} } ka.s += fieldBit__Msg3_Woot ka.state = maState_expectValue ka.f = 1 return nil case "waga": if ka.s&fieldBit__Msg3_Waga != 0 { return datamodel.ErrRepeatedMapKey{Key: &fieldName__Msg3_Waga_serial} } ka.s += fieldBit__Msg3_Waga ka.state = maState_expectValue ka.f = 2 return nil } return schema.ErrInvalidKey{TypeName: "gendemo.Msg3.Repr", Key: &_String{k}} } func (_Msg3__ReprKeyAssembler) AssignBytes([]byte) error { return mixins.StringAssembler{TypeName: "gendemo.Msg3.Repr.KeyAssembler"}.AssignBytes(nil) } func (_Msg3__ReprKeyAssembler) AssignLink(datamodel.Link) error { return mixins.StringAssembler{TypeName: "gendemo.Msg3.Repr.KeyAssembler"}.AssignLink(nil) } func (ka *_Msg3__ReprKeyAssembler) AssignNode(v datamodel.Node) error { if v2, err := v.AsString(); err != nil { return err } else { return ka.AssignString(v2) } } func (_Msg3__ReprKeyAssembler) Prototype() datamodel.NodePrototype { return _String__Prototype{} } func (n String) String() string { return n.x } func (_String__Prototype) fromString(w *_String, v string) error { *w = _String{v} return nil } func (_String__Prototype) FromString(v string) (String, error) { n := _String{v} return &n, nil } type _String__Maybe struct { m schema.Maybe v _String } type MaybeString = *_String__Maybe func (m MaybeString) IsNull() bool { return m.m == schema.Maybe_Null } func (m MaybeString) IsAbsent() bool { return m.m == schema.Maybe_Absent } func (m MaybeString) Exists() bool { return m.m == schema.Maybe_Value } func (m MaybeString) AsNode() datamodel.Node { switch m.m { case schema.Maybe_Absent: return datamodel.Absent case schema.Maybe_Null: return datamodel.Null case schema.Maybe_Value: return &m.v default: panic("unreachable") } } func (m MaybeString) Must() String { if !m.Exists() { panic("unbox of a maybe rejected") } return &m.v } var _ datamodel.Node = (String)(&_String{}) var _ schema.TypedNode = (String)(&_String{}) func (String) Kind() datamodel.Kind { return datamodel.Kind_String } func (String) LookupByString(string) (datamodel.Node, error) { return mixins.String{TypeName: "gendemo.String"}.LookupByString("") } func (String) LookupByNode(datamodel.Node) (datamodel.Node, error) { return mixins.String{TypeName: "gendemo.String"}.LookupByNode(nil) } func (String) LookupByIndex(idx int64) (datamodel.Node, error) { return mixins.String{TypeName: "gendemo.String"}.LookupByIndex(0) } func (String) LookupBySegment(seg datamodel.PathSegment) (datamodel.Node, error) { return mixins.String{TypeName: "gendemo.String"}.LookupBySegment(seg) } func (String) MapIterator() datamodel.MapIterator { return nil } func (String) ListIterator() datamodel.ListIterator { return nil } func (String) Length() int64 { return -1 } func (String) IsAbsent() bool { return false } func (String) IsNull() bool { return false } func (String) AsBool() (bool, error) { return mixins.String{TypeName: "gendemo.String"}.AsBool() } func (String) AsInt() (int64, error) { return mixins.String{TypeName: "gendemo.String"}.AsInt() } func (String) AsFloat() (float64, error) { return mixins.String{TypeName: "gendemo.String"}.AsFloat() } func (n String) AsString() (string, error) { return n.x, nil } func (String) AsBytes() ([]byte, error) { return mixins.String{TypeName: "gendemo.String"}.AsBytes() } func (String) AsLink() (datamodel.Link, error) { return mixins.String{TypeName: "gendemo.String"}.AsLink() } func (String) Prototype() datamodel.NodePrototype { return _String__Prototype{} } type _String__Prototype struct{} func (_String__Prototype) NewBuilder() datamodel.NodeBuilder { var nb _String__Builder nb.Reset() return &nb } type _String__Builder struct { _String__Assembler } func (nb *_String__Builder) Build() datamodel.Node { if *nb.m != schema.Maybe_Value { panic("invalid state: cannot call Build on an assembler that's not finished") } return nb.w } func (nb *_String__Builder) Reset() { var w _String var m schema.Maybe *nb = _String__Builder{_String__Assembler{w: &w, m: &m}} } type _String__Assembler struct { w *_String m *schema.Maybe } func (na *_String__Assembler) reset() {} func (_String__Assembler) BeginMap(sizeHint int64) (datamodel.MapAssembler, error) { return mixins.StringAssembler{TypeName: "gendemo.String"}.BeginMap(0) } func (_String__Assembler) BeginList(sizeHint int64) (datamodel.ListAssembler, error) { return mixins.StringAssembler{TypeName: "gendemo.String"}.BeginList(0) } func (na *_String__Assembler) AssignNull() error { switch *na.m { case allowNull: *na.m = schema.Maybe_Null return nil case schema.Maybe_Absent: return mixins.StringAssembler{TypeName: "gendemo.String"}.AssignNull() case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") } panic("unreachable") } func (_String__Assembler) AssignBool(bool) error { return mixins.StringAssembler{TypeName: "gendemo.String"}.AssignBool(false) } func (_String__Assembler) AssignInt(int64) error { return mixins.StringAssembler{TypeName: "gendemo.String"}.AssignInt(0) } func (_String__Assembler) AssignFloat(float64) error { return mixins.StringAssembler{TypeName: "gendemo.String"}.AssignFloat(0) } func (na *_String__Assembler) AssignString(v string) error { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") } na.w.x = v *na.m = schema.Maybe_Value return nil } func (_String__Assembler) AssignBytes([]byte) error { return mixins.StringAssembler{TypeName: "gendemo.String"}.AssignBytes(nil) } func (_String__Assembler) AssignLink(datamodel.Link) error { return mixins.StringAssembler{TypeName: "gendemo.String"}.AssignLink(nil) } func (na *_String__Assembler) AssignNode(v datamodel.Node) error { if v.IsNull() { return na.AssignNull() } if v2, ok := v.(*_String); ok { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") } *na.w = *v2 *na.m = schema.Maybe_Value return nil } if v2, err := v.AsString(); err != nil { return err } else { return na.AssignString(v2) } } func (_String__Assembler) Prototype() datamodel.NodePrototype { return _String__Prototype{} } func (String) Type() schema.Type { return nil /*TODO:typelit*/ } func (n String) Representation() datamodel.Node { return (*_String__Repr)(n) } type _String__Repr = _String var _ datamodel.Node = &_String__Repr{} type _String__ReprPrototype = _String__Prototype type _String__ReprAssembler = _String__Assembler func (n _UnionKinded) AsInterface() _UnionKinded__iface { switch n.tag { case 1: return &n.x1 case 2: return &n.x2 case 3: return &n.x3 default: panic("invalid union state; how did you create this object?") } } type _UnionKinded__Maybe struct { m schema.Maybe v UnionKinded } type MaybeUnionKinded = *_UnionKinded__Maybe func (m MaybeUnionKinded) IsNull() bool { return m.m == schema.Maybe_Null } func (m MaybeUnionKinded) IsAbsent() bool { return m.m == schema.Maybe_Absent } func (m MaybeUnionKinded) Exists() bool { return m.m == schema.Maybe_Value } func (m MaybeUnionKinded) AsNode() datamodel.Node { switch m.m { case schema.Maybe_Absent: return datamodel.Absent case schema.Maybe_Null: return datamodel.Null case schema.Maybe_Value: return m.v default: panic("unreachable") } } func (m MaybeUnionKinded) Must() UnionKinded { if !m.Exists() { panic("unbox of a maybe rejected") } return m.v } var ( memberName__UnionKinded_Foo = _String{"Foo"} memberName__UnionKinded_Bar = _String{"Bar"} memberName__UnionKinded_Baz = _String{"Baz"} ) var _ datamodel.Node = (UnionKinded)(&_UnionKinded{}) var _ schema.TypedNode = (UnionKinded)(&_UnionKinded{}) func (UnionKinded) Kind() datamodel.Kind { return datamodel.Kind_Map } func (n UnionKinded) LookupByString(key string) (datamodel.Node, error) { switch key { case "Foo": if n.tag != 1 { return nil, datamodel.ErrNotExists{Segment: datamodel.PathSegmentOfString(key)} } return &n.x1, nil case "Bar": if n.tag != 2 { return nil, datamodel.ErrNotExists{Segment: datamodel.PathSegmentOfString(key)} } return &n.x2, nil case "Baz": if n.tag != 3 { return nil, datamodel.ErrNotExists{Segment: datamodel.PathSegmentOfString(key)} } return &n.x3, nil default: return nil, schema.ErrNoSuchField{Type: nil /*TODO*/, Field: datamodel.PathSegmentOfString(key)} } } func (n UnionKinded) LookupByNode(key datamodel.Node) (datamodel.Node, error) { ks, err := key.AsString() if err != nil { return nil, err } return n.LookupByString(ks) } func (UnionKinded) LookupByIndex(idx int64) (datamodel.Node, error) { return mixins.Map{TypeName: "gendemo.UnionKinded"}.LookupByIndex(0) } func (n UnionKinded) LookupBySegment(seg datamodel.PathSegment) (datamodel.Node, error) { return n.LookupByString(seg.String()) } func (n UnionKinded) MapIterator() datamodel.MapIterator { return &_UnionKinded__MapItr{n, false} } type _UnionKinded__MapItr struct { n UnionKinded done bool } func (itr *_UnionKinded__MapItr) Next() (k datamodel.Node, v datamodel.Node, _ error) { if itr.done { return nil, nil, datamodel.ErrIteratorOverread{} } switch itr.n.tag { case 1: k, v = &memberName__UnionKinded_Foo, &itr.n.x1 case 2: k, v = &memberName__UnionKinded_Bar, &itr.n.x2 case 3: k, v = &memberName__UnionKinded_Baz, &itr.n.x3 default: panic("unreachable") } itr.done = true return } func (itr *_UnionKinded__MapItr) Done() bool { return itr.done } func (UnionKinded) ListIterator() datamodel.ListIterator { return nil } func (UnionKinded) Length() int64 { return 1 } func (UnionKinded) IsAbsent() bool { return false } func (UnionKinded) IsNull() bool { return false } func (UnionKinded) AsBool() (bool, error) { return mixins.Map{TypeName: "gendemo.UnionKinded"}.AsBool() } func (UnionKinded) AsInt() (int64, error) { return mixins.Map{TypeName: "gendemo.UnionKinded"}.AsInt() } func (UnionKinded) AsFloat() (float64, error) { return mixins.Map{TypeName: "gendemo.UnionKinded"}.AsFloat() } func (UnionKinded) AsString() (string, error) { return mixins.Map{TypeName: "gendemo.UnionKinded"}.AsString() } func (UnionKinded) AsBytes() ([]byte, error) { return mixins.Map{TypeName: "gendemo.UnionKinded"}.AsBytes() } func (UnionKinded) AsLink() (datamodel.Link, error) { return mixins.Map{TypeName: "gendemo.UnionKinded"}.AsLink() } func (UnionKinded) Prototype() datamodel.NodePrototype { return _UnionKinded__Prototype{} } type _UnionKinded__Prototype struct{} func (_UnionKinded__Prototype) NewBuilder() datamodel.NodeBuilder { var nb _UnionKinded__Builder nb.Reset() return &nb } type _UnionKinded__Builder struct { _UnionKinded__Assembler } func (nb *_UnionKinded__Builder) Build() datamodel.Node { if *nb.m != schema.Maybe_Value { panic("invalid state: cannot call Build on an assembler that's not finished") } return nb.w } func (nb *_UnionKinded__Builder) Reset() { var w _UnionKinded var m schema.Maybe *nb = _UnionKinded__Builder{_UnionKinded__Assembler{w: &w, m: &m}} } type _UnionKinded__Assembler struct { w *_UnionKinded m *schema.Maybe state maState cm schema.Maybe ca1 _Foo__Assembler ca2 _Bar__Assembler ca3 _Baz__Assembler ca uint } func (na *_UnionKinded__Assembler) reset() { na.state = maState_initial switch na.ca { case 0: return case 1: na.ca1.reset() case 2: na.ca2.reset() case 3: na.ca3.reset() default: panic("unreachable") } na.ca = 0 na.cm = schema.Maybe_Absent } func (na *_UnionKinded__Assembler) BeginMap(int64) (datamodel.MapAssembler, error) { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") case midvalue: panic("invalid state: it makes no sense to 'begin' twice on the same assembler!") } *na.m = midvalue if na.w == nil { na.w = &_UnionKinded{} } return na, nil } func (_UnionKinded__Assembler) BeginList(sizeHint int64) (datamodel.ListAssembler, error) { return mixins.MapAssembler{TypeName: "gendemo.UnionKinded"}.BeginList(0) } func (na *_UnionKinded__Assembler) AssignNull() error { switch *na.m { case allowNull: *na.m = schema.Maybe_Null return nil case schema.Maybe_Absent: return mixins.MapAssembler{TypeName: "gendemo.UnionKinded"}.AssignNull() case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") case midvalue: panic("invalid state: cannot assign null into an assembler that's already begun working on recursive structures!") } panic("unreachable") } func (_UnionKinded__Assembler) AssignBool(bool) error { return mixins.MapAssembler{TypeName: "gendemo.UnionKinded"}.AssignBool(false) } func (_UnionKinded__Assembler) AssignInt(int64) error { return mixins.MapAssembler{TypeName: "gendemo.UnionKinded"}.AssignInt(0) } func (_UnionKinded__Assembler) AssignFloat(float64) error { return mixins.MapAssembler{TypeName: "gendemo.UnionKinded"}.AssignFloat(0) } func (_UnionKinded__Assembler) AssignString(string) error { return mixins.MapAssembler{TypeName: "gendemo.UnionKinded"}.AssignString("") } func (_UnionKinded__Assembler) AssignBytes([]byte) error { return mixins.MapAssembler{TypeName: "gendemo.UnionKinded"}.AssignBytes(nil) } func (_UnionKinded__Assembler) AssignLink(datamodel.Link) error { return mixins.MapAssembler{TypeName: "gendemo.UnionKinded"}.AssignLink(nil) } func (na *_UnionKinded__Assembler) AssignNode(v datamodel.Node) error { if v.IsNull() { return na.AssignNull() } if v2, ok := v.(*_UnionKinded); ok { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") case midvalue: panic("invalid state: cannot assign null into an assembler that's already begun working on recursive structures!") } if na.w == nil { na.w = v2 *na.m = schema.Maybe_Value return nil } *na.w = *v2 *na.m = schema.Maybe_Value return nil } if v.Kind() != datamodel.Kind_Map { return datamodel.ErrWrongKind{TypeName: "gendemo.UnionKinded", MethodName: "AssignNode", AppropriateKind: datamodel.KindSet_JustMap, ActualKind: v.Kind()} } itr := v.MapIterator() for !itr.Done() { k, v, err := itr.Next() if err != nil { return err } if err := na.AssembleKey().AssignNode(k); err != nil { return err } if err := na.AssembleValue().AssignNode(v); err != nil { return err } } return na.Finish() } func (_UnionKinded__Assembler) Prototype() datamodel.NodePrototype { return _UnionKinded__Prototype{} } func (ma *_UnionKinded__Assembler) valueFinishTidy() bool { switch ma.cm { case schema.Maybe_Value: ma.state = maState_initial return true default: return false } } func (ma *_UnionKinded__Assembler) AssembleEntry(k string) (datamodel.NodeAssembler, error) { switch ma.state { case maState_initial: // carry on case maState_midKey: panic("invalid state: AssembleEntry cannot be called when in the middle of assembling another key") case maState_expectValue: panic("invalid state: AssembleEntry cannot be called when expecting start of value assembly") case maState_midValue: if !ma.valueFinishTidy() { panic("invalid state: AssembleEntry cannot be called when in the middle of assembling a value") } // if tidy success: carry on for the moment, but we'll still be erroring shortly. case maState_finished: panic("invalid state: AssembleEntry cannot be called on an assembler that's already finished") } if ma.ca != 0 { return nil, schema.ErrNotUnionStructure{TypeName: "gendemo.UnionKinded", Detail: "cannot add another entry -- a union can only contain one thing!"} } switch k { case "Foo": ma.state = maState_midValue ma.ca = 1 ma.w.tag = 1 ma.ca1.w = &ma.w.x1 ma.ca1.m = &ma.cm return &ma.ca1, nil case "Bar": ma.state = maState_midValue ma.ca = 2 ma.w.tag = 2 ma.ca2.w = &ma.w.x2 ma.ca2.m = &ma.cm return &ma.ca2, nil case "Baz": ma.state = maState_midValue ma.ca = 3 ma.w.tag = 3 ma.ca3.w = &ma.w.x3 ma.ca3.m = &ma.cm return &ma.ca3, nil } return nil, schema.ErrInvalidKey{TypeName: "gendemo.UnionKinded", Key: &_String{k}} } func (ma *_UnionKinded__Assembler) AssembleKey() datamodel.NodeAssembler { switch ma.state { case maState_initial: // carry on case maState_midKey: panic("invalid state: AssembleKey cannot be called when in the middle of assembling another key") case maState_expectValue: panic("invalid state: AssembleKey cannot be called when expecting start of value assembly") case maState_midValue: if !ma.valueFinishTidy() { panic("invalid state: AssembleKey cannot be called when in the middle of assembling a value") } // if tidy success: carry on for the moment, but we'll still be erroring shortly... or rather, the keyassembler will be. case maState_finished: panic("invalid state: AssembleKey cannot be called on an assembler that's already finished") } ma.state = maState_midKey return (*_UnionKinded__KeyAssembler)(ma) } func (ma *_UnionKinded__Assembler) AssembleValue() datamodel.NodeAssembler { switch ma.state { case maState_initial: panic("invalid state: AssembleValue cannot be called when no key is primed") case maState_midKey: panic("invalid state: AssembleValue cannot be called when in the middle of assembling a key") case maState_expectValue: // carry on case maState_midValue: panic("invalid state: AssembleValue cannot be called when in the middle of assembling another value") case maState_finished: panic("invalid state: AssembleValue cannot be called on an assembler that's already finished") } ma.state = maState_midValue switch ma.ca { case 1: ma.ca1.w = &ma.w.x1 ma.ca1.m = &ma.cm return &ma.ca1 case 2: ma.ca2.w = &ma.w.x2 ma.ca2.m = &ma.cm return &ma.ca2 case 3: ma.ca3.w = &ma.w.x3 ma.ca3.m = &ma.cm return &ma.ca3 default: panic("unreachable") } } func (ma *_UnionKinded__Assembler) Finish() error { switch ma.state { case maState_initial: // carry on case maState_midKey: panic("invalid state: Finish cannot be called when in the middle of assembling a key") case maState_expectValue: panic("invalid state: Finish cannot be called when expecting start of value assembly") case maState_midValue: if !ma.valueFinishTidy() { panic("invalid state: Finish cannot be called when in the middle of assembling a value") } // if tidy success: carry on case maState_finished: panic("invalid state: Finish cannot be called on an assembler that's already finished") } if ma.ca == 0 { return schema.ErrNotUnionStructure{TypeName: "gendemo.UnionKinded", Detail: "a union must have exactly one entry (not none)!"} } ma.state = maState_finished *ma.m = schema.Maybe_Value return nil } func (ma *_UnionKinded__Assembler) KeyPrototype() datamodel.NodePrototype { return _String__Prototype{} } func (ma *_UnionKinded__Assembler) ValuePrototype(k string) datamodel.NodePrototype { switch k { case "Foo": return _Foo__Prototype{} case "Bar": return _Bar__Prototype{} case "Baz": return _Baz__Prototype{} default: return nil } } type _UnionKinded__KeyAssembler _UnionKinded__Assembler func (_UnionKinded__KeyAssembler) BeginMap(sizeHint int64) (datamodel.MapAssembler, error) { return mixins.StringAssembler{TypeName: "gendemo.UnionKinded.KeyAssembler"}.BeginMap(0) } func (_UnionKinded__KeyAssembler) BeginList(sizeHint int64) (datamodel.ListAssembler, error) { return mixins.StringAssembler{TypeName: "gendemo.UnionKinded.KeyAssembler"}.BeginList(0) } func (na *_UnionKinded__KeyAssembler) AssignNull() error { return mixins.StringAssembler{TypeName: "gendemo.UnionKinded.KeyAssembler"}.AssignNull() } func (_UnionKinded__KeyAssembler) AssignBool(bool) error { return mixins.StringAssembler{TypeName: "gendemo.UnionKinded.KeyAssembler"}.AssignBool(false) } func (_UnionKinded__KeyAssembler) AssignInt(int64) error { return mixins.StringAssembler{TypeName: "gendemo.UnionKinded.KeyAssembler"}.AssignInt(0) } func (_UnionKinded__KeyAssembler) AssignFloat(float64) error { return mixins.StringAssembler{TypeName: "gendemo.UnionKinded.KeyAssembler"}.AssignFloat(0) } func (ka *_UnionKinded__KeyAssembler) AssignString(k string) error { if ka.state != maState_midKey { panic("misuse: KeyAssembler held beyond its valid lifetime") } if ka.ca != 0 { return schema.ErrNotUnionStructure{TypeName: "gendemo.UnionKinded", Detail: "cannot add another entry -- a union can only contain one thing!"} } switch k { case "Foo": ka.ca = 1 ka.w.tag = 1 ka.state = maState_expectValue return nil case "Bar": ka.ca = 2 ka.w.tag = 2 ka.state = maState_expectValue return nil case "Baz": ka.ca = 3 ka.w.tag = 3 ka.state = maState_expectValue return nil } return schema.ErrInvalidKey{TypeName: "gendemo.UnionKinded", Key: &_String{k}} // TODO: error quality: ErrInvalidUnionDiscriminant ? } func (_UnionKinded__KeyAssembler) AssignBytes([]byte) error { return mixins.StringAssembler{TypeName: "gendemo.UnionKinded.KeyAssembler"}.AssignBytes(nil) } func (_UnionKinded__KeyAssembler) AssignLink(datamodel.Link) error { return mixins.StringAssembler{TypeName: "gendemo.UnionKinded.KeyAssembler"}.AssignLink(nil) } func (ka *_UnionKinded__KeyAssembler) AssignNode(v datamodel.Node) error { if v2, err := v.AsString(); err != nil { return err } else { return ka.AssignString(v2) } } func (_UnionKinded__KeyAssembler) Prototype() datamodel.NodePrototype { return _String__Prototype{} } func (UnionKinded) Type() schema.Type { return nil /*TODO:typelit*/ } func (n UnionKinded) Representation() datamodel.Node { return (*_UnionKinded__Repr)(n) } type _UnionKinded__Repr _UnionKinded var _ datamodel.Node = &_UnionKinded__Repr{} func (n *_UnionKinded__Repr) Kind() datamodel.Kind { switch n.tag { case 1: return datamodel.Kind_Int case 2: return datamodel.Kind_Bool case 3: return datamodel.Kind_String default: panic("unreachable") } } func (n *_UnionKinded__Repr) LookupByString(key string) (datamodel.Node, error) { return nil, datamodel.ErrWrongKind{TypeName: "gendemo.UnionKinded.Repr", MethodName: "LookupByString", AppropriateKind: datamodel.KindSet_JustMap, ActualKind: n.Kind()} } func (n *_UnionKinded__Repr) LookupByNode(key datamodel.Node) (datamodel.Node, error) { return nil, datamodel.ErrWrongKind{TypeName: "gendemo.UnionKinded.Repr", MethodName: "LookupByNode", AppropriateKind: datamodel.KindSet_Recursive, ActualKind: n.Kind()} } func (n *_UnionKinded__Repr) LookupByIndex(idx int64) (datamodel.Node, error) { return nil, datamodel.ErrWrongKind{TypeName: "gendemo.UnionKinded.Repr", MethodName: "LookupByIndex", AppropriateKind: datamodel.KindSet_JustList, ActualKind: n.Kind()} } func (n *_UnionKinded__Repr) LookupBySegment(seg datamodel.PathSegment) (datamodel.Node, error) { return nil, datamodel.ErrWrongKind{TypeName: "gendemo.UnionKinded.Repr", MethodName: "LookupBySegment", AppropriateKind: datamodel.KindSet_Recursive, ActualKind: n.Kind()} } func (n *_UnionKinded__Repr) MapIterator() datamodel.MapIterator { return nil } func (n *_UnionKinded__Repr) ListIterator() datamodel.ListIterator { return nil } func (n *_UnionKinded__Repr) Length() int64 { return -1 } func (n *_UnionKinded__Repr) IsAbsent() bool { return false } func (n *_UnionKinded__Repr) IsNull() bool { return false } func (n *_UnionKinded__Repr) AsBool() (bool, error) { switch n.tag { case 2: return n.x2.Representation().AsBool() default: return false, datamodel.ErrWrongKind{TypeName: "gendemo.UnionKinded.Repr", MethodName: "AsBool", AppropriateKind: datamodel.KindSet_JustBool, ActualKind: n.Kind()} } } func (n *_UnionKinded__Repr) AsInt() (int64, error) { switch n.tag { case 1: return n.x1.Representation().AsInt() default: return 0, datamodel.ErrWrongKind{TypeName: "gendemo.UnionKinded.Repr", MethodName: "AsInt", AppropriateKind: datamodel.KindSet_JustInt, ActualKind: n.Kind()} } } func (n *_UnionKinded__Repr) AsFloat() (float64, error) { return 0, datamodel.ErrWrongKind{TypeName: "gendemo.UnionKinded.Repr", MethodName: "AsFloat", AppropriateKind: datamodel.KindSet_JustFloat, ActualKind: n.Kind()} } func (n *_UnionKinded__Repr) AsString() (string, error) { switch n.tag { case 3: return n.x3.Representation().AsString() default: return "", datamodel.ErrWrongKind{TypeName: "gendemo.UnionKinded.Repr", MethodName: "AsString", AppropriateKind: datamodel.KindSet_JustString, ActualKind: n.Kind()} } } func (n *_UnionKinded__Repr) AsBytes() ([]byte, error) { return nil, datamodel.ErrWrongKind{TypeName: "gendemo.UnionKinded.Repr", MethodName: "AsBytes", AppropriateKind: datamodel.KindSet_JustBytes, ActualKind: n.Kind()} } func (n *_UnionKinded__Repr) AsLink() (datamodel.Link, error) { return nil, datamodel.ErrWrongKind{TypeName: "gendemo.UnionKinded.Repr", MethodName: "AsLink", AppropriateKind: datamodel.KindSet_JustLink, ActualKind: n.Kind()} } func (_UnionKinded__Repr) Prototype() datamodel.NodePrototype { return _UnionKinded__ReprPrototype{} } type _UnionKinded__ReprPrototype struct{} func (_UnionKinded__ReprPrototype) NewBuilder() datamodel.NodeBuilder { var nb _UnionKinded__ReprBuilder nb.Reset() return &nb } type _UnionKinded__ReprBuilder struct { _UnionKinded__ReprAssembler } func (nb *_UnionKinded__ReprBuilder) Build() datamodel.Node { if *nb.m != schema.Maybe_Value { panic("invalid state: cannot call Build on an assembler that's not finished") } return nb.w } func (nb *_UnionKinded__ReprBuilder) Reset() { var w _UnionKinded var m schema.Maybe *nb = _UnionKinded__ReprBuilder{_UnionKinded__ReprAssembler{w: &w, m: &m}} } type _UnionKinded__ReprAssembler struct { w *_UnionKinded m *schema.Maybe ca1 _Foo__ReprAssembler ca2 _Bar__ReprAssembler ca3 _Baz__ReprAssembler ca uint } func (na *_UnionKinded__ReprAssembler) reset() { switch na.ca { case 0: return case 1: na.ca1.reset() case 2: na.ca2.reset() case 3: na.ca3.reset() default: panic("unreachable") } na.ca = 0 } func (na *_UnionKinded__ReprAssembler) BeginMap(sizeHint int64) (datamodel.MapAssembler, error) { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") case midvalue: panic("invalid state: cannot assign into assembler that's already working on a larger structure!") } return nil, schema.ErrNotUnionStructure{TypeName: "gendemo.UnionKinded.Repr", Detail: "BeginMap called but is not valid for any of the kinds that are valid members of this union"} } func (na *_UnionKinded__ReprAssembler) BeginList(sizeHint int64) (datamodel.ListAssembler, error) { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") case midvalue: panic("invalid state: cannot assign into assembler that's already working on a larger structure!") } return nil, schema.ErrNotUnionStructure{TypeName: "gendemo.UnionKinded.Repr", Detail: "BeginList called but is not valid for any of the kinds that are valid members of this union"} } func (na *_UnionKinded__ReprAssembler) AssignNull() error { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") case midvalue: panic("invalid state: cannot assign into assembler that's already working on a larger structure!") } return schema.ErrNotUnionStructure{TypeName: "gendemo.UnionKinded.Repr", Detail: "AssignNull called but is not valid for any of the kinds that are valid members of this union"} } func (na *_UnionKinded__ReprAssembler) AssignBool(v bool) error { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") case midvalue: panic("invalid state: cannot assign into assembler that's already working on a larger structure!") } if na.w == nil { na.w = &_UnionKinded{} } na.ca = 2 na.w.tag = 2 na.ca2.w = &na.w.x2 na.ca2.m = na.m return na.ca2.AssignBool(v) } func (na *_UnionKinded__ReprAssembler) AssignInt(v int64) error { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") case midvalue: panic("invalid state: cannot assign into assembler that's already working on a larger structure!") } if na.w == nil { na.w = &_UnionKinded{} } na.ca = 1 na.w.tag = 1 na.ca1.w = &na.w.x1 na.ca1.m = na.m return na.ca1.AssignInt(v) } func (na *_UnionKinded__ReprAssembler) AssignFloat(v float64) error { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") case midvalue: panic("invalid state: cannot assign into assembler that's already working on a larger structure!") } return schema.ErrNotUnionStructure{TypeName: "gendemo.UnionKinded.Repr", Detail: "AssignFloat called but is not valid for any of the kinds that are valid members of this union"} } func (na *_UnionKinded__ReprAssembler) AssignString(v string) error { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") case midvalue: panic("invalid state: cannot assign into assembler that's already working on a larger structure!") } if na.w == nil { na.w = &_UnionKinded{} } na.ca = 3 na.w.tag = 3 na.ca3.w = &na.w.x3 na.ca3.m = na.m return na.ca3.AssignString(v) } func (na *_UnionKinded__ReprAssembler) AssignBytes(v []byte) error { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") case midvalue: panic("invalid state: cannot assign into assembler that's already working on a larger structure!") } return schema.ErrNotUnionStructure{TypeName: "gendemo.UnionKinded.Repr", Detail: "AssignBytes called but is not valid for any of the kinds that are valid members of this union"} } func (na *_UnionKinded__ReprAssembler) AssignLink(v datamodel.Link) error { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") case midvalue: panic("invalid state: cannot assign into assembler that's already working on a larger structure!") } return schema.ErrNotUnionStructure{TypeName: "gendemo.UnionKinded.Repr", Detail: "AssignLink called but is not valid for any of the kinds that are valid members of this union"} } func (na *_UnionKinded__ReprAssembler) AssignNode(v datamodel.Node) error { if v.IsNull() { return na.AssignNull() } if v2, ok := v.(*_UnionKinded); ok { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") case midvalue: panic("invalid state: cannot assign null into an assembler that's already begun working on recursive structures!") } if na.w == nil { na.w = v2 *na.m = schema.Maybe_Value return nil } *na.w = *v2 *na.m = schema.Maybe_Value return nil } switch v.Kind() { case datamodel.Kind_Bool: v2, _ := v.AsBool() return na.AssignBool(v2) case datamodel.Kind_Int: v2, _ := v.AsInt() return na.AssignInt(v2) case datamodel.Kind_Float: v2, _ := v.AsFloat() return na.AssignFloat(v2) case datamodel.Kind_String: v2, _ := v.AsString() return na.AssignString(v2) case datamodel.Kind_Bytes: v2, _ := v.AsBytes() return na.AssignBytes(v2) case datamodel.Kind_Map: na, err := na.BeginMap(v.Length()) if err != nil { return err } itr := v.MapIterator() for !itr.Done() { k, v, err := itr.Next() if err != nil { return err } if err := na.AssembleKey().AssignNode(k); err != nil { return err } if err := na.AssembleValue().AssignNode(v); err != nil { return err } } return na.Finish() case datamodel.Kind_List: na, err := na.BeginList(v.Length()) if err != nil { return err } itr := v.ListIterator() for !itr.Done() { _, v, err := itr.Next() if err != nil { return err } if err := na.AssembleValue().AssignNode(v); err != nil { return err } } return na.Finish() case datamodel.Kind_Link: v2, _ := v.AsLink() return na.AssignLink(v2) default: panic("unreachable") } } func (na *_UnionKinded__ReprAssembler) Prototype() datamodel.NodePrototype { return _UnionKinded__ReprPrototype{} } ================================================ FILE: node/gendemo/ipldsch_types.go ================================================ package gendemo // Code generated by go-ipld-prime gengo. DO NOT EDIT. import ( "github.com/ipld/go-ipld-prime/datamodel" ) var _ datamodel.Node = nil // suppress errors when this dependency is not referenced // Type is a struct embeding a NodePrototype/Type for every Node implementation in this package. // One of its major uses is to start the construction of a value. // You can use it like this: // // gendemo.Type.YourTypeName.NewBuilder().BeginMap() //... // // and: // // gendemo.Type.OtherTypeName.NewBuilder().AssignString("x") // ... var Type typeSlab type typeSlab struct { Bar _Bar__Prototype Bar__Repr _Bar__ReprPrototype Baz _Baz__Prototype Baz__Repr _Baz__ReprPrototype Foo _Foo__Prototype Foo__Repr _Foo__ReprPrototype Int _Int__Prototype Int__Repr _Int__ReprPrototype Map__String__Msg3 _Map__String__Msg3__Prototype Map__String__Msg3__Repr _Map__String__Msg3__ReprPrototype Msg3 _Msg3__Prototype Msg3__Repr _Msg3__ReprPrototype String _String__Prototype String__Repr _String__ReprPrototype UnionKinded _UnionKinded__Prototype UnionKinded__Repr _UnionKinded__ReprPrototype } // --- type definitions follow --- // Bar matches the IPLD Schema type "Bar". It has bool kind. type Bar = *_Bar type _Bar struct{ x bool } // Baz matches the IPLD Schema type "Baz". It has string kind. type Baz = *_Baz type _Baz struct{ x string } // Foo matches the IPLD Schema type "Foo". It has int kind. type Foo = *_Foo type _Foo struct{ x int64 } // Int matches the IPLD Schema type "Int". It has int kind. type Int = *_Int type _Int struct{ x int64 } // Map__String__Msg3 matches the IPLD Schema type "Map__String__Msg3". It has map kind. type Map__String__Msg3 = *_Map__String__Msg3 type _Map__String__Msg3 struct { m map[_String]*_Msg3 t []_Map__String__Msg3__entry } type _Map__String__Msg3__entry struct { k _String v _Msg3 } // Msg3 matches the IPLD Schema type "Msg3". It has struct type-kind, and may be interrogated like map kind. type Msg3 = *_Msg3 type _Msg3 struct { whee _Int woot _Int waga _Int } // String matches the IPLD Schema type "String". It has string kind. type String = *_String type _String struct{ x string } // UnionKinded matches the IPLD Schema type "UnionKinded". // UnionKinded has union typekind, which means its data model behaviors are that of a map kind. type UnionKinded = *_UnionKinded type _UnionKinded struct { tag uint x1 _Foo x2 _Bar x3 _Baz } type _UnionKinded__iface interface { _UnionKinded__member() } func (_Foo) _UnionKinded__member() {} func (_Bar) _UnionKinded__member() {} func (_Baz) _UnionKinded__member() {} ================================================ FILE: node/mixins/HACKME.md ================================================ node mixins and how to use them =============================== These mixins are here to: 1. reduce the amount of code you need to write to create a new Node implementation, and 2. standardize a lot of the error handling for common cases (especially, around kinds). "Reduce the amount of code" also has an application in codegen, where while it doesn't save any human effort, it does reduce GLOC size. (Or more precisely, it doesn't save *lines*, since we use them in verbose style, but it does make those lines an awful lot shorter.) Note that these mixins are _not_ particularly here to help with performance. - all `ErrWrongKind` error are returned by value, which means a `runtime.convT2I` which means a heap allocation. The error paths will therefore never be "fast"; it will *always* be cheaper to check `kind` in advance than to probe and handle errors, if efficiency is your goal. - in general, there's really no way to improve upon the performance of having these methods simply writen directlyon your type. These mixins will affect struct size if you use them via embed. They can also be used without any effect on struct size if used more verbosely. The binary/assembly output size is not affected by use of the mixins. (If using them verbosely -- e.g. still declaring methods on your type and using `return mixins.Kind{"TypeName"}.Method()` in the method body -- the end result is the inliner kicks in, and the end result is almost identical binary size.) Summary: - SLOC: good, or neutral depending on use - GLOC: good - standardized: good - speed: neutral - mem size: neutral if used verbosely, bad if used most tersely - asm size: neutral ================================================ FILE: node/mixins/bool.go ================================================ package mixins import ( "github.com/ipld/go-ipld-prime/datamodel" ) // Bool can be embedded in a struct to provide all the methods that // have fixed output for any int-kinded nodes. // (Mostly this includes all the methods which simply return ErrWrongKind.) // Other methods will still need to be implemented to finish conforming to Node. // // To conserve memory and get a TypeName in errors without embedding, // write methods on your type with a body that simply initializes this struct // and immediately uses the relevant method; // this is more verbose in source, but compiles to a tighter result: // in memory, there's no embed; and in runtime, the calls will be inlined // and thus have no cost in execution time. type Bool struct { TypeName string } func (Bool) Kind() datamodel.Kind { return datamodel.Kind_Bool } func (x Bool) LookupByString(string) (datamodel.Node, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "LookupByString", AppropriateKind: datamodel.KindSet_JustMap, ActualKind: datamodel.Kind_Bool} } func (x Bool) LookupByNode(key datamodel.Node) (datamodel.Node, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "LookupByNode", AppropriateKind: datamodel.KindSet_JustMap, ActualKind: datamodel.Kind_Bool} } func (x Bool) LookupByIndex(idx int64) (datamodel.Node, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "LookupByIndex", AppropriateKind: datamodel.KindSet_JustList, ActualKind: datamodel.Kind_Bool} } func (x Bool) LookupBySegment(datamodel.PathSegment) (datamodel.Node, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "LookupBySegment", AppropriateKind: datamodel.KindSet_Recursive, ActualKind: datamodel.Kind_Bool} } func (Bool) MapIterator() datamodel.MapIterator { return nil } func (Bool) ListIterator() datamodel.ListIterator { return nil } func (Bool) Length() int64 { return -1 } func (Bool) IsAbsent() bool { return false } func (Bool) IsNull() bool { return false } func (x Bool) AsInt() (int64, error) { return 0, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsInt", AppropriateKind: datamodel.KindSet_JustInt, ActualKind: datamodel.Kind_Bool} } func (x Bool) AsFloat() (float64, error) { return 0, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsFloat", AppropriateKind: datamodel.KindSet_JustFloat, ActualKind: datamodel.Kind_Bool} } func (x Bool) AsString() (string, error) { return "", datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsString", AppropriateKind: datamodel.KindSet_JustString, ActualKind: datamodel.Kind_Bool} } func (x Bool) AsBytes() ([]byte, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsBytes", AppropriateKind: datamodel.KindSet_JustBytes, ActualKind: datamodel.Kind_Bool} } func (x Bool) AsLink() (datamodel.Link, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsLink", AppropriateKind: datamodel.KindSet_JustLink, ActualKind: datamodel.Kind_Bool} } // BoolAssembler has similar purpose as Bool, but for (you guessed it) // the NodeAssembler interface rather than the Node interface. type BoolAssembler struct { TypeName string } func (x BoolAssembler) BeginMap(sizeHint int64) (datamodel.MapAssembler, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "BeginMap", AppropriateKind: datamodel.KindSet_JustMap, ActualKind: datamodel.Kind_Bool} } func (x BoolAssembler) BeginList(sizeHint int64) (datamodel.ListAssembler, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "BeginList", AppropriateKind: datamodel.KindSet_JustList, ActualKind: datamodel.Kind_Bool} } func (x BoolAssembler) AssignNull() error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignNull", AppropriateKind: datamodel.KindSet_JustNull, ActualKind: datamodel.Kind_Bool} } func (x BoolAssembler) AssignInt(int64) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignInt", AppropriateKind: datamodel.KindSet_JustInt, ActualKind: datamodel.Kind_Bool} } func (x BoolAssembler) AssignFloat(float64) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignFloat", AppropriateKind: datamodel.KindSet_JustFloat, ActualKind: datamodel.Kind_Bool} } func (x BoolAssembler) AssignString(string) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignString", AppropriateKind: datamodel.KindSet_JustString, ActualKind: datamodel.Kind_Bool} } func (x BoolAssembler) AssignBytes([]byte) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignBytes", AppropriateKind: datamodel.KindSet_JustBytes, ActualKind: datamodel.Kind_Bool} } func (x BoolAssembler) AssignLink(datamodel.Link) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignLink", AppropriateKind: datamodel.KindSet_JustLink, ActualKind: datamodel.Kind_Bool} } ================================================ FILE: node/mixins/bytes.go ================================================ package mixins import ( "github.com/ipld/go-ipld-prime/datamodel" ) // Bytes can be embedded in a struct to provide all the methods that // have fixed output for any int-kinded nodes. // (Mostly this includes all the methods which simply return ErrWrongKind.) // Other methods will still need to be implemented to finish conforming to Node. // // To conserve memory and get a TypeName in errors without embedding, // write methods on your type with a body that simply initializes this struct // and immediately uses the relevant method; // this is more verbose in source, but compiles to a tighter result: // in memory, there's no embed; and in runtime, the calls will be inlined // and thus have no cost in execution time. type Bytes struct { TypeName string } func (Bytes) Kind() datamodel.Kind { return datamodel.Kind_Bytes } func (x Bytes) LookupByString(string) (datamodel.Node, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "LookupByString", AppropriateKind: datamodel.KindSet_JustMap, ActualKind: datamodel.Kind_Bytes} } func (x Bytes) LookupByNode(key datamodel.Node) (datamodel.Node, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "LookupByNode", AppropriateKind: datamodel.KindSet_JustMap, ActualKind: datamodel.Kind_Bytes} } func (x Bytes) LookupByIndex(idx int64) (datamodel.Node, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "LookupByIndex", AppropriateKind: datamodel.KindSet_JustList, ActualKind: datamodel.Kind_Bytes} } func (x Bytes) LookupBySegment(datamodel.PathSegment) (datamodel.Node, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "LookupBySegment", AppropriateKind: datamodel.KindSet_Recursive, ActualKind: datamodel.Kind_Bytes} } func (Bytes) MapIterator() datamodel.MapIterator { return nil } func (Bytes) ListIterator() datamodel.ListIterator { return nil } func (Bytes) Length() int64 { return -1 } func (Bytes) IsAbsent() bool { return false } func (Bytes) IsNull() bool { return false } func (x Bytes) AsBool() (bool, error) { return false, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsBool", AppropriateKind: datamodel.KindSet_JustBool, ActualKind: datamodel.Kind_Bytes} } func (x Bytes) AsInt() (int64, error) { return 0, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsInt", AppropriateKind: datamodel.KindSet_JustInt, ActualKind: datamodel.Kind_Bytes} } func (x Bytes) AsFloat() (float64, error) { return 0, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsFloat", AppropriateKind: datamodel.KindSet_JustFloat, ActualKind: datamodel.Kind_Bytes} } func (x Bytes) AsString() (string, error) { return "", datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsString", AppropriateKind: datamodel.KindSet_JustString, ActualKind: datamodel.Kind_Bytes} } func (x Bytes) AsLink() (datamodel.Link, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsLink", AppropriateKind: datamodel.KindSet_JustLink, ActualKind: datamodel.Kind_Bytes} } // BytesAssembler has similar purpose as Bytes, but for (you guessed it) // the NodeAssembler interface rather than the Node interface. type BytesAssembler struct { TypeName string } func (x BytesAssembler) BeginMap(sizeHint int64) (datamodel.MapAssembler, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "BeginMap", AppropriateKind: datamodel.KindSet_JustMap, ActualKind: datamodel.Kind_Bytes} } func (x BytesAssembler) BeginList(sizeHint int64) (datamodel.ListAssembler, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "BeginList", AppropriateKind: datamodel.KindSet_JustList, ActualKind: datamodel.Kind_Bytes} } func (x BytesAssembler) AssignNull() error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignNull", AppropriateKind: datamodel.KindSet_JustNull, ActualKind: datamodel.Kind_Bytes} } func (x BytesAssembler) AssignBool(bool) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignBool", AppropriateKind: datamodel.KindSet_JustBool, ActualKind: datamodel.Kind_Bytes} } func (x BytesAssembler) AssignInt(int64) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignInt", AppropriateKind: datamodel.KindSet_JustInt, ActualKind: datamodel.Kind_Bytes} } func (x BytesAssembler) AssignFloat(float64) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignFloat", AppropriateKind: datamodel.KindSet_JustFloat, ActualKind: datamodel.Kind_Bytes} } func (x BytesAssembler) AssignString(string) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignString", AppropriateKind: datamodel.KindSet_JustString, ActualKind: datamodel.Kind_Bytes} } func (x BytesAssembler) AssignLink(datamodel.Link) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignLink", AppropriateKind: datamodel.KindSet_JustLink, ActualKind: datamodel.Kind_Bytes} } ================================================ FILE: node/mixins/delim.go ================================================ package mixins // This file is a little different than most of its siblings in this package. // It's not really much of a "mixin". More of a util function junkdrawer. // // Implementations of Data Model Nodes are unlikely to need these. // Implementations of Schema-level Node *are* likely to need these, however. // // Our codegen implementation emits calls to these functions. // (And having these functions in a package that's already an unconditional // import in files emitted by codegen makes the codegen significantly simpler.) import ( "fmt" "strings" ) // SplitExact is much like strings.Split but will error if the number of // substrings is other than the expected count. // // SplitExact is used by the 'stringjoin' representation for structs. // // The 'count' parameter is a length. In other words, if you expect // the zero'th index to be present in the result, you should ask for // a count of at least '1'. // Using this function with 'count' less than 2 is rather strange. func SplitExact(s string, sep string, count int) ([]string, error) { ss := strings.Split(s, sep) if len(ss) != count { return nil, fmt.Errorf("expected %d instances of the delimiter, found %d", count-1, len(ss)-1) } return ss, nil } // SplitN is an alias of strings.SplitN, which is only present here to // make it usable in codegen packages without requiring conditional imports // in the generation process. func SplitN(s, sep string, n int) []string { return strings.SplitN(s, sep, n) } ================================================ FILE: node/mixins/delim_test.go ================================================ package mixins import ( "fmt" "reflect" "testing" "github.com/google/go-cmp/cmp" qt "github.com/frankban/quicktest" ) func TestSplitExact(t *testing.T) { type expect struct { value []string err error } type tcase struct { s string sep string count int expect expect } for _, ent := range []tcase{ {"", "", 0, expect{[]string{}, nil}}, {"", ":", 1, expect{[]string{""}, nil}}, {"x", ":", 1, expect{[]string{"x"}, nil}}, {"x:y", ":", 2, expect{[]string{"x", "y"}, nil}}, {"x:y:", ":", 2, expect{nil, fmt.Errorf("expected 1 instances of the delimiter, found 2")}}, {":x:y", ":", 2, expect{nil, fmt.Errorf("expected 1 instances of the delimiter, found 2")}}, {"x:y:", ":", 3, expect{[]string{"x", "y", ""}, nil}}, } { value, err := SplitExact(ent.s, ent.sep, ent.count) ent2 := tcase{ent.s, ent.sep, ent.count, expect{value, err}} qt.Check(t, ent2, qt.CmpEquals(cmp.Exporter(func(reflect.Type) bool { return true })), ent) } } ================================================ FILE: node/mixins/float.go ================================================ package mixins import ( "github.com/ipld/go-ipld-prime/datamodel" ) // Float can be embedded in a struct to provide all the methods that // have fixed output for any int-kinded nodes. // (Mostly this includes all the methods which simply return ErrWrongKind.) // Other methods will still need to be implemented to finish conforming to Node. // // To conserve memory and get a TypeName in errors without embedding, // write methods on your type with a body that simply initializes this struct // and immediately uses the relevant method; // this is more verbose in source, but compiles to a tighter result: // in memory, there's no embed; and in runtime, the calls will be inlined // and thus have no cost in execution time. type Float struct { TypeName string } func (Float) Kind() datamodel.Kind { return datamodel.Kind_Float } func (x Float) LookupByString(string) (datamodel.Node, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "LookupByString", AppropriateKind: datamodel.KindSet_JustMap, ActualKind: datamodel.Kind_Float} } func (x Float) LookupByNode(key datamodel.Node) (datamodel.Node, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "LookupByNode", AppropriateKind: datamodel.KindSet_JustMap, ActualKind: datamodel.Kind_Float} } func (x Float) LookupByIndex(idx int64) (datamodel.Node, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "LookupByIndex", AppropriateKind: datamodel.KindSet_JustList, ActualKind: datamodel.Kind_Float} } func (x Float) LookupBySegment(datamodel.PathSegment) (datamodel.Node, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "LookupBySegment", AppropriateKind: datamodel.KindSet_Recursive, ActualKind: datamodel.Kind_Float} } func (Float) MapIterator() datamodel.MapIterator { return nil } func (Float) ListIterator() datamodel.ListIterator { return nil } func (Float) Length() int64 { return -1 } func (Float) IsAbsent() bool { return false } func (Float) IsNull() bool { return false } func (x Float) AsBool() (bool, error) { return false, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsBool", AppropriateKind: datamodel.KindSet_JustBool, ActualKind: datamodel.Kind_Float} } func (x Float) AsInt() (int64, error) { return 0, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsInt", AppropriateKind: datamodel.KindSet_JustInt, ActualKind: datamodel.Kind_Float} } func (x Float) AsString() (string, error) { return "", datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsString", AppropriateKind: datamodel.KindSet_JustString, ActualKind: datamodel.Kind_Float} } func (x Float) AsBytes() ([]byte, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsBytes", AppropriateKind: datamodel.KindSet_JustBytes, ActualKind: datamodel.Kind_Float} } func (x Float) AsLink() (datamodel.Link, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsLink", AppropriateKind: datamodel.KindSet_JustLink, ActualKind: datamodel.Kind_Float} } // FloatAssembler has similar purpose as Float, but for (you guessed it) // the NodeAssembler interface rather than the Node interface. type FloatAssembler struct { TypeName string } func (x FloatAssembler) BeginMap(sizeHint int64) (datamodel.MapAssembler, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "BeginMap", AppropriateKind: datamodel.KindSet_JustMap, ActualKind: datamodel.Kind_Float} } func (x FloatAssembler) BeginList(sizeHint int64) (datamodel.ListAssembler, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "BeginList", AppropriateKind: datamodel.KindSet_JustList, ActualKind: datamodel.Kind_Float} } func (x FloatAssembler) AssignNull() error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignNull", AppropriateKind: datamodel.KindSet_JustNull, ActualKind: datamodel.Kind_Float} } func (x FloatAssembler) AssignBool(bool) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignBool", AppropriateKind: datamodel.KindSet_JustBool, ActualKind: datamodel.Kind_Float} } func (x FloatAssembler) AssignInt(int64) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignInt", AppropriateKind: datamodel.KindSet_JustInt, ActualKind: datamodel.Kind_Float} } func (x FloatAssembler) AssignString(string) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignString", AppropriateKind: datamodel.KindSet_JustString, ActualKind: datamodel.Kind_Float} } func (x FloatAssembler) AssignBytes([]byte) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignBytes", AppropriateKind: datamodel.KindSet_JustBytes, ActualKind: datamodel.Kind_Float} } func (x FloatAssembler) AssignLink(datamodel.Link) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignLink", AppropriateKind: datamodel.KindSet_JustLink, ActualKind: datamodel.Kind_Float} } ================================================ FILE: node/mixins/int.go ================================================ package mixins import ( "github.com/ipld/go-ipld-prime/datamodel" ) // Int can be embedded in a struct to provide all the methods that // have fixed output for any int-kinded nodes. // (Mostly this includes all the methods which simply return ErrWrongKind.) // Other methods will still need to be implemented to finish conforming to Node. // // To conserve memory and get a TypeName in errors without embedding, // write methods on your type with a body that simply initializes this struct // and immediately uses the relevant method; // this is more verbose in source, but compiles to a tighter result: // in memory, there's no embed; and in runtime, the calls will be inlined // and thus have no cost in execution time. type Int struct { TypeName string } func (Int) Kind() datamodel.Kind { return datamodel.Kind_Int } func (x Int) LookupByString(string) (datamodel.Node, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "LookupByString", AppropriateKind: datamodel.KindSet_JustMap, ActualKind: datamodel.Kind_Int} } func (x Int) LookupByNode(key datamodel.Node) (datamodel.Node, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "LookupByNode", AppropriateKind: datamodel.KindSet_JustMap, ActualKind: datamodel.Kind_Int} } func (x Int) LookupByIndex(idx int64) (datamodel.Node, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "LookupByIndex", AppropriateKind: datamodel.KindSet_JustList, ActualKind: datamodel.Kind_Int} } func (x Int) LookupBySegment(datamodel.PathSegment) (datamodel.Node, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "LookupBySegment", AppropriateKind: datamodel.KindSet_Recursive, ActualKind: datamodel.Kind_Int} } func (Int) MapIterator() datamodel.MapIterator { return nil } func (Int) ListIterator() datamodel.ListIterator { return nil } func (Int) Length() int64 { return -1 } func (Int) IsAbsent() bool { return false } func (Int) IsNull() bool { return false } func (x Int) AsBool() (bool, error) { return false, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsBool", AppropriateKind: datamodel.KindSet_JustBool, ActualKind: datamodel.Kind_Int} } func (x Int) AsFloat() (float64, error) { return 0, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsFloat", AppropriateKind: datamodel.KindSet_JustFloat, ActualKind: datamodel.Kind_Int} } func (x Int) AsString() (string, error) { return "", datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsString", AppropriateKind: datamodel.KindSet_JustString, ActualKind: datamodel.Kind_Int} } func (x Int) AsBytes() ([]byte, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsBytes", AppropriateKind: datamodel.KindSet_JustBytes, ActualKind: datamodel.Kind_Int} } func (x Int) AsLink() (datamodel.Link, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsLink", AppropriateKind: datamodel.KindSet_JustLink, ActualKind: datamodel.Kind_Int} } // IntAssembler has similar purpose as Int, but for (you guessed it) // the NodeAssembler interface rather than the Node interface. type IntAssembler struct { TypeName string } func (x IntAssembler) BeginMap(sizeHint int64) (datamodel.MapAssembler, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "BeginMap", AppropriateKind: datamodel.KindSet_JustMap, ActualKind: datamodel.Kind_Int} } func (x IntAssembler) BeginList(sizeHint int64) (datamodel.ListAssembler, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "BeginList", AppropriateKind: datamodel.KindSet_JustList, ActualKind: datamodel.Kind_Int} } func (x IntAssembler) AssignNull() error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignNull", AppropriateKind: datamodel.KindSet_JustNull, ActualKind: datamodel.Kind_Int} } func (x IntAssembler) AssignBool(bool) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignBool", AppropriateKind: datamodel.KindSet_JustBool, ActualKind: datamodel.Kind_Int} } func (x IntAssembler) AssignFloat(float64) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignFloat", AppropriateKind: datamodel.KindSet_JustFloat, ActualKind: datamodel.Kind_Int} } func (x IntAssembler) AssignString(string) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignString", AppropriateKind: datamodel.KindSet_JustString, ActualKind: datamodel.Kind_Int} } func (x IntAssembler) AssignBytes([]byte) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignBytes", AppropriateKind: datamodel.KindSet_JustBytes, ActualKind: datamodel.Kind_Int} } func (x IntAssembler) AssignLink(datamodel.Link) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignLink", AppropriateKind: datamodel.KindSet_JustLink, ActualKind: datamodel.Kind_Int} } ================================================ FILE: node/mixins/link.go ================================================ package mixins import ( "github.com/ipld/go-ipld-prime/datamodel" ) // Link can be embedded in a struct to provide all the methods that // have fixed output for any int-kinded nodes. // (Mostly this includes all the methods which simply return ErrWrongKind.) // Other methods will still need to be implemented to finish conforming to Node. // // To conserve memory and get a TypeName in errors without embedding, // write methods on your type with a body that simply initializes this struct // and immediately uses the relevant method; // this is more verbose in source, but compiles to a tighter result: // in memory, there's no embed; and in runtime, the calls will be inlined // and thus have no cost in execution time. type Link struct { TypeName string } func (Link) Kind() datamodel.Kind { return datamodel.Kind_Link } func (x Link) LookupByString(string) (datamodel.Node, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "LookupByString", AppropriateKind: datamodel.KindSet_JustMap, ActualKind: datamodel.Kind_Link} } func (x Link) LookupByNode(key datamodel.Node) (datamodel.Node, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "LookupByNode", AppropriateKind: datamodel.KindSet_JustMap, ActualKind: datamodel.Kind_Link} } func (x Link) LookupByIndex(idx int64) (datamodel.Node, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "LookupByIndex", AppropriateKind: datamodel.KindSet_JustList, ActualKind: datamodel.Kind_Link} } func (x Link) LookupBySegment(datamodel.PathSegment) (datamodel.Node, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "LookupBySegment", AppropriateKind: datamodel.KindSet_Recursive, ActualKind: datamodel.Kind_Link} } func (Link) MapIterator() datamodel.MapIterator { return nil } func (Link) ListIterator() datamodel.ListIterator { return nil } func (Link) Length() int64 { return -1 } func (Link) IsAbsent() bool { return false } func (Link) IsNull() bool { return false } func (x Link) AsBool() (bool, error) { return false, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsBool", AppropriateKind: datamodel.KindSet_JustBool, ActualKind: datamodel.Kind_Link} } func (x Link) AsInt() (int64, error) { return 0, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsInt", AppropriateKind: datamodel.KindSet_JustInt, ActualKind: datamodel.Kind_Link} } func (x Link) AsFloat() (float64, error) { return 0, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsFloat", AppropriateKind: datamodel.KindSet_JustFloat, ActualKind: datamodel.Kind_Link} } func (x Link) AsString() (string, error) { return "", datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsString", AppropriateKind: datamodel.KindSet_JustString, ActualKind: datamodel.Kind_Link} } func (x Link) AsBytes() ([]byte, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsBytes", AppropriateKind: datamodel.KindSet_JustBytes, ActualKind: datamodel.Kind_Link} } // LinkAssembler has similar purpose as Link, but for (you guessed it) // the NodeAssembler interface rather than the Node interface. type LinkAssembler struct { TypeName string } func (x LinkAssembler) BeginMap(sizeHint int64) (datamodel.MapAssembler, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "BeginMap", AppropriateKind: datamodel.KindSet_JustMap, ActualKind: datamodel.Kind_Link} } func (x LinkAssembler) BeginList(sizeHint int64) (datamodel.ListAssembler, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "BeginList", AppropriateKind: datamodel.KindSet_JustList, ActualKind: datamodel.Kind_Link} } func (x LinkAssembler) AssignNull() error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignNull", AppropriateKind: datamodel.KindSet_JustNull, ActualKind: datamodel.Kind_Link} } func (x LinkAssembler) AssignBool(bool) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignBool", AppropriateKind: datamodel.KindSet_JustBool, ActualKind: datamodel.Kind_Link} } func (x LinkAssembler) AssignInt(int64) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignInt", AppropriateKind: datamodel.KindSet_JustInt, ActualKind: datamodel.Kind_Link} } func (x LinkAssembler) AssignFloat(float64) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignFloat", AppropriateKind: datamodel.KindSet_JustFloat, ActualKind: datamodel.Kind_Link} } func (x LinkAssembler) AssignString(string) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignString", AppropriateKind: datamodel.KindSet_JustString, ActualKind: datamodel.Kind_Link} } func (x LinkAssembler) AssignBytes([]byte) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignBytes", AppropriateKind: datamodel.KindSet_JustBytes, ActualKind: datamodel.Kind_Link} } ================================================ FILE: node/mixins/list.go ================================================ package mixins import ( "github.com/ipld/go-ipld-prime/datamodel" ) // List can be embedded in a struct to provide all the methods that // have fixed output for any int-kinded nodes. // (Mostly this includes all the methods which simply return ErrWrongKind.) // Other methods will still need to be implemented to finish conforming to Node. // // To conserve memory and get a TypeName in errors without embedding, // write methods on your type with a body that simply initializes this struct // and immediately uses the relevant method; // this is more verbose in source, but compiles to a tighter result: // in memory, there's no embed; and in runtime, the calls will be inlined // and thus have no cost in execution time. type List struct { TypeName string } func (List) Kind() datamodel.Kind { return datamodel.Kind_List } func (x List) LookupByString(string) (datamodel.Node, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "LookupByString", AppropriateKind: datamodel.KindSet_JustMap, ActualKind: datamodel.Kind_List} } func (x List) LookupByNode(key datamodel.Node) (datamodel.Node, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "LookupByNode", AppropriateKind: datamodel.KindSet_JustMap, ActualKind: datamodel.Kind_List} } func (List) MapIterator() datamodel.MapIterator { return nil } func (List) IsAbsent() bool { return false } func (List) IsNull() bool { return false } func (x List) AsBool() (bool, error) { return false, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsBool", AppropriateKind: datamodel.KindSet_JustBool, ActualKind: datamodel.Kind_List} } func (x List) AsInt() (int64, error) { return 0, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsInt", AppropriateKind: datamodel.KindSet_JustInt, ActualKind: datamodel.Kind_List} } func (x List) AsFloat() (float64, error) { return 0, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsFloat", AppropriateKind: datamodel.KindSet_JustFloat, ActualKind: datamodel.Kind_List} } func (x List) AsString() (string, error) { return "", datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsString", AppropriateKind: datamodel.KindSet_JustString, ActualKind: datamodel.Kind_List} } func (x List) AsBytes() ([]byte, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsBytes", AppropriateKind: datamodel.KindSet_JustBytes, ActualKind: datamodel.Kind_List} } func (x List) AsLink() (datamodel.Link, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsLink", AppropriateKind: datamodel.KindSet_JustLink, ActualKind: datamodel.Kind_List} } // ListAssembler has similar purpose as List, but for (you guessed it) // the NodeAssembler interface rather than the Node interface. type ListAssembler struct { TypeName string } func (x ListAssembler) BeginMap(sizeHint int64) (datamodel.MapAssembler, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "BeginMap", AppropriateKind: datamodel.KindSet_JustMap, ActualKind: datamodel.Kind_List} } func (x ListAssembler) AssignNull() error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignNull", AppropriateKind: datamodel.KindSet_JustNull, ActualKind: datamodel.Kind_List} } func (x ListAssembler) AssignBool(bool) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignBool", AppropriateKind: datamodel.KindSet_JustBool, ActualKind: datamodel.Kind_List} } func (x ListAssembler) AssignInt(int64) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignInt", AppropriateKind: datamodel.KindSet_JustInt, ActualKind: datamodel.Kind_List} } func (x ListAssembler) AssignFloat(float64) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignFloat", AppropriateKind: datamodel.KindSet_JustFloat, ActualKind: datamodel.Kind_List} } func (x ListAssembler) AssignString(string) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignString", AppropriateKind: datamodel.KindSet_JustString, ActualKind: datamodel.Kind_List} } func (x ListAssembler) AssignBytes([]byte) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignBytes", AppropriateKind: datamodel.KindSet_JustBytes, ActualKind: datamodel.Kind_List} } func (x ListAssembler) AssignLink(datamodel.Link) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignLink", AppropriateKind: datamodel.KindSet_JustLink, ActualKind: datamodel.Kind_List} } ================================================ FILE: node/mixins/map.go ================================================ package mixins import ( "github.com/ipld/go-ipld-prime/datamodel" ) // Map can be embedded in a struct to provide all the methods that // have fixed output for any map-kinded nodes. // (Mostly this includes all the methods which simply return ErrWrongKind.) // Other methods will still need to be implemented to finish conforming to Node. // // To conserve memory and get a TypeName in errors without embedding, // write methods on your type with a body that simply initializes this struct // and immediately uses the relevant method; // this is more verbose in source, but compiles to a tighter result: // in memory, there's no embed; and in runtime, the calls will be inlined // and thus have no cost in execution time. type Map struct { TypeName string } func (Map) Kind() datamodel.Kind { return datamodel.Kind_Map } func (x Map) LookupByIndex(idx int64) (datamodel.Node, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "LookupByIndex", AppropriateKind: datamodel.KindSet_JustList, ActualKind: datamodel.Kind_Map} } func (Map) ListIterator() datamodel.ListIterator { return nil } func (Map) IsAbsent() bool { return false } func (Map) IsNull() bool { return false } func (x Map) AsBool() (bool, error) { return false, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsBool", AppropriateKind: datamodel.KindSet_JustBool, ActualKind: datamodel.Kind_Map} } func (x Map) AsInt() (int64, error) { return 0, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsInt", AppropriateKind: datamodel.KindSet_JustInt, ActualKind: datamodel.Kind_Map} } func (x Map) AsFloat() (float64, error) { return 0, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsFloat", AppropriateKind: datamodel.KindSet_JustFloat, ActualKind: datamodel.Kind_Map} } func (x Map) AsString() (string, error) { return "", datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsString", AppropriateKind: datamodel.KindSet_JustString, ActualKind: datamodel.Kind_Map} } func (x Map) AsBytes() ([]byte, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsBytes", AppropriateKind: datamodel.KindSet_JustBytes, ActualKind: datamodel.Kind_Map} } func (x Map) AsLink() (datamodel.Link, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsLink", AppropriateKind: datamodel.KindSet_JustLink, ActualKind: datamodel.Kind_Map} } // MapAssembler has similar purpose as Map, but for (you guessed it) // the NodeAssembler interface rather than the Node interface. type MapAssembler struct { TypeName string } func (x MapAssembler) BeginList(sizeHint int64) (datamodel.ListAssembler, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "BeginList", AppropriateKind: datamodel.KindSet_JustList, ActualKind: datamodel.Kind_Map} } func (x MapAssembler) AssignNull() error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignNull", AppropriateKind: datamodel.KindSet_JustNull, ActualKind: datamodel.Kind_Map} } func (x MapAssembler) AssignBool(bool) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignBool", AppropriateKind: datamodel.KindSet_JustBool, ActualKind: datamodel.Kind_Map} } func (x MapAssembler) AssignInt(int64) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignInt", AppropriateKind: datamodel.KindSet_JustInt, ActualKind: datamodel.Kind_Map} } func (x MapAssembler) AssignFloat(float64) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignFloat", AppropriateKind: datamodel.KindSet_JustFloat, ActualKind: datamodel.Kind_Map} } func (x MapAssembler) AssignString(string) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignString", AppropriateKind: datamodel.KindSet_JustString, ActualKind: datamodel.Kind_Map} } func (x MapAssembler) AssignBytes([]byte) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignBytes", AppropriateKind: datamodel.KindSet_JustBytes, ActualKind: datamodel.Kind_Map} } func (x MapAssembler) AssignLink(datamodel.Link) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignLink", AppropriateKind: datamodel.KindSet_JustLink, ActualKind: datamodel.Kind_Map} } ================================================ FILE: node/mixins/string.go ================================================ package mixins import ( "github.com/ipld/go-ipld-prime/datamodel" ) // String can be embedded in a struct to provide all the methods that // have fixed output for any string-kinded nodes. // (Mostly this includes all the methods which simply return ErrWrongKind.) // Other methods will still need to be implemented to finish conforming to Node. // // To conserve memory and get a TypeName in errors without embedding, // write methods on your type with a body that simply initializes this struct // and immediately uses the relevant method; // this is more verbose in source, but compiles to a tighter result: // in memory, there's no embed; and in runtime, the calls will be inlined // and thus have no cost in execution time. type String struct { TypeName string } func (String) Kind() datamodel.Kind { return datamodel.Kind_String } func (x String) LookupByString(string) (datamodel.Node, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "LookupByString", AppropriateKind: datamodel.KindSet_JustMap, ActualKind: datamodel.Kind_String} } func (x String) LookupByNode(key datamodel.Node) (datamodel.Node, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "LookupByNode", AppropriateKind: datamodel.KindSet_JustMap, ActualKind: datamodel.Kind_String} } func (x String) LookupByIndex(idx int64) (datamodel.Node, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "LookupByIndex", AppropriateKind: datamodel.KindSet_JustList, ActualKind: datamodel.Kind_String} } func (x String) LookupBySegment(datamodel.PathSegment) (datamodel.Node, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "LookupBySegment", AppropriateKind: datamodel.KindSet_Recursive, ActualKind: datamodel.Kind_String} } func (String) MapIterator() datamodel.MapIterator { return nil } func (String) ListIterator() datamodel.ListIterator { return nil } func (String) Length() int64 { return -1 } func (String) IsAbsent() bool { return false } func (String) IsNull() bool { return false } func (x String) AsBool() (bool, error) { return false, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsBool", AppropriateKind: datamodel.KindSet_JustBool, ActualKind: datamodel.Kind_String} } func (x String) AsInt() (int64, error) { return 0, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsInt", AppropriateKind: datamodel.KindSet_JustInt, ActualKind: datamodel.Kind_String} } func (x String) AsFloat() (float64, error) { return 0, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsFloat", AppropriateKind: datamodel.KindSet_JustFloat, ActualKind: datamodel.Kind_String} } func (x String) AsBytes() ([]byte, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsBytes", AppropriateKind: datamodel.KindSet_JustBytes, ActualKind: datamodel.Kind_String} } func (x String) AsLink() (datamodel.Link, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsLink", AppropriateKind: datamodel.KindSet_JustLink, ActualKind: datamodel.Kind_String} } // StringAssembler has similar purpose as String, but for (you guessed it) // the NodeAssembler interface rather than the Node interface. type StringAssembler struct { TypeName string } func (x StringAssembler) BeginMap(sizeHint int64) (datamodel.MapAssembler, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "BeginMap", AppropriateKind: datamodel.KindSet_JustMap, ActualKind: datamodel.Kind_String} } func (x StringAssembler) BeginList(sizeHint int64) (datamodel.ListAssembler, error) { return nil, datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "BeginList", AppropriateKind: datamodel.KindSet_JustList, ActualKind: datamodel.Kind_String} } func (x StringAssembler) AssignNull() error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignNull", AppropriateKind: datamodel.KindSet_JustNull, ActualKind: datamodel.Kind_String} } func (x StringAssembler) AssignBool(bool) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignBool", AppropriateKind: datamodel.KindSet_JustBool, ActualKind: datamodel.Kind_String} } func (x StringAssembler) AssignInt(int64) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignInt", AppropriateKind: datamodel.KindSet_JustInt, ActualKind: datamodel.Kind_String} } func (x StringAssembler) AssignFloat(float64) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignFloat", AppropriateKind: datamodel.KindSet_JustFloat, ActualKind: datamodel.Kind_String} } func (x StringAssembler) AssignBytes([]byte) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignBytes", AppropriateKind: datamodel.KindSet_JustBytes, ActualKind: datamodel.Kind_String} } func (x StringAssembler) AssignLink(datamodel.Link) error { return datamodel.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignLink", AppropriateKind: datamodel.KindSet_JustLink, ActualKind: datamodel.Kind_String} } ================================================ FILE: node/mixins/tmplMixin.txt ================================================ // copy this and remove methods that aren't relevant to your kind. // this has not been scripted. // (the first part is trivial; the second part is not; and this updates rarely. https://xkcd.com/1205/ applies.) package mixins import ( "github.com/ipld/go-ipld-prime/datamodel" ) // @Kind@ can be embedded in a struct to provide all the methods that // have fixed output for any int-kinded nodes. // (Mostly this includes all the methods which simply return ErrWrongKind.) // Other methods will still need to be implemented to finish conforming to Node. // // To conserve memory and get a TypeName in errors without embedding, // write methods on your type with a body that simply initializes this struct // and immediately uses the relevant method; // this is more verbose in source, but compiles to a tighter result: // in memory, there's no embed; and in runtime, the calls will be inlined // and thus have no cost in execution time. type @Kind@ struct { TypeName string } func (@Kind@) Kind() ipld.Kind { return ipld.Kind_@Kind@ } func (x @Kind@) LookupByString(string) (ipld.Node, error) { return nil, ipld.ErrWrongKind{TypeName: x.TypeName, MethodName: "LookupByString", AppropriateKind: ipld.KindSet_JustMap, ActualKind: ipld.Kind_@Kind@} } func (x @Kind@) LookupByNode(key ipld.Node) (ipld.Node, error) { return nil, ipld.ErrWrongKind{TypeName: x.TypeName, MethodName: "LookupByNode", AppropriateKind: ipld.KindSet_JustMap, ActualKind: ipld.Kind_@Kind@} } func (x @Kind@) LookupByIndex(idx int) (ipld.Node, error) { return nil, ipld.ErrWrongKind{TypeName: x.TypeName, MethodName: "LookupByIndex", AppropriateKind: ipld.KindSet_JustList, ActualKind: ipld.Kind_@Kind@} } func (x @Kind@) LookupBySegment(ipld.PathSegment) (ipld.Node, error) { return nil, ipld.ErrWrongKind{TypeName: x.TypeName, MethodName: "LookupBySegment", AppropriateKind: ipld.KindSet_Recursive, ActualKind: ipld.Kind_@Kind@} } func (@Kind@) MapIterator() ipld.MapIterator { return nil } func (@Kind@) ListIterator() ipld.ListIterator { return nil } func (@Kind@) Length() int { return -1 } func (@Kind@) IsAbsent() bool { return false } func (@Kind@) IsNull() bool { return false } func (x @Kind@) AsBool() (bool, error) { return false, ipld.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsBool", AppropriateKind: ipld.KindSet_JustBool, ActualKind: ipld.Kind_@Kind@} } func (x @Kind@) AsInt() (int, error) { return 0, ipld.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsInt", AppropriateKind: ipld.KindSet_JustInt, ActualKind: ipld.Kind_@Kind@} } func (x @Kind@) AsFloat() (float64, error) { return 0, ipld.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsFloat", AppropriateKind: ipld.KindSet_JustFloat, ActualKind: ipld.Kind_@Kind@} } func (x @Kind@) AsString() (string, error) { return "", ipld.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsString", AppropriateKind: ipld.KindSet_JustString, ActualKind: ipld.Kind_@Kind@} } func (x @Kind@) AsBytes() ([]byte, error) { return nil, ipld.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsBytes", AppropriateKind: ipld.KindSet_JustBytes, ActualKind: ipld.Kind_@Kind@} } func (x @Kind@) AsLink() (ipld.Link, error) { return nil, ipld.ErrWrongKind{TypeName: x.TypeName, MethodName: "AsLink", AppropriateKind: ipld.KindSet_JustLink, ActualKind: ipld.Kind_@Kind@} } // @Kind@Assembler has similar purpose as @Kind@, but for (you guessed it) // the NodeAssembler interface rather than the Node interface. type @Kind@Assembler struct { TypeName string } func (x @Kind@Assembler) BeginMap(sizeHint int) (ipld.MapAssembler, error) { return nil, ipld.ErrWrongKind{TypeName: x.TypeName, MethodName: "BeginMap", AppropriateKind: ipld.KindSet_JustMap, ActualKind: ipld.Kind_@Kind@} } func (x @Kind@Assembler) BeginList(sizeHint int) (ipld.ListAssembler, error) { return nil, ipld.ErrWrongKind{TypeName: x.TypeName, MethodName: "BeginList", AppropriateKind: ipld.KindSet_JustList, ActualKind: ipld.Kind_@Kind@} } func (x @Kind@Assembler) AssignNull() error { return ipld.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignNull", AppropriateKind: ipld.KindSet_JustNull, ActualKind: ipld.Kind_@Kind@} } func (x @Kind@Assembler) AssignBool(bool) error { return ipld.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignBool", AppropriateKind: ipld.KindSet_JustBool, ActualKind: ipld.Kind_@Kind@} } func (x @Kind@Assembler) AssignInt(int) error { return ipld.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignInt", AppropriateKind: ipld.KindSet_JustInt, ActualKind: ipld.Kind_@Kind@} } func (x @Kind@Assembler) AssignFloat(float64) error { return ipld.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignFloat", AppropriateKind: ipld.KindSet_JustFloat, ActualKind: ipld.Kind_@Kind@} } func (x @Kind@Assembler) AssignString(string) error { return ipld.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignString", AppropriateKind: ipld.KindSet_JustString, ActualKind: ipld.Kind_@Kind@} } func (x @Kind@Assembler) AssignBytes([]byte) error { return ipld.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignBytes", AppropriateKind: ipld.KindSet_JustBytes, ActualKind: ipld.Kind_@Kind@} } func (x @Kind@Assembler) AssignLink(ipld.Link) error { return ipld.ErrWrongKind{TypeName: x.TypeName, MethodName: "AssignLink", AppropriateKind: ipld.KindSet_JustLink, ActualKind: ipld.Kind_@Kind@} } ================================================ FILE: node/tests/HACKME.md ================================================ HACKME ====== This package is for reusable tests and benchmarks. These test and benchmark functions work over the Node and NodeBuilder interfaces, so they should work to test compatibility and compare performance of various implementations of Node. This is easier said than done. Naming conventions ------------------ ### name prefix All reusable test functions start with the name prefix `TestSpec_`. All reusable benchmarks start with the name prefix `BenchmarkSpec_`. The "Test" and "Benchmark" prefixes are as per the requirements of the golang standard `testing` package. They take `*testing.T` and `*testing.B` arguments respectively. They also take at least one interface argument which is how you give your Node implementation to the test spec. The word "Spec" reflects on the fact that these are reusable/standardized tests. We recommend you copy-paste these method names outright into the package of your Node implementation. It's not necessary, but it's nice for consistency. (In the future, there may be tooling to help make automated comparisons of different Node implementation's relative performance; this would necessarily rely on consistent names across packages.) If your Node implementation package has *more* tests and benchmarks that *are not* from this reusable set, that's great -- but don't use the "Spec" word as a segment of their name; it'll make processing bulk output easier. ### full pattern The full pattern is: `BenchmarkSpec_{Application}_{FixtureCohort}/codec={codec}/n={size}` - `{Application}` means what feature or big-picture behavior we're testing. Examples include "Marshal", "Unmarshal", "Walk", etc. - `{FixtureCohort}` means... well, see the names from the 'corpus' subpackage; it should be literally one of those strings. - `n={size}` will be present for variable-scale benchmarks. You'll have to consider the Application and FixtureCohort to understand the context of what part of the data is being varied in size, though. - `codec={codec}` is an example of extra info that might exist for some applications. For example, it might include "json" and "cbor" for "Marshal" and "Unmarshal", but will not be seen at all in other applications like "Walk". The parts after the slash are those which are handled internally. For those, you call the `BenchmarkSpec_*` function name (stopping before the first slash), and that function will call `b.Run` to make sub-tests for all the variations. For example, when you call `BenchmarkSpec_Walk_MapNStrMap3StrInt`, that one call will result in a suite of tests for various sizes, each of which will be denoted in the output by `BenchmarkSpec_Walk_MapNStrMap3StrInt/n=1`, then `.../n=2`, etc. ### variable scale benchmarks Some corpuses have fixed sizes. Some are variable. With fixed-size corpuses, you'll see an integer in the "FixtureCohort" name. For variable-size corpuses, you'll see the letter "N" in place of an integer. See the docs in the 'corpus' subpackage for more discussion of this. ================================================ FILE: node/tests/byteSpecs.go ================================================ package tests import ( "io" "testing" qt "github.com/frankban/quicktest" "github.com/ipld/go-ipld-prime/datamodel" ) func SpecTestBytes(t *testing.T, np datamodel.NodePrototype) { t.Run("byte node", func(t *testing.T) { nb := np.NewBuilder() err := nb.AssignBytes([]byte("asdf")) qt.Check(t, err, qt.IsNil) n := nb.Build() qt.Check(t, n.Kind(), qt.Equals, datamodel.Kind_Bytes) qt.Check(t, n.IsNull(), qt.IsFalse) x, err := n.AsBytes() qt.Check(t, err, qt.IsNil) qt.Check(t, x, qt.DeepEquals, []byte("asdf")) lbn, ok := n.(datamodel.LargeBytesNode) if ok { str, err := lbn.AsLargeBytes() qt.Check(t, err, qt.IsNil) bytes, err := io.ReadAll(str) qt.Check(t, err, qt.IsNil) qt.Check(t, bytes, qt.DeepEquals, []byte("asdf")) } }) } ================================================ FILE: node/tests/checkers.go ================================================ package tests import ( "errors" qt "github.com/frankban/quicktest" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/printer" ) // NodeContentEquals checks whether two nodes have equal content by first encoding them via // printer.Sprint, then checking that the generated encodings are identical. // // Use DeepNodeContentsEquals if you want a less strict comparison that does not // require map keys to be in the same order. // // See: printer.Sprint. var NodeContentEquals = &nodeContentEqualsChecker{} type nodeContentEqualsChecker struct{} func (n *nodeContentEqualsChecker) Check(got interface{}, args []interface{}, note func(key string, value interface{})) error { want := args[0] if want == nil { return qt.IsNil.Check(got, args, note) } if got == nil { return qt.IsNotNil.Check(got, args, note) } wantNode, ok := want.(datamodel.Node) if !ok { return errors.New("this checker only supports checking datamodel.Node values") } wantPrint := printer.Sprint(wantNode) gotNode, ok := got.(datamodel.Node) if !ok { return errors.New("this checker only supports checking datamodel.Node values") } gotPrint := printer.Sprint(gotNode) return qt.Equals.Check(gotPrint, []interface{}{wantPrint}, note) } func (n *nodeContentEqualsChecker) ArgNames() []string { return []string{"got", "want node"} } // DeepNodeContentsEquals checks whether two nodes have equal content by // walking the nodes recursively and comparing their contents. This is similar // to datamodel.DeepEquals except that map keys DO NOT need to be strictly // the same order to be considered equal. // // Use NodeContentEquals if you want a more strict comparison. var DeepNodeContentsEquals = &deepNodeContentsEqualsChecker{} type deepNodeContentsEqualsChecker struct{} func (n *deepNodeContentsEqualsChecker) Check(got interface{}, args []interface{}, note func(key string, value interface{})) error { want := args[0] if want == nil { return qt.IsNil.Check(got, args, note) } if got == nil { return qt.IsNotNil.Check(got, args, note) } wantNode, ok := want.(datamodel.Node) if !ok { return errors.New("this checker only supports checking datamodel.Node values") } gotNode, ok := got.(datamodel.Node) if !ok { return errors.New("this checker only supports checking datamodel.Node values") } return deepEqualCheck(gotNode, wantNode, note) } func (n *deepNodeContentsEqualsChecker) ArgNames() []string { return []string{"got", "want node"} } func deepEqualCheck(x, y datamodel.Node, note func(key string, value interface{})) error { if x == nil || y == nil { if err := qt.Equals.Check(x, []interface{}{y}, note); err != nil { return err } } xk, yk := x.Kind(), y.Kind() if err := qt.Equals.Check(xk, []interface{}{yk}, note); err != nil { return err } switch xk { // Scalar kinds. case datamodel.Kind_Null: return qt.Equals.Check(x.IsNull(), []interface{}{y.IsNull()}, note) case datamodel.Kind_Bool: xv, err := x.AsBool() if err != nil { panic(err) } yv, err := y.AsBool() if err != nil { panic(err) } return qt.Equals.Check(xv, []interface{}{yv}, note) case datamodel.Kind_Int: xv, err := x.AsInt() if err != nil { panic(err) } yv, err := y.AsInt() if err != nil { panic(err) } return qt.Equals.Check(xv, []interface{}{yv}, note) case datamodel.Kind_Float: xv, err := x.AsFloat() if err != nil { panic(err) } yv, err := y.AsFloat() if err != nil { panic(err) } return qt.Equals.Check(xv, []interface{}{yv}, note) case datamodel.Kind_String: xv, err := x.AsString() if err != nil { panic(err) } yv, err := y.AsString() if err != nil { panic(err) } return qt.Equals.Check(xv, []interface{}{yv}, note) case datamodel.Kind_Bytes: xv, err := x.AsBytes() if err != nil { panic(err) } yv, err := y.AsBytes() if err != nil { panic(err) } return qt.Equals.Check(string(xv), []interface{}{string(yv)}, note) case datamodel.Kind_Link: xv, err := x.AsLink() if err != nil { panic(err) } yv, err := y.AsLink() if err != nil { panic(err) } // Links are just compared via ==. // This requires the types to exactly match, // and the values to be equal as per == too. // This will generally work, // as ipld-prime assumes link types to be consistent. return qt.Equals.Check(xv, []interface{}{yv}, note) // Recursive kinds. case datamodel.Kind_Map: if err := qt.Equals.Check(x.Length(), []interface{}{y.Length()}, note); err != nil { return err } ykeys := make([]string, y.Length()) yitr := y.MapIterator() for !yitr.Done() { ykey, _, err := yitr.Next() if err != nil { panic(err) } yk, err := ykey.AsString() if err != nil { panic(err) } ykeys = append(ykeys, yk) } xitr := x.MapIterator() for !xitr.Done() { xkey, xval, err := xitr.Next() if err != nil { panic(err) } xk, err := xkey.AsString() if err != nil { panic(err) } if err := qt.Contains.Check(ykeys, []interface{}{xk}, note); err != nil { return err } yval, err := y.LookupByNode(xkey) if err != nil { panic(err) } if err := deepEqualCheck(xval, yval, note); err != nil { return err } } return nil case datamodel.Kind_List: if err := qt.Equals.Check(x.Length(), []interface{}{y.Length()}, note); err != nil { return err } xitr := x.ListIterator() yitr := y.ListIterator() for !xitr.Done() && !yitr.Done() { _, xval, err := xitr.Next() if err != nil { panic(err) } _, yval, err := yitr.Next() if err != nil { panic(err) } if err := deepEqualCheck(xval, yval, note); err != nil { return err } } return nil // As per the docs, other kinds such as Invalid are not deeply equal. default: panic("bad kind") } } ================================================ FILE: node/tests/checkers_test.go ================================================ package tests import ( "testing" qt "github.com/frankban/quicktest" "github.com/ipld/go-ipld-prime/node/basicnode" ) func Test_nodeContentEqualsChecker_Check(t *testing.T) { someNode := basicnode.Prototype__String{}.NewBuilder().Build() nb := basicnode.Prototype__String{}.NewBuilder() err := nb.AssignString("fish") qt.Assert(t, err, qt.IsNil) someOtherNode := nb.Build() tests := []struct { name string got interface{} want interface{} wantErr string }{ { name: "nilWantIsError", got: "not a node", want: nil, wantErr: "got non-nil value", }, { name: "nonNodeAsWantIsError", got: "not a node", want: someNode, wantErr: "this checker only supports checking datamodel.Node values", }, { name: "nonNodeAsGotIsError", got: someNode, want: "not a node", wantErr: "this checker only supports checking datamodel.Node values", }, { name: "nilWantAndGotAreEqual", got: nil, want: nil, }, { name: "equivalentNodesAreEqual", got: someNode, want: someNode, }, { name: "differentNodesAreNotEqual", got: someNode, want: someOtherNode, wantErr: "values are not equal", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { err := NodeContentEquals.Check(tt.got, []interface{}{tt.want}, nil) if tt.wantErr == "" { qt.Assert(t, err, qt.IsNil) } else { qt.Assert(t, err, qt.Not(qt.IsNil)) qt.Assert(t, err.Error(), qt.Equals, tt.wantErr) } }) } } ================================================ FILE: node/tests/corpus/corpus.go ================================================ /* The corpus package exports some values useful for building tests and benchmarks. Values come as JSON strings. It is assumed you can unmarshal those. The reason we do this is so that this corpus package doesn't import any particular concrete implementation of ipld.Node... since that would make it ironically incapable of being used for that Node's tests. The naming convention is roughly as follows: - {Kind}{{Count}|N}{KeyKind}{ValueKind} - 'Kind' is usually 'Map' or 'List'. It can also be a scalar like 'Int', in which case that's it. - If a specific int is given for 'Count', that's the size of the thing; - if 'N' is present, it's a scalable corpus and you can decide the size. - 'KeyKind' is present for maps (it will be string...). - 'ValueKind' is present for maps and lists. It can recurse. Of course, this naming convention is not perfectly specific, but it's usually enough for our needs, or at least enough to get started. Some corpuses designed for probing (for example) tuple-represented structs will end up with interesting designations for various reasons: - some corpuses are meant to test struct semantics. This is usually what it means when you see fixed size maps. "List5Various" can also be this reason (it's for tuple-represented structs). - some corpuses are meant to test nullable or optional semantics. These might have name suffixes like "WithNull" to indicate this. Everything is exported as a function, for consistency. Many functions need no args. Some functions need an argument for "N". If you're using these corpuses in a benchmark, don't forget to call `b.ResetTimer()` after getting the corpus. */ package corpus import ( "fmt" ) func Map3StrInt() string { return `{"whee":1,"woot":2,"waga":3}` } func MapNStrInt(n int) string { return `{` + ents(n, func(i int) string { return fmt.Sprintf(`"k%d":%d`, i, i) }) + `}` } func MapNStrMap3StrInt(n int) string { return `{` + ents(n, func(i int) string { return fmt.Sprintf(`"k%d":`, i) + fmt.Sprintf(`{"whee":%d,"woot":%d,"waga":%d}`, i*3+1, i*3+2, i*3+3) }) + `}` } ================================================ FILE: node/tests/corpus/corpus_test.go ================================================ package corpus import ( "encoding/json" "testing" "github.com/ipld/go-ipld-prime/must" ) /* Sanity check that our corpuses are actually correct JSON in this package, before we start letting other packages find that out for us the hard way. */ func TestCorpusValidity(t *testing.T) { must.True(json.Valid([]byte(MapNStrInt(0)))) must.True(json.Valid([]byte(MapNStrInt(1)))) must.True(json.Valid([]byte(MapNStrInt(2)))) must.True(json.Valid([]byte(MapNStrMap3StrInt(0)))) must.True(json.Valid([]byte(MapNStrMap3StrInt(1)))) must.True(json.Valid([]byte(MapNStrMap3StrInt(2)))) } ================================================ FILE: node/tests/corpus/util.go ================================================ package corpus import ( "bytes" ) func ents(n int, segFn func(i int) string) string { if n <= 0 { return "" } var buf bytes.Buffer for i := 0; i < n; i++ { buf.WriteString(segFn(i)) buf.WriteString(`,`) } buf.Truncate(buf.Len() - 1) return buf.String() } ================================================ FILE: node/tests/listSpecs.go ================================================ package tests import ( "testing" qt "github.com/frankban/quicktest" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/fluent" "github.com/ipld/go-ipld-prime/must" ) func SpecTestListString(t *testing.T, np datamodel.NodePrototype) { t.Run("list, 3 entries", func(t *testing.T) { n := fluent.MustBuildList(np, 3, func(la fluent.ListAssembler) { la.AssembleValue().AssignString("one") la.AssembleValue().AssignString("two") la.AssembleValue().AssignString("three") }) t.Run("reads back out", func(t *testing.T) { qt.Check(t, n.Length(), qt.Equals, int64(3)) v, err := n.LookupByIndex(0) qt.Check(t, err, qt.IsNil) qt.Check(t, must.String(v), qt.Equals, "one") v, err = n.LookupByIndex(1) qt.Check(t, err, qt.IsNil) qt.Check(t, must.String(v), qt.Equals, "two") v, err = n.LookupByIndex(2) qt.Check(t, err, qt.IsNil) qt.Check(t, must.String(v), qt.Equals, "three") }) t.Run("reads via iteration", func(t *testing.T) { itr := n.ListIterator() qt.Check(t, itr.Done(), qt.IsFalse) idx, v, err := itr.Next() qt.Check(t, err, qt.IsNil) qt.Check(t, idx, qt.Equals, int64(0)) qt.Check(t, must.String(v), qt.Equals, "one") qt.Check(t, itr.Done(), qt.IsFalse) idx, v, err = itr.Next() qt.Check(t, err, qt.IsNil) qt.Check(t, idx, qt.Equals, int64(1)) qt.Check(t, must.String(v), qt.Equals, "two") qt.Check(t, itr.Done(), qt.IsFalse) idx, v, err = itr.Next() qt.Check(t, err, qt.IsNil) qt.Check(t, idx, qt.Equals, int64(2)) qt.Check(t, must.String(v), qt.Equals, "three") qt.Check(t, itr.Done(), qt.IsTrue) idx, v, err = itr.Next() qt.Check(t, err, qt.Equals, datamodel.ErrIteratorOverread{}) qt.Check(t, idx, qt.Equals, int64(-1)) qt.Check(t, v, qt.IsNil) }) }) } ================================================ FILE: node/tests/mapBenchmarks.go ================================================ package tests import ( "testing" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/must" ) func SpecBenchmarkMapStrInt_3n_AssembleStandard(b *testing.B, np datamodel.NodePrototype) { for i := 0; i < b.N; i++ { sink = buildMapStrIntN3(np) } } func SpecBenchmarkMapStrInt_3n_AssembleEntry(b *testing.B, np datamodel.NodePrototype) { for i := 0; i < b.N; i++ { nb := np.NewBuilder() ma, err := nb.BeginMap(3) if err != nil { panic(err) } if va, err := ma.AssembleEntry("whee"); err != nil { panic(err) } else { must.NotError(va.AssignInt(1)) } if va, err := ma.AssembleEntry("woot"); err != nil { panic(err) } else { must.NotError(va.AssignInt(2)) } if va, err := ma.AssembleEntry("waga"); err != nil { panic(err) } else { must.NotError(va.AssignInt(3)) } must.NotError(ma.Finish()) sink = nb.Build() } } func SpecBenchmarkMapStrInt_3n_Iteration(b *testing.B, np datamodel.NodePrototype) { n := buildMapStrIntN3(np) b.ResetTimer() for i := 0; i < b.N; i++ { itr := n.MapIterator() for k, v, _ := itr.Next(); !itr.Done(); k, v, _ = itr.Next() { sink = k sink = v } } } // n25 --> func SpecBenchmarkMapStrInt_25n_AssembleStandard(b *testing.B, np datamodel.NodePrototype) { for i := 0; i < b.N; i++ { sink = buildMapStrIntN25(np) } } func SpecBenchmarkMapStrInt_25n_AssembleEntry(b *testing.B, np datamodel.NodePrototype) { for i := 0; i < b.N; i++ { nb := np.NewBuilder() ma, err := nb.BeginMap(25) if err != nil { panic(err) } for i := 1; i <= 25; i++ { if va, err := ma.AssembleEntry(tableStrInt[i-1].s); err != nil { panic(err) } else { must.NotError(va.AssignInt(tableStrInt[i-1].i)) } } must.NotError(ma.Finish()) sink = nb.Build() } } func SpecBenchmarkMapStrInt_25n_Iteration(b *testing.B, np datamodel.NodePrototype) { n := buildMapStrIntN25(np) b.ResetTimer() for i := 0; i < b.N; i++ { itr := n.MapIterator() for k, v, _ := itr.Next(); !itr.Done(); k, v, _ = itr.Next() { sink = k sink = v } } } ================================================ FILE: node/tests/mapBenchmarks_test.go ================================================ package tests import ( "encoding/json" "testing" "github.com/ipld/go-ipld-prime/must" ) // This is analogous to the 'MapStrInt_3n' suite of benchmarks, // but against a golang native map in regular go code, // for getting a baseline impression to compare other things against. func BenchmarkMapStrInt_3n_BaselineNativeMapAssignSimpleKeys(b *testing.B) { for i := 0; i < b.N; i++ { x := make(map[string]int, 3) x["whee"] = 1 x["woot"] = 2 x["waga"] = 3 sink = x } } func BenchmarkMapStrInt_3n_BaselineJsonUnmarshalMapSimpleKeys(b *testing.B) { for i := 0; i < b.N; i++ { x := make(map[string]int, 3) must.NotError(json.Unmarshal([]byte(`{"whee":1,"woot":2,"waga":3}`), &x)) sink = x } } func BenchmarkMapStrInt_3n_BaselineJsonMarshalMapSimpleKeys(b *testing.B) { x := map[string]int{"whee": 1, "woot": 2, "waga": 3} for i := 0; i < b.N; i++ { bs, err := json.Marshal(x) must.NotError(err) sink = bs } } var ( sink_s string sink_i int64 ) func BenchmarkMapStrInt_3n_BaselineNativeMapIterationSimpleKeys(b *testing.B) { x := make(map[string]int64, 3) x["whee"] = 1 x["woot"] = 2 x["waga"] = 3 sink = x b.ResetTimer() for i := 0; i < b.N; i++ { for k, v := range x { sink_s = k sink_i = v } } } // n25 --> func BenchmarkMapStrInt_25n_BaselineNativeMapAssignSimpleKeys(b *testing.B) { for i := 0; i < b.N; i++ { x := make(map[string]int64, 25) for i := 1; i <= 25; i++ { x[tableStrInt[i-1].s] = tableStrInt[i-1].i } sink = x } } func BenchmarkMapStrInt_25n_BaselineNativeMapIterationSimpleKeys(b *testing.B) { x := make(map[string]int64, 25) for i := 1; i <= 25; i++ { x[tableStrInt[i-1].s] = tableStrInt[i-1].i } sink = x b.ResetTimer() for i := 0; i < b.N; i++ { for k, v := range x { sink_s = k sink_i = v } } } ================================================ FILE: node/tests/mapFixtures.go ================================================ package tests import ( "fmt" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/must" ) var tableStrInt = [25]struct { s string i int64 }{} func init() { for i := int64(1); i <= 25; i++ { tableStrInt[i-1] = struct { s string i int64 }{fmt.Sprintf("k%d", i), i} } } // extracted for reuse between correctness tests and benchmarks func buildMapStrIntN3(np datamodel.NodePrototype) datamodel.Node { nb := np.NewBuilder() ma, err := nb.BeginMap(3) must.NotError(err) must.NotError(ma.AssembleKey().AssignString("whee")) must.NotError(ma.AssembleValue().AssignInt(1)) must.NotError(ma.AssembleKey().AssignString("woot")) must.NotError(ma.AssembleValue().AssignInt(2)) must.NotError(ma.AssembleKey().AssignString("waga")) must.NotError(ma.AssembleValue().AssignInt(3)) must.NotError(ma.Finish()) return nb.Build() } // extracted for reuse across benchmarks func buildMapStrIntN25(np datamodel.NodePrototype) datamodel.Node { nb := np.NewBuilder() ma, err := nb.BeginMap(25) must.NotError(err) for i := 1; i <= 25; i++ { must.NotError(ma.AssembleKey().AssignString(tableStrInt[i-1].s)) must.NotError(ma.AssembleValue().AssignInt(tableStrInt[i-1].i)) } must.NotError(ma.Finish()) return nb.Build() } ================================================ FILE: node/tests/mapSpecs.go ================================================ package tests import ( "testing" qt "github.com/frankban/quicktest" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/must" ) func SpecTestMapStrInt(t *testing.T, np datamodel.NodePrototype) { t.Run("map, 3 entries", func(t *testing.T) { n := buildMapStrIntN3(np) t.Run("reads back out", func(t *testing.T) { qt.Check(t, n.Length(), qt.Equals, int64(3)) v, err := n.LookupByString("whee") qt.Check(t, err, qt.IsNil) v2, err := v.AsInt() qt.Check(t, err, qt.IsNil) qt.Check(t, v2, qt.Equals, int64(1)) v, err = n.LookupByString("waga") qt.Check(t, err, qt.IsNil) v2, err = v.AsInt() qt.Check(t, err, qt.IsNil) qt.Check(t, v2, qt.Equals, int64(3)) v, err = n.LookupByString("woot") qt.Check(t, err, qt.IsNil) v2, err = v.AsInt() qt.Check(t, err, qt.IsNil) qt.Check(t, v2, qt.Equals, int64(2)) }) t.Run("reads via iteration", func(t *testing.T) { itr := n.MapIterator() qt.Check(t, itr.Done(), qt.IsFalse) k, v, err := itr.Next() qt.Check(t, err, qt.IsNil) k2, err := k.AsString() qt.Check(t, err, qt.IsNil) qt.Check(t, k2, qt.Equals, "whee") v2, err := v.AsInt() qt.Check(t, err, qt.IsNil) qt.Check(t, v2, qt.Equals, int64(1)) qt.Check(t, itr.Done(), qt.IsFalse) k, v, err = itr.Next() qt.Check(t, err, qt.IsNil) k2, err = k.AsString() qt.Check(t, err, qt.IsNil) qt.Check(t, k2, qt.Equals, "woot") v2, err = v.AsInt() qt.Check(t, err, qt.IsNil) qt.Check(t, v2, qt.Equals, int64(2)) qt.Check(t, itr.Done(), qt.IsFalse) k, v, err = itr.Next() qt.Check(t, err, qt.IsNil) k2, err = k.AsString() qt.Check(t, err, qt.IsNil) qt.Check(t, k2, qt.Equals, "waga") v2, err = v.AsInt() qt.Check(t, err, qt.IsNil) qt.Check(t, v2, qt.Equals, int64(3)) qt.Check(t, itr.Done(), qt.IsTrue) k, v, err = itr.Next() qt.Check(t, err, qt.Equals, datamodel.ErrIteratorOverread{}) qt.Check(t, k, qt.IsNil) qt.Check(t, v, qt.IsNil) }) t.Run("reads for absent keys error sensibly", func(t *testing.T) { v, err := n.LookupByString("nope") qt.Check(t, err, qt.ErrorAs, &datamodel.ErrNotExists{}) qt.Check(t, err, qt.ErrorMatches, `key not found: "nope"`) qt.Check(t, v, qt.IsNil) }) }) t.Run("repeated key should error", func(t *testing.T) { nb := np.NewBuilder() ma, err := nb.BeginMap(3) if err != nil { panic(err) } if err := ma.AssembleKey().AssignString("whee"); err != nil { panic(err) } if err := ma.AssembleValue().AssignInt(1); err != nil { panic(err) } if err := ma.AssembleKey().AssignString("whee"); err != nil { qt.Check(t, err, qt.ErrorAs, &datamodel.ErrRepeatedMapKey{}) // No string assertion at present -- how that should be presented for typed stuff is unsettled // (and if it's clever, it'll differ from untyped, which will mean no assertion possible!). } }) t.Run("using expired child assemblers should panic", func(t *testing.T) { nb := np.NewBuilder() ma, err := nb.BeginMap(3) must.NotError(err) // Assemble a key, and then try to assign it again. Latter should fail. ka := ma.AssembleKey() must.NotError(ka.AssignString("whee")) func() { defer func() { recover() }() ka.AssignString("woo") t.Fatal("must not be reached") }() // Assemble a value, and then try to assign it again. Latter should fail. // (This does assume your system can continue after disregarding the last error.) va := ma.AssembleValue() must.NotError(va.AssignInt(1)) func() { defer func() { recover() }() va.AssignInt(2) t.Fatal("must not be reached") }() // ... and neither of these should've had visible effects! qt.Check(t, ma.Finish(), qt.IsNil) n := nb.Build() qt.Check(t, n.Length(), qt.Equals, int64(1)) v, err := n.LookupByString("whee") qt.Check(t, err, qt.IsNil) v2, err := v.AsInt() qt.Check(t, err, qt.IsNil) qt.Check(t, v2, qt.Equals, int64(1)) }) t.Run("builder reset works", func(t *testing.T) { // TODO }) } func SpecTestMapStrMapStrInt(t *testing.T, np datamodel.NodePrototype) { t.Run("map>", func(t *testing.T) { nb := np.NewBuilder() ma, err := nb.BeginMap(3) must.NotError(err) must.NotError(ma.AssembleKey().AssignString("whee")) func(ma datamodel.MapAssembler, err error) { must.NotError(ma.AssembleKey().AssignString("m1k1")) must.NotError(ma.AssembleValue().AssignInt(1)) must.NotError(ma.AssembleKey().AssignString("m1k2")) must.NotError(ma.AssembleValue().AssignInt(2)) must.NotError(ma.Finish()) }(ma.AssembleValue().BeginMap(2)) must.NotError(ma.AssembleKey().AssignString("woot")) func(ma datamodel.MapAssembler, err error) { must.NotError(ma.AssembleKey().AssignString("m2k1")) must.NotError(ma.AssembleValue().AssignInt(3)) must.NotError(ma.AssembleKey().AssignString("m2k2")) must.NotError(ma.AssembleValue().AssignInt(4)) must.NotError(ma.Finish()) }(ma.AssembleValue().BeginMap(2)) must.NotError(ma.AssembleKey().AssignString("waga")) func(ma datamodel.MapAssembler, err error) { must.NotError(ma.AssembleKey().AssignString("m3k1")) must.NotError(ma.AssembleValue().AssignInt(5)) must.NotError(ma.AssembleKey().AssignString("m3k2")) must.NotError(ma.AssembleValue().AssignInt(6)) must.NotError(ma.Finish()) }(ma.AssembleValue().BeginMap(2)) must.NotError(ma.Finish()) n := nb.Build() t.Run("reads back out", func(t *testing.T) { qt.Check(t, n.Length(), qt.Equals, int64(3)) v, err := n.LookupByString("woot") qt.Check(t, err, qt.IsNil) v2, err := v.LookupByString("m2k1") qt.Check(t, err, qt.IsNil) v3, err := v2.AsInt() qt.Check(t, err, qt.IsNil) qt.Check(t, v3, qt.Equals, int64(3)) v2, err = v.LookupByString("m2k2") qt.Check(t, err, qt.IsNil) v3, err = v2.AsInt() qt.Check(t, err, qt.IsNil) qt.Check(t, v3, qt.Equals, int64(4)) }) }) } func SpecTestMapStrListStr(t *testing.T, np datamodel.NodePrototype) { t.Run("map>", func(t *testing.T) { nb := np.NewBuilder() ma, err := nb.BeginMap(3) must.NotError(err) must.NotError(ma.AssembleKey().AssignString("asdf")) func(la datamodel.ListAssembler, err error) { must.NotError(la.AssembleValue().AssignString("eleven")) must.NotError(la.AssembleValue().AssignString("twelve")) must.NotError(la.AssembleValue().AssignString("thirteen")) must.NotError(la.Finish()) }(ma.AssembleValue().BeginList(3)) must.NotError(ma.AssembleKey().AssignString("qwer")) func(la datamodel.ListAssembler, err error) { must.NotError(la.AssembleValue().AssignString("twentyone")) must.NotError(la.AssembleValue().AssignString("twentytwo")) must.NotError(la.Finish()) }(ma.AssembleValue().BeginList(2)) must.NotError(ma.AssembleKey().AssignString("zxcv")) func(la datamodel.ListAssembler, err error) { must.NotError(la.AssembleValue().AssignString("thirtyone")) must.NotError(la.Finish()) }(ma.AssembleValue().BeginList(1)) must.NotError(ma.Finish()) n := nb.Build() t.Run("reads back out", func(t *testing.T) { qt.Check(t, n.Length(), qt.Equals, int64(3)) v, err := n.LookupByString("qwer") qt.Check(t, err, qt.IsNil) v2, err := v.LookupByIndex(1) qt.Check(t, err, qt.IsNil) v3, err := v2.AsString() qt.Check(t, err, qt.IsNil) qt.Check(t, v3, qt.Equals, "twentytwo") }) }) } ================================================ FILE: node/tests/marshalBenchmarks.go ================================================ package tests import ( "bytes" "fmt" "strings" "testing" "github.com/ipld/go-ipld-prime/codec/json" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/must" "github.com/ipld/go-ipld-prime/node/tests/corpus" ) // All of the marshalling and unmarshalling benchmark specs use JSON. // This does mean we're measuring a bunch of stuff that has nothing to do // with the core operations of the Node/NodeBuilder interface. // We do this so that: // - we get a reasonable picture of how much time is spent in the IPLD Data Model // versus how much time is spent in the serialization efforts; // - we can make direct comparisons to the standard library json marshalling // and unmarshalling, thus having a back-of-the-envelope baseline to compare. func BenchmarkSpec_Marshal_Map3StrInt(b *testing.B, np datamodel.NodePrototype) { nb := np.NewBuilder() must.NotError(json.Decode(nb, strings.NewReader(`{"whee":1,"woot":2,"waga":3}`))) n := nb.Build() b.ResetTimer() var err error for i := 0; i < b.N; i++ { var buf bytes.Buffer err = json.Encode(n, &buf) sink = buf } if err != nil { panic(err) } } func BenchmarkSpec_Marshal_MapNStrMap3StrInt(b *testing.B, np datamodel.NodePrototype) { for _, n := range []int{0, 1, 2, 4, 8, 16, 32} { b.Run(fmt.Sprintf("n=%d", n), func(b *testing.B) { msg := corpus.MapNStrMap3StrInt(n) node := mustNodeFromJsonString(np, msg) b.ResetTimer() var buf bytes.Buffer var err error for i := 0; i < b.N; i++ { buf = bytes.Buffer{} err = json.Encode(node, &buf) } b.StopTimer() if err != nil { b.Fatalf("encode errored: %s", err) } if buf.String() != msg { b.Fatalf("encode result didn't match corpus") } }) } } ================================================ FILE: node/tests/schema.go ================================================ package tests import "testing" // use a table instead of a map, to get a consistent order var allSchemaTests = []struct { name string fn func(*testing.T, Engine) }{ {"Links", SchemaTestLinks}, {"ListsContainingMaybe", SchemaTestListsContainingMaybe}, {"ListsContainingLists", SchemaTestListsContainingLists}, {"MapsContainingMaybe", SchemaTestMapsContainingMaybe}, {"MapsContainingMaps", SchemaTestMapsContainingMaps}, {"MapsWithComplexKeys", SchemaTestMapsWithComplexKeys}, {"Scalars", SchemaTestScalars}, {"RequiredFields", SchemaTestRequiredFields}, {"StructNesting", SchemaTestStructNesting}, {"StructReprStringjoin", SchemaTestStructReprStringjoin}, {"StructReprTuple", SchemaTestStructReprTuple}, {"StructReprListPairs", SchemaTestStructReprListPairs}, {"StructsContainingMaybe", SchemaTestStructsContainingMaybe}, {"UnionKeyed", SchemaTestUnionKeyed}, {"UnionKeyedComplexChildren", SchemaTestUnionKeyedComplexChildren}, {"UnionKeyedReset", SchemaTestUnionKeyedReset}, {"UnionKinded", SchemaTestUnionKinded}, {"UnionStringprefix", SchemaTestUnionStringprefix}, } type EngineSubtest struct { Name string // subtest name Engine Engine } func SchemaTestAll(t *testing.T, forTest func(name string) []EngineSubtest) { for _, test := range allSchemaTests { test := test // do not reuse the range variable t.Run(test.name, func(t *testing.T) { t.Parallel() subtests := forTest(test.name) if len(subtests) == 0 { t.Skip("no engine provided to SchemaTestAll") } if len(subtests) == 1 { sub := subtests[0] if sub.Name != "" { t.Fatal("a single engine shouldn't be named") } test.fn(t, sub.Engine) return } for _, sub := range subtests { if sub.Name == "" { t.Fatal("multiple engines should be named") } t.Run(sub.Name, func(t *testing.T) { test.fn(t, sub.Engine) }) } }) } } ================================================ FILE: node/tests/schemaLinks.go ================================================ package tests import ( "bytes" "fmt" "testing" qt "github.com/frankban/quicktest" "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime/codec/dagjson" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/fluent" "github.com/ipld/go-ipld-prime/linking" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/schema" "github.com/ipld/go-ipld-prime/storage/memstore" "github.com/ipld/go-ipld-prime/traversal" "github.com/ipld/go-ipld-prime/traversal/selector" "github.com/ipld/go-ipld-prime/traversal/selector/builder" ) var store = memstore.Store{} func encode(n datamodel.Node) (datamodel.Node, datamodel.Link) { lp := cidlink.LinkPrototype{Prefix: cid.Prefix{ Version: 1, Codec: 0x0129, MhType: 0x13, MhLength: 4, }} lsys := cidlink.DefaultLinkSystem() lsys.SetWriteStorage(&store) lnk, err := lsys.Store(linking.LinkContext{}, lp, n) if err != nil { panic(err) } return n, lnk } func SchemaTestLinks(t *testing.T, engine Engine) { ts := schema.TypeSystem{} ts.Init() ts.Accumulate(schema.SpawnInt("Int")) ts.Accumulate(schema.SpawnString("String")) ts.Accumulate(schema.SpawnList("ListOfStrings", "String", false)) ts.Accumulate(schema.SpawnLink("Link")) // &Any ts.Accumulate(schema.SpawnLinkReference("IntLink", "Int")) // &Int ts.Accumulate(schema.SpawnLinkReference("StringLink", "String")) // &String ts.Accumulate(schema.SpawnLinkReference("ListOfStringsLink", "ListOfStrings")) // &ListOfStrings ts.Accumulate(schema.SpawnStruct("LinkStruct", []schema.StructField{ schema.SpawnStructField("any", "Link", false, false), schema.SpawnStructField("int", "IntLink", false, false), schema.SpawnStructField("str", "StringLink", false, false), schema.SpawnStructField("strlist", "ListOfStringsLink", false, false), }, schema.SpawnStructRepresentationMap(map[string]string{}), )) engine.Init(t, ts) t.Run("typed linkage traversal", func(t *testing.T) { _, intNodeLnk := func() (datamodel.Node, datamodel.Link) { np := engine.PrototypeByName("Int") nb := np.NewBuilder() nb.AssignInt(101) return encode(nb.Build()) }() _, stringNodeLnk := encode(fluent.MustBuild(engine.PrototypeByName("String"), func(na fluent.NodeAssembler) { na.AssignString("a string") })) _, listOfStringsNodeLnk := encode(fluent.MustBuildList(engine.PrototypeByName("ListOfStrings"), 3, func(la fluent.ListAssembler) { la.AssembleValue().AssignString("s1") la.AssembleValue().AssignString("s2") la.AssembleValue().AssignString("s3") })) linkStructNode, _ := encode(fluent.MustBuildMap(engine.PrototypeByName("LinkStruct"), 4, func(ma fluent.MapAssembler) { ma.AssembleEntry("any").AssignLink(stringNodeLnk) ma.AssembleEntry("int").AssignLink(intNodeLnk) ma.AssembleEntry("str").AssignLink(stringNodeLnk) ma.AssembleEntry("strlist").AssignLink(listOfStringsNodeLnk) })) ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) ss := ssb.ExploreRecursive(selector.RecursionLimitDepth(3), ssb.ExploreUnion( ssb.Matcher(), ssb.ExploreAll(ssb.ExploreRecursiveEdge()), )) s, err := ss.Selector() qt.Check(t, err, qt.IsNil) var order int lsys := cidlink.DefaultLinkSystem() lsys.SetReadStorage(&store) err = traversal.Progress{ Cfg: &traversal.Config{ LinkSystem: lsys, LinkTargetNodePrototypeChooser: func(lnk datamodel.Link, lnkCtx linking.LinkContext) (datamodel.NodePrototype, error) { if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { return tlnkNd.LinkTargetNodePrototype(), nil } return basicnode.Prototype.Any, nil }, }, }.WalkMatching(linkStructNode, s, func(prog traversal.Progress, n datamodel.Node) error { buf := new(bytes.Buffer) dagjson.Encode(n, buf) fmt.Printf("Walked %d: %v\n", order, buf.String()) switch order { case 0: // root qt.Check(t, n.Prototype(), qt.Equals, engine.PrototypeByName("LinkStruct")) case 1: // from an &Any qt.Check(t, n.Prototype(), qt.Equals, basicnode.Prototype__String{}) case 2: // &Int qt.Check(t, n.Prototype(), qt.Equals, engine.PrototypeByName("Int")) case 3: // &String qt.Check(t, n.Prototype(), qt.Equals, engine.PrototypeByName("String")) case 4: // &ListOfStrings qt.Check(t, n.Prototype(), qt.Equals, engine.PrototypeByName("ListOfStrings")) case 5: fallthrough case 6: fallthrough case 7: qt.Check(t, n.Prototype(), qt.Equals, engine.PrototypeByName("String")) } order++ return nil }) qt.Check(t, err, qt.IsNil) qt.Check(t, order, qt.Equals, 8) }) } ================================================ FILE: node/tests/schemaLists.go ================================================ package tests import ( "testing" qt "github.com/frankban/quicktest" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/fluent" "github.com/ipld/go-ipld-prime/must" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/schema" ) func SchemaTestListsContainingMaybe(t *testing.T, engine Engine) { ts := schema.TypeSystem{} ts.Init() ts.Accumulate(schema.SpawnString("String")) ts.Accumulate(schema.SpawnList("List__String", "String", false)) ts.Accumulate(schema.SpawnList("List__nullableString", "String", true)) engine.Init(t, ts) t.Run("non-nullable", func(t *testing.T) { np := engine.PrototypeByName("List__String") nrp := engine.PrototypeByName("List__String.Repr") var n schema.TypedNode t.Run("typed-create", func(t *testing.T) { n = fluent.MustBuildList(np, 2, func(la fluent.ListAssembler) { la.AssembleValue().AssignString("1") la.AssembleValue().AssignString("2") }).(schema.TypedNode) t.Run("typed-read", func(t *testing.T) { qt.Assert(t, n.Kind(), qt.Equals, datamodel.Kind_List) qt.Check(t, n.Length(), qt.Equals, int64(2)) qt.Check(t, must.String(must.Node(n.LookupByIndex(0))), qt.Equals, "1") qt.Check(t, must.String(must.Node(n.LookupByIndex(1))), qt.Equals, "2") qt.Check(t, must.String(must.Node(n.LookupBySegment(datamodel.PathSegmentOfInt(0)))), qt.Equals, "1") qt.Check(t, must.String(must.Node(n.LookupByNode(basicnode.NewInt(0)))), qt.Equals, "1") _, err := n.LookupByIndex(3) qt.Check(t, err, qt.ErrorAs, &datamodel.ErrNotExists{}) }) t.Run("repr-read", func(t *testing.T) { nr := n.Representation() qt.Assert(t, nr.Kind(), qt.Equals, datamodel.Kind_List) qt.Check(t, nr.Length(), qt.Equals, int64(2)) qt.Check(t, must.String(must.Node(nr.LookupByIndex(0))), qt.Equals, "1") qt.Check(t, must.String(must.Node(nr.LookupByIndex(1))), qt.Equals, "2") qt.Check(t, must.String(must.Node(n.LookupBySegment(datamodel.PathSegmentOfInt(0)))), qt.Equals, "1") qt.Check(t, must.String(must.Node(n.LookupByNode(basicnode.NewInt(0)))), qt.Equals, "1") _, err := n.LookupByIndex(3) qt.Check(t, err, qt.ErrorAs, &datamodel.ErrNotExists{}) }) }) t.Run("repr-create", func(t *testing.T) { nr := fluent.MustBuildList(nrp, 2, func(la fluent.ListAssembler) { la.AssembleValue().AssignString("1") la.AssembleValue().AssignString("2") }) qt.Check(t, n, NodeContentEquals, nr) }) }) t.Run("nullable", func(t *testing.T) { np := engine.PrototypeByName("List__nullableString") nrp := engine.PrototypeByName("List__nullableString.Repr") var n schema.TypedNode t.Run("typed-create", func(t *testing.T) { n = fluent.MustBuildList(np, 2, func(la fluent.ListAssembler) { la.AssembleValue().AssignString("1") la.AssembleValue().AssignNull() }).(schema.TypedNode) t.Run("typed-read", func(t *testing.T) { qt.Assert(t, n.Kind(), qt.Equals, datamodel.Kind_List) qt.Check(t, n.Length(), qt.Equals, int64(2)) qt.Check(t, must.String(must.Node(n.LookupByIndex(0))), qt.Equals, "1") qt.Check(t, must.Node(n.LookupByIndex(1)), qt.Equals, datamodel.Null) _, err := n.LookupByIndex(3) qt.Check(t, err, qt.ErrorAs, &datamodel.ErrNotExists{}) }) t.Run("repr-read", func(t *testing.T) { nr := n.Representation() qt.Assert(t, nr.Kind(), qt.Equals, datamodel.Kind_List) qt.Check(t, nr.Length(), qt.Equals, int64(2)) qt.Check(t, must.String(must.Node(n.LookupByIndex(0))), qt.Equals, "1") qt.Check(t, must.Node(n.LookupByIndex(1)), qt.Equals, datamodel.Null) _, err := n.LookupByIndex(3) qt.Check(t, err, qt.ErrorAs, &datamodel.ErrNotExists{}) }) }) t.Run("repr-create", func(t *testing.T) { nr := fluent.MustBuildList(nrp, 2, func(la fluent.ListAssembler) { la.AssembleValue().AssignString("1") la.AssembleValue().AssignNull() }) qt.Check(t, n, NodeContentEquals, nr) }) }) } // TestListsContainingLists is probing *two* things: // - that lists can nest, obviously // - that representation semantics are held correctly when we recurse, both in builders and in reading // // To cover that latter situation, this depends on structs (so we can use rename directives on the representation to make it distinctive). func SchemaTestListsContainingLists(t *testing.T, engine Engine) { ts := schema.TypeSystem{} ts.Init() ts.Accumulate(schema.SpawnString("String")) ts.Accumulate(schema.SpawnStruct("Frub", []schema.StructField{ schema.SpawnStructField("field", "String", false, false), // plain field. }, schema.SpawnStructRepresentationMap(map[string]string{ "field": "encoded", }), )) ts.Accumulate(schema.SpawnList("List__Frub", "Frub", false)) ts.Accumulate(schema.SpawnList("List__List__Frub", "List__Frub", true)) engine.Init(t, ts) np := engine.PrototypeByName("List__List__Frub") nrp := engine.PrototypeByName("List__List__Frub.Repr") var n schema.TypedNode t.Run("typed-create", func(t *testing.T) { n = fluent.MustBuildList(np, 3, func(la fluent.ListAssembler) { la.AssembleValue().CreateList(3, func(la fluent.ListAssembler) { la.AssembleValue().CreateMap(1, func(ma fluent.MapAssembler) { ma.AssembleEntry("field").AssignString("11") }) la.AssembleValue().CreateMap(1, func(ma fluent.MapAssembler) { ma.AssembleEntry("field").AssignString("12") }) la.AssembleValue().CreateMap(1, func(ma fluent.MapAssembler) { ma.AssembleEntry("field").AssignString("13") }) }) la.AssembleValue().CreateList(1, func(la fluent.ListAssembler) { la.AssembleValue().CreateMap(1, func(ma fluent.MapAssembler) { ma.AssembleEntry("field").AssignString("21") }) }) la.AssembleValue().CreateList(2, func(la fluent.ListAssembler) { la.AssembleValue().CreateMap(1, func(ma fluent.MapAssembler) { ma.AssembleEntry("field").AssignString("31") }) la.AssembleValue().CreateMap(1, func(ma fluent.MapAssembler) { ma.AssembleEntry("field").AssignString("32") }) }) }).(schema.TypedNode) t.Run("typed-read", func(t *testing.T) { qt.Assert(t, n.Kind(), qt.Equals, datamodel.Kind_List) qt.Assert(t, n.Length(), qt.Equals, int64(3)) qt.Assert(t, must.Node(n.LookupByIndex(0)).Length(), qt.Equals, int64(3)) qt.Assert(t, must.Node(n.LookupByIndex(1)).Length(), qt.Equals, int64(1)) qt.Assert(t, must.Node(n.LookupByIndex(2)).Length(), qt.Equals, int64(2)) qt.Check(t, must.String(must.Node(must.Node(must.Node(n.LookupByIndex(0)).LookupByIndex(0)).LookupByString("field"))), qt.Equals, "11") qt.Check(t, must.String(must.Node(must.Node(must.Node(n.LookupByIndex(0)).LookupByIndex(2)).LookupByString("field"))), qt.Equals, "13") qt.Check(t, must.String(must.Node(must.Node(must.Node(n.LookupByIndex(1)).LookupByIndex(0)).LookupByString("field"))), qt.Equals, "21") qt.Check(t, must.String(must.Node(must.Node(must.Node(n.LookupByIndex(2)).LookupByIndex(1)).LookupByString("field"))), qt.Equals, "32") }) t.Run("repr-read", func(t *testing.T) { nr := n.Representation() qt.Assert(t, nr.Kind(), qt.Equals, datamodel.Kind_List) qt.Assert(t, nr.Length(), qt.Equals, int64(3)) qt.Assert(t, must.Node(nr.LookupByIndex(0)).Length(), qt.Equals, int64(3)) qt.Assert(t, must.Node(nr.LookupByIndex(1)).Length(), qt.Equals, int64(1)) qt.Assert(t, must.Node(nr.LookupByIndex(2)).Length(), qt.Equals, int64(2)) qt.Check(t, must.String(must.Node(must.Node(must.Node(nr.LookupByIndex(0)).LookupByIndex(0)).LookupByString("encoded"))), qt.Equals, "11") qt.Check(t, must.String(must.Node(must.Node(must.Node(nr.LookupByIndex(0)).LookupByIndex(2)).LookupByString("encoded"))), qt.Equals, "13") qt.Check(t, must.String(must.Node(must.Node(must.Node(nr.LookupByIndex(1)).LookupByIndex(0)).LookupByString("encoded"))), qt.Equals, "21") qt.Check(t, must.String(must.Node(must.Node(must.Node(nr.LookupByIndex(2)).LookupByIndex(1)).LookupByString("encoded"))), qt.Equals, "32") }) }) t.Run("repr-create", func(t *testing.T) { nr := fluent.MustBuildList(nrp, 2, func(la fluent.ListAssembler) { // This is the same as the type-level create earlier, except note the field names are now all different. la.AssembleValue().CreateList(3, func(la fluent.ListAssembler) { la.AssembleValue().CreateMap(1, func(ma fluent.MapAssembler) { ma.AssembleEntry("encoded").AssignString("11") }) la.AssembleValue().CreateMap(1, func(ma fluent.MapAssembler) { ma.AssembleEntry("encoded").AssignString("12") }) la.AssembleValue().CreateMap(1, func(ma fluent.MapAssembler) { ma.AssembleEntry("encoded").AssignString("13") }) }) la.AssembleValue().CreateList(1, func(la fluent.ListAssembler) { la.AssembleValue().CreateMap(1, func(ma fluent.MapAssembler) { ma.AssembleEntry("encoded").AssignString("21") }) }) la.AssembleValue().CreateList(2, func(la fluent.ListAssembler) { la.AssembleValue().CreateMap(1, func(ma fluent.MapAssembler) { ma.AssembleEntry("encoded").AssignString("31") }) la.AssembleValue().CreateMap(1, func(ma fluent.MapAssembler) { ma.AssembleEntry("encoded").AssignString("32") }) }) }) qt.Check(t, n, NodeContentEquals, nr) }) } ================================================ FILE: node/tests/schemaMaps.go ================================================ package tests import ( "testing" qt "github.com/frankban/quicktest" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/fluent" "github.com/ipld/go-ipld-prime/must" "github.com/ipld/go-ipld-prime/schema" ) func SchemaTestMapsContainingMaybe(t *testing.T, engine Engine) { ts := schema.TypeSystem{} ts.Init() ts.Accumulate(schema.SpawnString("String")) ts.Accumulate(schema.SpawnMap("Map__String__String", "String", "String", false)) ts.Accumulate(schema.SpawnMap("Map__String__nullableString", "String", "String", true)) engine.Init(t, ts) t.Run("non-nullable", func(t *testing.T) { np := engine.PrototypeByName("Map__String__String") nrp := engine.PrototypeByName("Map__String__String.Repr") var n schema.TypedNode t.Run("typed-create", func(t *testing.T) { n = fluent.MustBuildMap(np, 2, func(ma fluent.MapAssembler) { ma.AssembleEntry("one").AssignString("1") ma.AssembleEntry("two").AssignString("2") }).(schema.TypedNode) t.Run("typed-read", func(t *testing.T) { qt.Assert(t, n.Kind(), qt.Equals, datamodel.Kind_Map) qt.Check(t, n.Length(), qt.Equals, int64(2)) qt.Check(t, must.String(must.Node(n.LookupByString("one"))), qt.Equals, "1") qt.Check(t, must.String(must.Node(n.LookupByString("two"))), qt.Equals, "2") _, err := n.LookupByString("miss") qt.Check(t, err, qt.ErrorAs, &datamodel.ErrNotExists{}) }) t.Run("repr-read", func(t *testing.T) { nr := n.Representation() qt.Assert(t, nr.Kind(), qt.Equals, datamodel.Kind_Map) qt.Check(t, nr.Length(), qt.Equals, int64(2)) qt.Check(t, must.String(must.Node(nr.LookupByString("one"))), qt.Equals, "1") qt.Check(t, must.String(must.Node(nr.LookupByString("two"))), qt.Equals, "2") _, err := nr.LookupByString("miss") qt.Check(t, err, qt.ErrorAs, &datamodel.ErrNotExists{}) }) }) t.Run("repr-create", func(t *testing.T) { nr := fluent.MustBuildMap(nrp, 2, func(ma fluent.MapAssembler) { ma.AssembleEntry("one").AssignString("1") ma.AssembleEntry("two").AssignString("2") }) qt.Check(t, n, NodeContentEquals, nr) }) }) t.Run("nullable", func(t *testing.T) { np := engine.PrototypeByName("Map__String__nullableString") nrp := engine.PrototypeByName("Map__String__nullableString.Repr") var n schema.TypedNode t.Run("typed-create", func(t *testing.T) { n = fluent.MustBuildMap(np, 2, func(ma fluent.MapAssembler) { ma.AssembleEntry("one").AssignString("1") ma.AssembleEntry("none").AssignNull() }).(schema.TypedNode) t.Run("typed-read", func(t *testing.T) { qt.Assert(t, n.Kind(), qt.Equals, datamodel.Kind_Map) qt.Check(t, n.Length(), qt.Equals, int64(2)) qt.Check(t, must.String(must.Node(n.LookupByString("one"))), qt.Equals, "1") qt.Check(t, must.Node(n.LookupByString("none")), qt.Equals, datamodel.Null) _, err := n.LookupByString("miss") qt.Check(t, err, qt.ErrorAs, &datamodel.ErrNotExists{}) }) t.Run("repr-read", func(t *testing.T) { nr := n.Representation() qt.Assert(t, nr.Kind(), qt.Equals, datamodel.Kind_Map) qt.Check(t, nr.Length(), qt.Equals, int64(2)) qt.Check(t, must.String(must.Node(nr.LookupByString("one"))), qt.Equals, "1") qt.Check(t, must.Node(nr.LookupByString("none")), qt.Equals, datamodel.Null) _, err := nr.LookupByString("miss") qt.Check(t, err, qt.ErrorAs, &datamodel.ErrNotExists{}) }) }) t.Run("repr-create", func(t *testing.T) { nr := fluent.MustBuildMap(nrp, 2, func(ma fluent.MapAssembler) { ma.AssembleEntry("one").AssignString("1") ma.AssembleEntry("none").AssignNull() }) qt.Check(t, n, NodeContentEquals, nr) }) }) } // TestMapsContainingMaps is probing *two* things: // - that maps can nest, obviously // - that representation semantics are held correctly when we recurse, both in builders and in reading // // To cover that latter situation, this depends on structs (so we can use rename directives on the representation to make it distinctive). func SchemaTestMapsContainingMaps(t *testing.T, engine Engine) { ts := schema.TypeSystem{} ts.Init() ts.Accumulate(schema.SpawnString("String")) ts.Accumulate(schema.SpawnStruct("Frub", // "type Frub struct { field String (rename "encoded") }" []schema.StructField{ schema.SpawnStructField("field", "String", false, false), // plain field. }, schema.SpawnStructRepresentationMap(map[string]string{ "field": "encoded", }), )) ts.Accumulate(schema.SpawnMap("Map__String__Frub", // "{String:Frub}" "String", "Frub", false)) ts.Accumulate(schema.SpawnMap("Map__String__nullableMap__String__Frub", // "{String:nullable {String:Frub}}" "String", "Map__String__Frub", true)) engine.Init(t, ts) np := engine.PrototypeByName("Map__String__nullableMap__String__Frub") nrp := engine.PrototypeByName("Map__String__nullableMap__String__Frub.Repr") creation := func(t *testing.T, np datamodel.NodePrototype, fieldName string) schema.TypedNode { return fluent.MustBuildMap(np, 3, func(ma fluent.MapAssembler) { ma.AssembleEntry("one").CreateMap(2, func(ma fluent.MapAssembler) { ma.AssembleEntry("zot").CreateMap(1, func(ma fluent.MapAssembler) { ma.AssembleEntry(fieldName).AssignString("11") }) ma.AssembleEntry("zop").CreateMap(1, func(ma fluent.MapAssembler) { ma.AssembleEntry(fieldName).AssignString("12") }) }) ma.AssembleEntry("two").CreateMap(1, func(ma fluent.MapAssembler) { ma.AssembleEntry("zim").CreateMap(1, func(ma fluent.MapAssembler) { ma.AssembleEntry(fieldName).AssignString("21") }) }) ma.AssembleEntry("none").AssignNull() }).(schema.TypedNode) } reading := func(t *testing.T, n datamodel.Node, fieldName string) { withNode(n, func(n datamodel.Node) { qt.Assert(t, n.Kind(), qt.Equals, datamodel.Kind_Map) qt.Check(t, n.Length(), qt.Equals, int64(3)) withNode(must.Node(n.LookupByString("one")), func(n datamodel.Node) { qt.Assert(t, n.Kind(), qt.Equals, datamodel.Kind_Map) qt.Check(t, n.Length(), qt.Equals, int64(2)) withNode(must.Node(n.LookupByString("zot")), func(n datamodel.Node) { qt.Assert(t, n.Kind(), qt.Equals, datamodel.Kind_Map) qt.Check(t, n.Length(), qt.Equals, int64(1)) qt.Check(t, must.String(must.Node(n.LookupByString(fieldName))), qt.Equals, "11") }) withNode(must.Node(n.LookupByString("zop")), func(n datamodel.Node) { qt.Assert(t, n.Kind(), qt.Equals, datamodel.Kind_Map) qt.Check(t, n.Length(), qt.Equals, int64(1)) qt.Check(t, must.String(must.Node(n.LookupByString(fieldName))), qt.Equals, "12") }) }) withNode(must.Node(n.LookupByString("two")), func(n datamodel.Node) { qt.Check(t, n.Length(), qt.Equals, int64(1)) withNode(must.Node(n.LookupByString("zim")), func(n datamodel.Node) { qt.Assert(t, n.Kind(), qt.Equals, datamodel.Kind_Map) qt.Check(t, n.Length(), qt.Equals, int64(1)) qt.Check(t, must.String(must.Node(n.LookupByString(fieldName))), qt.Equals, "21") }) }) withNode(must.Node(n.LookupByString("none")), func(n datamodel.Node) { qt.Check(t, n, NodeContentEquals, datamodel.Null) }) _, err := n.LookupByString("miss") qt.Check(t, err, qt.ErrorAs, &datamodel.ErrNotExists{}) }) } var n schema.TypedNode t.Run("typed-create", func(t *testing.T) { n = creation(t, np, "field") t.Run("typed-read", func(t *testing.T) { reading(t, n, "field") }) t.Run("repr-read", func(t *testing.T) { reading(t, n.Representation(), "encoded") }) }) t.Run("repr-create", func(t *testing.T) { nr := creation(t, nrp, "encoded") qt.Check(t, n, NodeContentEquals, nr) }) } func SchemaTestMapsWithComplexKeys(t *testing.T, engine Engine) { ts := schema.TypeSystem{} ts.Init() ts.Accumulate(schema.SpawnString("String")) ts.Accumulate(schema.SpawnStruct("StringyStruct", []schema.StructField{ schema.SpawnStructField("foo", "String", false, false), schema.SpawnStructField("bar", "String", false, false), }, schema.SpawnStructRepresentationStringjoin(":"), )) ts.Accumulate(schema.SpawnMap("Map__StringyStruct__String", "StringyStruct", "String", false)) engine.Init(t, ts) np := engine.PrototypeByName("Map__StringyStruct__String") nrp := engine.PrototypeByName("Map__StringyStruct__String.Repr") var n schema.TypedNode t.Run("typed-create", func(t *testing.T) { n = fluent.MustBuildMap(np, 3, func(ma fluent.MapAssembler) { ma.AssembleKey().CreateMap(2, func(ma fluent.MapAssembler) { ma.AssembleEntry("foo").AssignString("a") ma.AssembleEntry("bar").AssignString("b") }) ma.AssembleValue().AssignString("1") ma.AssembleKey().CreateMap(2, func(ma fluent.MapAssembler) { ma.AssembleEntry("foo").AssignString("c") ma.AssembleEntry("bar").AssignString("d") }) ma.AssembleValue().AssignString("2") ma.AssembleKey().CreateMap(2, func(ma fluent.MapAssembler) { ma.AssembleEntry("foo").AssignString("e") ma.AssembleEntry("bar").AssignString("f") }) ma.AssembleValue().AssignString("3") }).(schema.TypedNode) t.Run("typed-read", func(t *testing.T) { qt.Assert(t, n.Kind(), qt.Equals, datamodel.Kind_Map) qt.Check(t, n.Length(), qt.Equals, int64(3)) n2 := must.Node(n.LookupByString("c:d")) qt.Assert(t, n2.Kind(), qt.Equals, datamodel.Kind_String) qt.Check(t, must.String(n2), qt.Equals, "2") }) t.Run("repr-read", func(t *testing.T) { nr := n.Representation() qt.Assert(t, nr.Kind(), qt.Equals, datamodel.Kind_Map) qt.Check(t, nr.Length(), qt.Equals, int64(3)) n2 := must.Node(nr.LookupByString("c:d")) qt.Assert(t, n2.Kind(), qt.Equals, datamodel.Kind_String) qt.Check(t, must.String(n2), qt.Equals, "2") }) }) t.Run("repr-create", func(t *testing.T) { nr := fluent.MustBuildMap(nrp, 3, func(ma fluent.MapAssembler) { ma.AssembleEntry("a:b").AssignString("1") ma.AssembleEntry("c:d").AssignString("2") ma.AssembleEntry("e:f").AssignString("3") }) qt.Check(t, n, NodeContentEquals, nr) }) } ================================================ FILE: node/tests/schemaScalars.go ================================================ package tests import ( "fmt" "testing" qt "github.com/frankban/quicktest" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/schema" ) func assignValue(am datamodel.NodeAssembler, value interface{}) error { switch value := value.(type) { case bool: return am.AssignBool(value) case int64: return am.AssignInt(value) case float64: return am.AssignFloat(value) case string: return am.AssignString(value) case []byte: return am.AssignBytes(value) default: panic(fmt.Sprintf("%T", value)) } } func SchemaTestScalars(t *testing.T, engine Engine) { ts := schema.TypeSystem{} ts.Init() ts.Accumulate(schema.SpawnBool("Bool")) ts.Accumulate(schema.SpawnInt("Int")) ts.Accumulate(schema.SpawnFloat("Float")) ts.Accumulate(schema.SpawnString("String")) ts.Accumulate(schema.SpawnBytes("Bytes")) engine.Init(t, ts) var tests = []struct { name string kind datamodel.Kind value interface{} }{ {"Bool", datamodel.Kind_Bool, true}, {"Int", datamodel.Kind_Int, int64(23)}, {"Float", datamodel.Kind_Float, 12.25}, {"String", datamodel.Kind_String, "foo"}, {"Bytes", datamodel.Kind_Bytes, []byte("bar")}, } // We test each of the five scalar prototypes in subtests. for _, testProto := range tests { // For both the regular node and its repr version, // getting the right value for the kind should work. for _, useRepr := range []bool{false, true} { protoName := testProto.name if useRepr { protoName += ".Repr" } np := engine.PrototypeByName(protoName) // For each prototype, we try assigning all scalar values. for _, testAssign := range tests { // We try both AssignKind and AssignNode. for _, useAssignNode := range []bool{false, true} { testName := fmt.Sprintf("%s-Assign%s", protoName, testAssign.name) if useAssignNode { testName = fmt.Sprintf("%s-AssignNode-%s", protoName, testAssign.name) } t.Run(testName, func(t *testing.T) { nb := np.NewBuilder() // Assigning null, a list, or a map, should always fail. err := nb.AssignNull() qt.Check(t, err, qt.Not(qt.IsNil)) _, err = nb.BeginMap(-1) qt.Check(t, err, qt.Not(qt.IsNil)) _, err = nb.BeginList(-1) qt.Check(t, err, qt.Not(qt.IsNil)) // Assigning the right value for the kind should succeed. if useAssignNode { np2 := engine.PrototypeByName(testAssign.name) nb2 := np2.NewBuilder() qt.Check(t, assignValue(nb2, testAssign.value), qt.IsNil) n2 := nb2.Build() err = nb.AssignNode(n2) } else { err = assignValue(nb, testAssign.value) } if testAssign.kind == testProto.kind { qt.Check(t, err, qt.IsNil) } else { qt.Check(t, err, qt.Not(qt.IsNil)) // Assign something anyway, just so we can Build later. err := assignValue(nb, testProto.value) qt.Check(t, err, qt.IsNil) } n := nb.Build() var gotValue interface{} err = nil switch testAssign.kind { case datamodel.Kind_Bool: gotValue, err = n.AsBool() case datamodel.Kind_Int: gotValue, err = n.AsInt() case datamodel.Kind_Float: gotValue, err = n.AsFloat() case datamodel.Kind_String: gotValue, err = n.AsString() case datamodel.Kind_Bytes: gotValue, err = n.AsBytes() default: t.Fatal(testAssign.kind) } if testAssign.kind == testProto.kind { qt.Check(t, err, qt.IsNil) qt.Check(t, gotValue, qt.DeepEquals, testAssign.value) } else { qt.Check(t, err, qt.Not(qt.IsNil)) } // Using Node methods which should never // work on scalar kinds. _, err = n.LookupByString("foo") qt.Check(t, err, qt.Not(qt.IsNil)) _, err = n.LookupByIndex(3) qt.Check(t, err, qt.Not(qt.IsNil)) qt.Check(t, n.MapIterator(), qt.IsNil) qt.Check(t, n.ListIterator(), qt.IsNil) qt.Check(t, n.Length(), qt.Equals, int64(-1)) qt.Check(t, n.IsAbsent(), qt.IsFalse) qt.Check(t, n.IsNull(), qt.IsFalse) }) } } } } } ================================================ FILE: node/tests/schemaStruct.go ================================================ package tests import ( "testing" qt "github.com/frankban/quicktest" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/fluent" "github.com/ipld/go-ipld-prime/must" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/schema" ) func SchemaTestRequiredFields(t *testing.T, engine Engine) { ts := schema.TypeSystem{} ts.Init() ts.Accumulate(schema.SpawnString("String")) ts.Accumulate(schema.SpawnStruct("StructOne", []schema.StructField{ schema.SpawnStructField("a", "String", false, false), schema.SpawnStructField("b", "String", false, false), }, schema.SpawnStructRepresentationMap(map[string]string{ // no renames. we expect a simpler error message in this case. }), )) ts.Accumulate(schema.SpawnStruct("StructTwo", []schema.StructField{ schema.SpawnStructField("a", "String", false, false), schema.SpawnStructField("b", "String", false, false), }, schema.SpawnStructRepresentationMap(map[string]string{ "b": "z", }), )) engine.Init(t, ts) t.Run("building-type-without-required-fields-errors", func(t *testing.T) { np := engine.PrototypeByName("StructOne") nb := np.NewBuilder() ma, _ := nb.BeginMap(0) err := ma.Finish() qt.Check(t, err, qt.ErrorAs, &schema.ErrMissingRequiredField{}) qt.Check(t, err.Error(), qt.Equals, `missing required fields: a,b`) }) t.Run("building-representation-without-required-fields-errors", func(t *testing.T) { nrp := engine.PrototypeByName("StructOne.Repr") nb := nrp.NewBuilder() ma, _ := nb.BeginMap(0) err := ma.Finish() qt.Check(t, err, qt.ErrorAs, &schema.ErrMissingRequiredField{}) qt.Check(t, err.Error(), qt.Equals, `missing required fields: a,b`) }) t.Run("building-representation-with-renames-without-required-fields-errors", func(t *testing.T) { nrp := engine.PrototypeByName("StructTwo.Repr") nb := nrp.NewBuilder() ma, _ := nb.BeginMap(0) err := ma.Finish() qt.Check(t, err, qt.ErrorAs, &schema.ErrMissingRequiredField{}) qt.Check(t, err.Error(), qt.Equals, `missing required fields: a,b (serial:"z")`) }) } func SchemaTestStructNesting(t *testing.T, engine Engine) { ts := schema.TypeSystem{} ts.Init() ts.Accumulate(schema.SpawnString("String")) ts.Accumulate(schema.SpawnStruct("SmolStruct", []schema.StructField{ schema.SpawnStructField("s", "String", false, false), }, schema.SpawnStructRepresentationMap(map[string]string{ "s": "q", }), )) ts.Accumulate(schema.SpawnStruct("GulpoStruct", []schema.StructField{ schema.SpawnStructField("x", "SmolStruct", false, false), }, schema.SpawnStructRepresentationMap(map[string]string{ "x": "r", }), )) engine.Init(t, ts) np := engine.PrototypeByName("GulpoStruct") nrp := engine.PrototypeByName("GulpoStruct.Repr") var n schema.TypedNode t.Run("typed-create", func(t *testing.T) { n = fluent.MustBuildMap(np, 1, func(ma fluent.MapAssembler) { ma.AssembleEntry("x").CreateMap(1, func(ma fluent.MapAssembler) { ma.AssembleEntry("s").AssignString("woo") }) }).(schema.TypedNode) t.Run("typed-read", func(t *testing.T) { qt.Assert(t, n.Kind(), qt.Equals, datamodel.Kind_Map) qt.Check(t, n.Length(), qt.Equals, int64(1)) n2 := must.Node(n.LookupByString("x")) qt.Assert(t, n2.Kind(), qt.Equals, datamodel.Kind_Map) n2Seg := must.Node(n.LookupBySegment(datamodel.PathSegmentOfString("x"))) qt.Check(t, n2, NodeContentEquals, n2Seg) n2Node := must.Node(n.LookupByNode(basicnode.NewString("x"))) qt.Check(t, n2, NodeContentEquals, n2Node) qt.Check(t, must.String(must.Node(n2.LookupByString("s"))), qt.Equals, "woo") }) t.Run("repr-read", func(t *testing.T) { nr := n.Representation() qt.Assert(t, nr.Kind(), qt.Equals, datamodel.Kind_Map) qt.Check(t, nr.Length(), qt.Equals, int64(1)) n2 := must.Node(nr.LookupByString("r")) qt.Assert(t, n2.Kind(), qt.Equals, datamodel.Kind_Map) n2Seg := must.Node(nr.LookupBySegment(datamodel.PathSegmentOfString("r"))) qt.Check(t, n2, NodeContentEquals, n2Seg) n2Node := must.Node(nr.LookupByNode(basicnode.NewString("r"))) qt.Check(t, n2, NodeContentEquals, n2Node) qt.Check(t, must.String(must.Node(n2.LookupByString("q"))), qt.Equals, "woo") }) }) t.Run("repr-create", func(t *testing.T) { nr := fluent.MustBuildMap(nrp, 1, func(ma fluent.MapAssembler) { ma.AssembleEntry("r").CreateMap(1, func(ma fluent.MapAssembler) { ma.AssembleEntry("q").AssignString("woo") }) }) qt.Check(t, n, NodeContentEquals, nr) }) } ================================================ FILE: node/tests/schemaStructReprListpairs.go ================================================ package tests import ( "testing" qt "github.com/frankban/quicktest" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/fluent" "github.com/ipld/go-ipld-prime/must" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/schema" ) func SchemaTestStructReprListPairs(t *testing.T, engine Engine) { ts := schema.TypeSystem{} ts.Init() ts.Accumulate(schema.SpawnString("String")) ts.Accumulate(schema.SpawnList("List__String", "String", false)) ts.Accumulate(schema.SpawnStruct("OneListPair", []schema.StructField{ schema.SpawnStructField("field", "String", false, false), }, schema.SpawnStructRepresentationListPairs(), )) ts.Accumulate(schema.SpawnStruct("FourListPairs", []schema.StructField{ schema.SpawnStructField("foo", "String", false, true), schema.SpawnStructField("bar", "String", true, true), schema.SpawnStructField("baz", "String", true, false), schema.SpawnStructField("qux", "List__String", false, false), }, schema.SpawnStructRepresentationListPairs(), )) ts.Accumulate(schema.SpawnStruct("NestedListPairs", []schema.StructField{ schema.SpawnStructField("str", "String", false, false), schema.SpawnStructField("lp", "OneListPair", false, false), }, schema.SpawnStructRepresentationListPairs(), )) engine.Init(t, ts) t.Run("onelistpair works", func(t *testing.T) { np := engine.PrototypeByName("OneListPair") nrp := engine.PrototypeByName("OneListPair.Repr") var n schema.TypedNode t.Run("typed-create", func(t *testing.T) { n = fluent.MustBuildMap(np, 1, func(ma fluent.MapAssembler) { ma.AssembleEntry("field").AssignString("valoo") }).(schema.TypedNode) t.Run("typed-read", func(t *testing.T) { qt.Assert(t, n.Kind(), qt.Equals, datamodel.Kind_Map) qt.Check(t, n.Length(), qt.Equals, int64(1)) qt.Check(t, must.String(must.Node(n.LookupByString("field"))), qt.Equals, "valoo") }) t.Run("repr-read", func(t *testing.T) { nr := n.Representation() qt.Assert(t, nr.Kind(), qt.Equals, datamodel.Kind_List) qt.Check(t, nr.Length(), qt.Equals, int64(1)) kv := must.Node(nr.LookupByIndex(0)) qt.Assert(t, nr.Kind(), qt.Equals, datamodel.Kind_List) qt.Check(t, kv.Length(), qt.Equals, int64(2)) qt.Check(t, must.String(must.Node(kv.LookupByIndex(0))), qt.Equals, "field") qt.Check(t, must.String(must.Node(kv.LookupByIndex(1))), qt.Equals, "valoo") }) }) t.Run("repr-create", func(t *testing.T) { nr := fluent.MustBuildList(nrp, 1, func(la fluent.ListAssembler) { la.AssembleValue().CreateList(2, func(la fluent.ListAssembler) { la.AssembleValue().AssignString("field") la.AssembleValue().AssignString("valoo") }) }) qt.Check(t, n, NodeContentEquals, nr) }) }) t.Run("fourlistpairs works", func(t *testing.T) { np := engine.PrototypeByName("FourListPairs") nrp := engine.PrototypeByName("FourListPairs.Repr") var n schema.TypedNode t.Run("typed-create", func(t *testing.T) { n = fluent.MustBuildMap(np, 4, func(ma fluent.MapAssembler) { ma.AssembleEntry("foo").AssignString("0") ma.AssembleEntry("bar").AssignString("1") ma.AssembleEntry("baz").AssignString("2") ma.AssembleEntry("qux").CreateList(2, func(la fluent.ListAssembler) { la.AssembleValue().AssignString("3") la.AssembleValue().AssignString("4") }) }).(schema.TypedNode) t.Run("typed-read", func(t *testing.T) { qt.Assert(t, n.Kind(), qt.Equals, datamodel.Kind_Map) qt.Check(t, n.Length(), qt.Equals, int64(4)) qt.Check(t, must.String(must.Node(n.LookupByString("foo"))), qt.Equals, "0") qt.Check(t, must.String(must.Node(n.LookupByString("bar"))), qt.Equals, "1") qt.Check(t, must.String(must.Node(n.LookupByString("baz"))), qt.Equals, "2") qux := must.Node(n.LookupByString("qux")) qt.Assert(t, qux.Kind(), qt.Equals, datamodel.Kind_List) qt.Check(t, qux.Length(), qt.Equals, int64(2)) qt.Check(t, must.String(must.Node(qux.LookupByIndex(0))), qt.Equals, "3") qt.Check(t, must.String(must.Node(qux.LookupByIndex(1))), qt.Equals, "4") }) t.Run("repr-read", func(t *testing.T) { nr := n.Representation() qt.Assert(t, nr.Kind(), qt.Equals, datamodel.Kind_List) qt.Check(t, nr.Length(), qt.Equals, int64(4)) kv := must.Node(nr.LookupByIndex(0)) qt.Assert(t, nr.Kind(), qt.Equals, datamodel.Kind_List) qt.Check(t, kv.Length(), qt.Equals, int64(2)) qt.Check(t, must.String(must.Node(kv.LookupByIndex(0))), qt.Equals, "foo") qt.Check(t, must.String(must.Node(kv.LookupByIndex(1))), qt.Equals, "0") kv = must.Node(nr.LookupByIndex(1)) qt.Assert(t, nr.Kind(), qt.Equals, datamodel.Kind_List) qt.Check(t, kv.Length(), qt.Equals, int64(2)) qt.Check(t, must.String(must.Node(kv.LookupByIndex(0))), qt.Equals, "bar") qt.Check(t, must.String(must.Node(kv.LookupByIndex(1))), qt.Equals, "1") kv = must.Node(nr.LookupByIndex(2)) qt.Assert(t, nr.Kind(), qt.Equals, datamodel.Kind_List) qt.Check(t, kv.Length(), qt.Equals, int64(2)) qt.Check(t, must.String(must.Node(kv.LookupByIndex(0))), qt.Equals, "baz") qt.Check(t, must.String(must.Node(kv.LookupByIndex(1))), qt.Equals, "2") kv = must.Node(nr.LookupByIndex(3)) qt.Assert(t, nr.Kind(), qt.Equals, datamodel.Kind_List) qt.Check(t, kv.Length(), qt.Equals, int64(2)) qt.Check(t, must.String(must.Node(kv.LookupByIndex(0))), qt.Equals, "qux") qux := must.Node(kv.LookupByIndex(1)) qt.Assert(t, qux.Kind(), qt.Equals, datamodel.Kind_List) qt.Check(t, qux.Length(), qt.Equals, int64(2)) qt.Check(t, must.String(must.Node(qux.LookupByIndex(0))), qt.Equals, "3") qt.Check(t, must.String(must.Node(qux.LookupByIndex(1))), qt.Equals, "4") }) }) t.Run("repr-create", func(t *testing.T) { nr := fluent.MustBuildList(nrp, 4, func(la fluent.ListAssembler) { la.AssembleValue().CreateList(2, func(la fluent.ListAssembler) { la.AssembleValue().AssignString("foo") la.AssembleValue().AssignString("0") }) la.AssembleValue().CreateList(2, func(la fluent.ListAssembler) { la.AssembleValue().AssignString("bar") la.AssembleValue().AssignString("1") }) la.AssembleValue().CreateList(2, func(la fluent.ListAssembler) { la.AssembleValue().AssignString("baz") la.AssembleValue().AssignString("2") }) la.AssembleValue().CreateList(2, func(la fluent.ListAssembler) { la.AssembleValue().AssignString("qux") la.AssembleValue().CreateList(2, func(la fluent.ListAssembler) { la.AssembleValue().AssignString("3") la.AssembleValue().AssignString("4") }) }) }) qt.Check(t, n, NodeContentEquals, nr) }) t.Run("repr-create out-of-order", func(t *testing.T) { nr := fluent.MustBuildList(nrp, 4, func(la fluent.ListAssembler) { la.AssembleValue().CreateList(2, func(la fluent.ListAssembler) { la.AssembleValue().AssignString("bar") la.AssembleValue().AssignString("1") }) la.AssembleValue().CreateList(2, func(la fluent.ListAssembler) { la.AssembleValue().AssignString("foo") la.AssembleValue().AssignString("0") }) la.AssembleValue().CreateList(2, func(la fluent.ListAssembler) { la.AssembleValue().AssignString("qux") la.AssembleValue().CreateList(2, func(la fluent.ListAssembler) { la.AssembleValue().AssignString("3") la.AssembleValue().AssignString("4") }) }) la.AssembleValue().CreateList(2, func(la fluent.ListAssembler) { la.AssembleValue().AssignString("baz") la.AssembleValue().AssignString("2") }) }) qt.Check(t, n, NodeContentEquals, nr) }) }) t.Run("fourlistpairs with absents", func(t *testing.T) { np := engine.PrototypeByName("FourListPairs") nrp := engine.PrototypeByName("FourListPairs.Repr") var n schema.TypedNode t.Run("typed-create", func(t *testing.T) { n = fluent.MustBuildMap(np, 2, func(ma fluent.MapAssembler) { ma.AssembleEntry("foo").AssignNull() ma.AssembleEntry("qux").CreateList(2, func(la fluent.ListAssembler) { la.AssembleValue().AssignString("1") la.AssembleValue().AssignString("2") }) }).(schema.TypedNode) t.Run("typed-read", func(t *testing.T) { qt.Assert(t, n.Kind(), qt.Equals, datamodel.Kind_Map) qt.Check(t, n.Length(), qt.Equals, int64(4)) qt.Check(t, must.Node(n.LookupByString("foo")), qt.Equals, datamodel.Null) qt.Check(t, must.Node(n.LookupByString("bar")), qt.Equals, datamodel.Absent) qt.Check(t, must.Node(n.LookupByString("baz")), qt.Equals, datamodel.Absent) qux := must.Node(n.LookupByString("qux")) qt.Assert(t, qux.Kind(), qt.Equals, datamodel.Kind_List) qt.Check(t, qux.Length(), qt.Equals, int64(2)) qt.Check(t, must.String(must.Node(qux.LookupByIndex(0))), qt.Equals, "1") qt.Check(t, must.String(must.Node(qux.LookupByIndex(1))), qt.Equals, "2") }) t.Run("repr-read", func(t *testing.T) { nr := n.Representation() qt.Assert(t, nr.Kind(), qt.Equals, datamodel.Kind_List) qt.Check(t, nr.Length(), qt.Equals, int64(2)) kv := must.Node(nr.LookupByIndex(0)) qt.Assert(t, nr.Kind(), qt.Equals, datamodel.Kind_List) qt.Check(t, kv.Length(), qt.Equals, int64(2)) qt.Check(t, must.String(must.Node(kv.LookupByIndex(0))), qt.Equals, "foo") qt.Check(t, must.Node(kv.LookupByIndex(1)), qt.Equals, datamodel.Null) kv = must.Node(nr.LookupByIndex(1)) qt.Assert(t, nr.Kind(), qt.Equals, datamodel.Kind_List) qt.Check(t, kv.Length(), qt.Equals, int64(2)) qt.Check(t, must.String(must.Node(kv.LookupByIndex(0))), qt.Equals, "qux") qux := must.Node(kv.LookupByIndex(1)) qt.Assert(t, qux.Kind(), qt.Equals, datamodel.Kind_List) qt.Check(t, qux.Length(), qt.Equals, int64(2)) qt.Check(t, must.String(must.Node(qux.LookupByIndex(0))), qt.Equals, "1") qt.Check(t, must.String(must.Node(qux.LookupByIndex(1))), qt.Equals, "2") }) }) t.Run("repr-create", func(t *testing.T) { nr := fluent.MustBuildList(nrp, 2, func(la fluent.ListAssembler) { la.AssembleValue().CreateList(2, func(la fluent.ListAssembler) { la.AssembleValue().AssignString("foo") la.AssembleValue().AssignNull() }) la.AssembleValue().CreateList(2, func(la fluent.ListAssembler) { la.AssembleValue().AssignString("qux") la.AssembleValue().CreateList(2, func(la fluent.ListAssembler) { la.AssembleValue().AssignString("1") la.AssembleValue().AssignString("2") }) }) }) qt.Check(t, n, NodeContentEquals, nr) }) t.Run("repr-create with AssignNode", func(t *testing.T) { nr := fluent.MustBuildList(basicnode.Prototype.Any, 2, func(la fluent.ListAssembler) { la.AssembleValue().CreateList(2, func(la fluent.ListAssembler) { la.AssembleValue().AssignString("foo") la.AssembleValue().AssignNull() }) la.AssembleValue().CreateList(2, func(la fluent.ListAssembler) { la.AssembleValue().AssignString("qux") la.AssembleValue().CreateList(2, func(la fluent.ListAssembler) { la.AssembleValue().AssignString("1") la.AssembleValue().AssignString("2") }) }) }) builder := nrp.NewBuilder() err := builder.AssignNode(nr) qt.Assert(t, err, qt.IsNil) anr := builder.Build() qt.Check(t, n, NodeContentEquals, anr) }) }) t.Run("nestedlistpairs works", func(t *testing.T) { np := engine.PrototypeByName("NestedListPairs") nrp := engine.PrototypeByName("NestedListPairs.Repr") var n schema.TypedNode t.Run("typed-create", func(t *testing.T) { n = fluent.MustBuildMap(np, 1, func(ma fluent.MapAssembler) { ma.AssembleEntry("str").AssignString("boop") ma.AssembleEntry("lp").CreateMap(1, func(ma fluent.MapAssembler) { ma.AssembleEntry("field").AssignString("valoo") }) }).(schema.TypedNode) t.Run("typed-read", func(t *testing.T) { qt.Assert(t, n.Kind(), qt.Equals, datamodel.Kind_Map) qt.Check(t, n.Length(), qt.Equals, int64(2)) qt.Check(t, must.String(must.Node(n.LookupByString("str"))), qt.Equals, "boop") lp := must.Node(n.LookupByString("lp")) qt.Check(t, lp.Kind(), qt.Equals, datamodel.Kind_Map) qt.Check(t, must.String(must.Node(lp.LookupByString("field"))), qt.Equals, "valoo") }) t.Run("repr-read", func(t *testing.T) { nr := n.Representation() qt.Assert(t, nr.Kind(), qt.Equals, datamodel.Kind_List) qt.Check(t, nr.Length(), qt.Equals, int64(2)) kv := must.Node(nr.LookupByIndex(0)) qt.Assert(t, nr.Kind(), qt.Equals, datamodel.Kind_List) qt.Check(t, kv.Length(), qt.Equals, int64(2)) qt.Check(t, must.String(must.Node(kv.LookupByIndex(0))), qt.Equals, "str") qt.Check(t, must.String(must.Node(kv.LookupByIndex(1))), qt.Equals, "boop") kv = must.Node(nr.LookupByIndex(1)) qt.Assert(t, nr.Kind(), qt.Equals, datamodel.Kind_List) qt.Check(t, kv.Length(), qt.Equals, int64(2)) qt.Check(t, must.String(must.Node(kv.LookupByIndex(0))), qt.Equals, "lp") lp := must.Node(kv.LookupByIndex(1)) qt.Check(t, lp.Kind(), qt.Equals, datamodel.Kind_List) kv = must.Node(lp.LookupByIndex(0)) qt.Check(t, kv.Length(), qt.Equals, int64(2)) qt.Check(t, must.String(must.Node(kv.LookupByIndex(0))), qt.Equals, "field") qt.Check(t, must.String(must.Node(kv.LookupByIndex(1))), qt.Equals, "valoo") }) }) t.Run("repr-create", func(t *testing.T) { nr := fluent.MustBuildList(nrp, 1, func(la fluent.ListAssembler) { la.AssembleValue().CreateList(2, func(la fluent.ListAssembler) { la.AssembleValue().AssignString("str") la.AssembleValue().AssignString("boop") }) la.AssembleValue().CreateList(2, func(la fluent.ListAssembler) { la.AssembleValue().AssignString("lp") la.AssembleValue().CreateList(2, func(la fluent.ListAssembler) { la.AssembleValue().CreateList(2, func(la fluent.ListAssembler) { la.AssembleValue().AssignString("field") la.AssembleValue().AssignString("valoo") }) }) }) }) qt.Check(t, n, NodeContentEquals, nr) }) }) } ================================================ FILE: node/tests/schemaStructReprStringjoin.go ================================================ package tests import ( "testing" qt "github.com/frankban/quicktest" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/fluent" "github.com/ipld/go-ipld-prime/must" "github.com/ipld/go-ipld-prime/schema" ) // TestStructReprStringjoin exercises... well, what it says on the tin. // // These should pass even if the natural map representation doesn't. // No maybes are exercised. func SchemaTestStructReprStringjoin(t *testing.T, engine Engine) { ts := schema.TypeSystem{} ts.Init() ts.Accumulate(schema.SpawnString("String")) ts.Accumulate(schema.SpawnStruct("StringyStruct", []schema.StructField{ schema.SpawnStructField("field", "String", false, false), }, schema.SpawnStructRepresentationStringjoin(":"), )) ts.Accumulate(schema.SpawnStruct("ManystringStruct", []schema.StructField{ schema.SpawnStructField("foo", "String", false, false), schema.SpawnStructField("bar", "String", false, false), }, schema.SpawnStructRepresentationStringjoin(":"), )) ts.Accumulate(schema.SpawnStruct("Recurzorator", []schema.StructField{ schema.SpawnStructField("foo", "String", false, false), schema.SpawnStructField("zap", "ManystringStruct", false, false), schema.SpawnStructField("bar", "String", false, false), }, schema.SpawnStructRepresentationStringjoin("-"), )) engine.Init(t, ts) t.Run("single field works", func(t *testing.T) { np := engine.PrototypeByName("StringyStruct") nrp := engine.PrototypeByName("StringyStruct.Repr") var n schema.TypedNode t.Run("typed-create", func(t *testing.T) { n = fluent.MustBuildMap(np, 1, func(ma fluent.MapAssembler) { ma.AssembleEntry("field").AssignString("valoo") }).(schema.TypedNode) t.Run("typed-read", func(t *testing.T) { qt.Assert(t, n.Kind(), qt.Equals, datamodel.Kind_Map) qt.Check(t, n.Length(), qt.Equals, int64(1)) qt.Check(t, must.String(must.Node(n.LookupByString("field"))), qt.Equals, "valoo") }) t.Run("repr-read", func(t *testing.T) { nr := n.Representation() qt.Assert(t, nr.Kind(), qt.Equals, datamodel.Kind_String) qt.Check(t, must.String(nr), qt.Equals, "valoo") }) }) t.Run("repr-create", func(t *testing.T) { nr := fluent.MustBuild(nrp, func(na fluent.NodeAssembler) { na.AssignString("valoo") }) qt.Check(t, n, NodeContentEquals, nr) }) }) t.Run("several fields work", func(t *testing.T) { np := engine.PrototypeByName("ManystringStruct") nrp := engine.PrototypeByName("ManystringStruct.Repr") var n schema.TypedNode t.Run("typed-create", func(t *testing.T) { n = fluent.MustBuildMap(np, 2, func(ma fluent.MapAssembler) { ma.AssembleEntry("foo").AssignString("v1") ma.AssembleEntry("bar").AssignString("v2") }).(schema.TypedNode) t.Run("typed-read", func(t *testing.T) { qt.Assert(t, n.Kind(), qt.Equals, datamodel.Kind_Map) qt.Check(t, n.Length(), qt.Equals, int64(2)) qt.Check(t, must.String(must.Node(n.LookupByString("foo"))), qt.Equals, "v1") qt.Check(t, must.String(must.Node(n.LookupByString("bar"))), qt.Equals, "v2") }) t.Run("repr-read", func(t *testing.T) { nr := n.Representation() qt.Assert(t, nr.Kind(), qt.Equals, datamodel.Kind_String) qt.Check(t, must.String(nr), qt.Equals, "v1:v2") }) }) t.Run("repr-create", func(t *testing.T) { nr := fluent.MustBuild(nrp, func(na fluent.NodeAssembler) { na.AssignString("v1:v2") }) qt.Check(t, n, NodeContentEquals, nr) }) }) t.Run("first field empty string works", func(t *testing.T) { np := engine.PrototypeByName("ManystringStruct") nrp := engine.PrototypeByName("ManystringStruct.Repr") var n schema.TypedNode t.Run("typed-create", func(t *testing.T) { n = fluent.MustBuildMap(np, 2, func(ma fluent.MapAssembler) { ma.AssembleEntry("foo").AssignString("") ma.AssembleEntry("bar").AssignString("v2") }).(schema.TypedNode) t.Run("typed-read", func(t *testing.T) { qt.Assert(t, n.Kind(), qt.Equals, datamodel.Kind_Map) qt.Check(t, n.Length(), qt.Equals, int64(2)) qt.Check(t, must.String(must.Node(n.LookupByString("foo"))), qt.Equals, "") qt.Check(t, must.String(must.Node(n.LookupByString("bar"))), qt.Equals, "v2") }) t.Run("repr-read", func(t *testing.T) { nr := n.Representation() qt.Assert(t, nr.Kind(), qt.Equals, datamodel.Kind_String) qt.Check(t, must.String(nr), qt.Equals, ":v2") // Note the leading colon is still present. }) }) t.Run("repr-create", func(t *testing.T) { nr := fluent.MustBuild(nrp, func(na fluent.NodeAssembler) { na.AssignString(":v2") }) qt.Check(t, n, NodeContentEquals, nr) }) }) t.Run("nested stringjoin structs work", func(t *testing.T) { np := engine.PrototypeByName("Recurzorator") nrp := engine.PrototypeByName("Recurzorator.Repr") var n schema.TypedNode t.Run("typed-create", func(t *testing.T) { n = fluent.MustBuildMap(np, 3, func(ma fluent.MapAssembler) { ma.AssembleEntry("foo").AssignString("v1") ma.AssembleEntry("zap").CreateMap(2, func(ma fluent.MapAssembler) { ma.AssembleEntry("foo").AssignString("v2") ma.AssembleEntry("bar").AssignString("v3") }) ma.AssembleEntry("bar").AssignString("v4") }).(schema.TypedNode) t.Run("typed-read", func(t *testing.T) { qt.Assert(t, n.Kind(), qt.Equals, datamodel.Kind_Map) qt.Check(t, n.Length(), qt.Equals, int64(3)) qt.Check(t, must.String(must.Node(n.LookupByString("foo"))), qt.Equals, "v1") qt.Check(t, must.String(must.Node(n.LookupByString("bar"))), qt.Equals, "v4") n2 := must.Node(n.LookupByString("zap")) qt.Check(t, n2.Length(), qt.Equals, int64(2)) qt.Check(t, must.String(must.Node(n2.LookupByString("foo"))), qt.Equals, "v2") qt.Check(t, must.String(must.Node(n2.LookupByString("bar"))), qt.Equals, "v3") }) t.Run("repr-read", func(t *testing.T) { nr := n.Representation() qt.Assert(t, nr.Kind(), qt.Equals, datamodel.Kind_String) qt.Check(t, must.String(nr), qt.Equals, "v1-v2:v3-v4") }) }) t.Run("repr-create", func(t *testing.T) { nr := fluent.MustBuild(nrp, func(na fluent.NodeAssembler) { na.AssignString("v1-v2:v3-v4") }) qt.Check(t, n, NodeContentEquals, nr) }) }) } ================================================ FILE: node/tests/schemaStructReprTuple.go ================================================ package tests import ( "testing" qt "github.com/frankban/quicktest" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/fluent" "github.com/ipld/go-ipld-prime/must" "github.com/ipld/go-ipld-prime/schema" ) func SchemaTestStructReprTuple(t *testing.T, engine Engine) { ts := schema.TypeSystem{} ts.Init() ts.Accumulate(schema.SpawnString("String")) ts.Accumulate(schema.SpawnStruct("OneTuple", []schema.StructField{ schema.SpawnStructField("field", "String", false, false), }, schema.SpawnStructRepresentationTuple(), )) ts.Accumulate(schema.SpawnStruct("FourTuple", []schema.StructField{ schema.SpawnStructField("foo", "String", false, false), schema.SpawnStructField("bar", "String", false, true), schema.SpawnStructField("baz", "String", true, true), schema.SpawnStructField("qux", "String", true, false), }, schema.SpawnStructRepresentationTuple(), )) engine.Init(t, ts) t.Run("onetuple works", func(t *testing.T) { np := engine.PrototypeByName("OneTuple") nrp := engine.PrototypeByName("OneTuple.Repr") var n schema.TypedNode t.Run("typed-create", func(t *testing.T) { n = fluent.MustBuildMap(np, 1, func(ma fluent.MapAssembler) { ma.AssembleEntry("field").AssignString("valoo") }).(schema.TypedNode) t.Run("typed-read", func(t *testing.T) { qt.Assert(t, n.Kind(), qt.Equals, datamodel.Kind_Map) qt.Check(t, n.Length(), qt.Equals, int64(1)) qt.Check(t, must.String(must.Node(n.LookupByString("field"))), qt.Equals, "valoo") }) t.Run("repr-read", func(t *testing.T) { nr := n.Representation() qt.Assert(t, nr.Kind(), qt.Equals, datamodel.Kind_List) qt.Check(t, nr.Length(), qt.Equals, int64(1)) qt.Check(t, must.String(must.Node(nr.LookupByIndex(0))), qt.Equals, "valoo") }) }) t.Run("repr-create", func(t *testing.T) { nr := fluent.MustBuildList(nrp, 1, func(la fluent.ListAssembler) { la.AssembleValue().AssignString("valoo") }) qt.Check(t, n, NodeContentEquals, nr) }) }) t.Run("fourtuple works", func(t *testing.T) { np := engine.PrototypeByName("FourTuple") nrp := engine.PrototypeByName("FourTuple.Repr") var n schema.TypedNode t.Run("typed-create", func(t *testing.T) { n = fluent.MustBuildMap(np, 4, func(ma fluent.MapAssembler) { ma.AssembleEntry("foo").AssignString("0") ma.AssembleEntry("bar").AssignString("1") ma.AssembleEntry("baz").AssignString("2") ma.AssembleEntry("qux").AssignString("3") }).(schema.TypedNode) t.Run("typed-read", func(t *testing.T) { qt.Assert(t, n.Kind(), qt.Equals, datamodel.Kind_Map) qt.Check(t, n.Length(), qt.Equals, int64(4)) qt.Check(t, must.String(must.Node(n.LookupByString("foo"))), qt.Equals, "0") qt.Check(t, must.String(must.Node(n.LookupByString("bar"))), qt.Equals, "1") qt.Check(t, must.String(must.Node(n.LookupByString("baz"))), qt.Equals, "2") qt.Check(t, must.String(must.Node(n.LookupByString("qux"))), qt.Equals, "3") }) t.Run("repr-read", func(t *testing.T) { nr := n.Representation() qt.Assert(t, nr.Kind(), qt.Equals, datamodel.Kind_List) qt.Check(t, nr.Length(), qt.Equals, int64(4)) qt.Check(t, must.String(must.Node(nr.LookupByIndex(0))), qt.Equals, "0") qt.Check(t, must.String(must.Node(nr.LookupByIndex(1))), qt.Equals, "1") qt.Check(t, must.String(must.Node(nr.LookupByIndex(2))), qt.Equals, "2") qt.Check(t, must.String(must.Node(nr.LookupByIndex(3))), qt.Equals, "3") }) }) t.Run("repr-create", func(t *testing.T) { nr := fluent.MustBuildList(nrp, 4, func(la fluent.ListAssembler) { la.AssembleValue().AssignString("0") la.AssembleValue().AssignString("1") la.AssembleValue().AssignString("2") la.AssembleValue().AssignString("3") }) qt.Check(t, n, NodeContentEquals, nr) }) }) t.Run("fourtuple with absents", func(t *testing.T) { np := engine.PrototypeByName("FourTuple") nrp := engine.PrototypeByName("FourTuple.Repr") var n schema.TypedNode t.Run("typed-create", func(t *testing.T) { n = fluent.MustBuildMap(np, 2, func(ma fluent.MapAssembler) { ma.AssembleEntry("foo").AssignString("0") ma.AssembleEntry("bar").AssignNull() }).(schema.TypedNode) t.Run("typed-read", func(t *testing.T) { qt.Assert(t, n.Kind(), qt.Equals, datamodel.Kind_Map) qt.Check(t, n.Length(), qt.Equals, int64(4)) qt.Check(t, must.String(must.Node(n.LookupByString("foo"))), qt.Equals, "0") qt.Check(t, must.Node(n.LookupByString("bar")), qt.Equals, datamodel.Null) qt.Check(t, must.Node(n.LookupByString("baz")), qt.Equals, datamodel.Absent) qt.Check(t, must.Node(n.LookupByString("qux")), qt.Equals, datamodel.Absent) }) t.Run("repr-read", func(t *testing.T) { nr := n.Representation() qt.Assert(t, nr.Kind(), qt.Equals, datamodel.Kind_List) qt.Check(t, nr.Length(), qt.Equals, int64(2)) qt.Check(t, must.String(must.Node(nr.LookupByIndex(0))), qt.Equals, "0") qt.Check(t, must.Node(nr.LookupByIndex(1)), qt.Equals, datamodel.Null) }) }) t.Run("repr-create", func(t *testing.T) { nr := fluent.MustBuildList(nrp, 4, func(la fluent.ListAssembler) { la.AssembleValue().AssignString("0") la.AssembleValue().AssignNull() }) qt.Check(t, n, NodeContentEquals, nr) }) }) } ================================================ FILE: node/tests/schemaStructsContainingMaybe.go ================================================ package tests import ( "testing" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/schema" ) // TestStructsContainingMaybe checks all the variations of "nullable" and "optional" on struct fields. // It does this twice: once for the child maybes being implemented with pointers, // and once with maybes implemented as embeds. // The child values are scalars. // // Both type-level generic build and access as well as representation build and access are exercised; // the representation used is map (the native representation for structs). func SchemaTestStructsContainingMaybe(t *testing.T, engine Engine) { // Type declarations. // The tests here will all be targetted against this "Stroct" type. ts := schema.TypeSystem{} ts.Init() ts.Accumulate(schema.SpawnString("String")) ts.Accumulate(schema.SpawnStruct("Stroct", []schema.StructField{ // Every field in this struct (including their order) is exercising an interesting case... schema.SpawnStructField("f1", "String", false, false), // plain field. schema.SpawnStructField("f2", "String", true, false), // optional; later we have more than one optional field, nonsequentially. schema.SpawnStructField("f3", "String", false, true), // nullable; but required. schema.SpawnStructField("f4", "String", true, true), // optional and nullable; trailing optional. schema.SpawnStructField("f5", "String", true, false), // optional; and the second one in a row, trailing. }, schema.SpawnStructRepresentationMap(map[string]string{ "f1": "r1", "f2": "r2", "f3": "r3", "f4": "r4", }), )) engine.Init(t, ts) // There's a lot of cases to cover so a shorthand code labels each case for clarity: // - 'v' -- value in that entry // - 'n' -- null in that entry // - 'z' -- absent entry // There's also a semantic description of the main detail being probed suffixed to the shortcode. specs := []testcase{ { name: "vvvvv-AllFieldsSet", typeJson: `{"f1":"a","f2":"b","f3":"c","f4":"d","f5":"e"}`, reprJson: `{"f5":"e","r1":"a","r2":"b","r3":"c","r4":"d"}`, typePoints: []testcasePoint{ {"", datamodel.Kind_Map}, {"f1", "a"}, {"f2", "b"}, {"f3", "c"}, {"f4", "d"}, {"f5", "e"}, }, reprPoints: []testcasePoint{ {"", datamodel.Kind_Map}, {"r1", "a"}, {"r2", "b"}, {"r3", "c"}, {"r4", "d"}, {"f5", "e"}, }, }, { name: "vvnnv-Nulls", typeJson: `{"f1":"a","f2":"b","f3":null,"f4":null,"f5":"e"}`, reprJson: `{"f5":"e","r1":"a","r2":"b","r3":null,"r4":null}`, typePoints: []testcasePoint{ {"", datamodel.Kind_Map}, {"f1", "a"}, {"f2", "b"}, {"f3", datamodel.Null}, {"f4", datamodel.Null}, {"f5", "e"}, }, reprPoints: []testcasePoint{ {"", datamodel.Kind_Map}, {"r1", "a"}, {"r2", "b"}, {"r3", datamodel.Null}, {"r4", datamodel.Null}, {"f5", "e"}, }, }, { name: "vzvzv-AbsentOptionals", typeJson: `{"f1":"a","f3":"c","f5":"e"}`, reprJson: `{"f5":"e","r1":"a","r3":"c"}`, typePoints: []testcasePoint{ {"", datamodel.Kind_Map}, {"f1", "a"}, {"f2", datamodel.Absent}, {"f3", "c"}, {"f4", datamodel.Absent}, {"f5", "e"}, }, reprPoints: []testcasePoint{ {"", datamodel.Kind_Map}, {"r1", "a"}, //{"r2", datamodel.ErrNotExists{}}, // TODO: need better error typing from traversal package. {"r3", "c"}, //{"r4", datamodel.ErrNotExists{}}, // TODO: need better error typing from traversal package. {"f5", "e"}, }, typeItr: []entry{ {"f1", "a"}, {"f2", datamodel.Absent}, {"f3", "c"}, {"f4", datamodel.Absent}, {"f5", "e"}, }, }, { name: "vvnzz-AbsentTrailingOptionals", typeJson: `{"f1":"a","f2":"b","f3":null}`, reprJson: `{"r1":"a","r2":"b","r3":null}`, typePoints: []testcasePoint{ {"", datamodel.Kind_Map}, {"f1", "a"}, {"f2", "b"}, {"f3", datamodel.Null}, {"f4", datamodel.Absent}, {"f5", datamodel.Absent}, }, reprPoints: []testcasePoint{ {"", datamodel.Kind_Map}, {"r1", "a"}, {"r2", "b"}, {"r3", datamodel.Null}, //{"r4", datamodel.ErrNotExists{}}, // TODO: need better error typing from traversal package. //{"f5", datamodel.ErrNotExists{}}, // TODO: need better error typing from traversal package. }, typeItr: []entry{ {"f1", "a"}, {"f2", "b"}, {"f3", datamodel.Null}, {"f4", datamodel.Absent}, {"f5", datamodel.Absent}, }, }, } for _, tcase := range specs { tcase.Test(t, engine.PrototypeByName("Stroct"), engine.PrototypeByName("Stroct.Repr")) } } ================================================ FILE: node/tests/schemaUnions.go ================================================ package tests import ( "testing" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/schema" ) func SchemaTestUnionKeyed(t *testing.T, engine Engine) { ts := schema.TypeSystem{} ts.Init() ts.Accumulate(schema.SpawnString("String")) ts.Accumulate(schema.SpawnString("Strung")) ts.Accumulate(schema.SpawnUnion("StrStr", []schema.TypeName{ "String", "Strung", }, schema.SpawnUnionRepresentationKeyed(map[string]schema.TypeName{ "a": "String", "b": "Strung", }), )) engine.Init(t, ts) specs := []testcase{ { name: "InhabitantA", typeJson: `{"String":"whee"}`, reprJson: `{"a":"whee"}`, typePoints: []testcasePoint{ {"", datamodel.Kind_Map}, {"String", "whee"}, //{"Strung", datamodel.ErrNotExists{}}, // TODO: need better error typing from traversal package. }, reprPoints: []testcasePoint{ {"", datamodel.Kind_Map}, {"a", "whee"}, //{"b", datamodel.ErrNotExists{}}, // TODO: need better error typing from traversal package. }, }, { name: "InhabitantB", typeJson: `{"Strung":"whee"}`, reprJson: `{"b":"whee"}`, typePoints: []testcasePoint{ {"", datamodel.Kind_Map}, //{"String", datamodel.ErrNotExists{}}, // TODO: need better error typing from traversal package. {"Strung", "whee"}, }, reprPoints: []testcasePoint{ {"", datamodel.Kind_Map}, //{"a", datamodel.ErrNotExists{}}, // TODO: need better error typing from traversal package. {"b", "whee"}, }, }, } np := engine.PrototypeByName("StrStr") nrp := engine.PrototypeByName("StrStr.Repr") for _, tcase := range specs { tcase.Test(t, np, nrp) } } // Test keyed unions again, but this time with more complex types as children. // // The previous tests used scalar types as the children; this exercises most things, // but also has a couple (extremely non-obvious) simplifications: // namely, because the default representation for strings are "natural" representations, // the ReprAssemblers are actually aliases of the type-level Assemblers! // Aaaand that makes a few things "work" by coincidence that wouldn't otherwise fly. func SchemaTestUnionKeyedComplexChildren(t *testing.T, engine Engine) { ts := schema.TypeSystem{} ts.Init() ts.Accumulate(schema.SpawnString("String")) ts.Accumulate(schema.SpawnStruct("SmolStruct", []schema.StructField{ schema.SpawnStructField("s", "String", false, false), }, schema.SpawnStructRepresentationMap(map[string]string{ "s": "q", }), )) ts.Accumulate(schema.SpawnUnion("WheeUnion", []schema.TypeName{ "String", "SmolStruct", }, schema.SpawnUnionRepresentationKeyed(map[string]schema.TypeName{ "a": "String", "b": "SmolStruct", }), )) engine.Init(t, ts) specs := []testcase{ { name: "InhabitantA", typeJson: `{"String":"whee"}`, reprJson: `{"a":"whee"}`, typePoints: []testcasePoint{ {"", datamodel.Kind_Map}, {"String", "whee"}, //{"SmolStruct", datamodel.ErrNotExists{}}, // TODO: need better error typing from traversal package. }, reprPoints: []testcasePoint{ {"", datamodel.Kind_Map}, {"a", "whee"}, //{"b", datamodel.ErrNotExists{}}, // TODO: need better error typing from traversal package. }, }, { name: "InhabitantB", typeJson: `{"SmolStruct":{"s":"whee"}}`, reprJson: `{"b":{"q":"whee"}}`, typePoints: []testcasePoint{ {"", datamodel.Kind_Map}, //{"String", datamodel.ErrNotExists{}}, // TODO: need better error typing from traversal package. {"SmolStruct", datamodel.Kind_Map}, {"SmolStruct/s", "whee"}, }, reprPoints: []testcasePoint{ {"", datamodel.Kind_Map}, //{"a", datamodel.ErrNotExists{}}, // TODO: need better error typing from traversal package. {"b", datamodel.Kind_Map}, {"b/q", "whee"}, }, }, } np := engine.PrototypeByName("WheeUnion") nrp := engine.PrototypeByName("WheeUnion.Repr") for _, tcase := range specs { tcase.Test(t, np, nrp) } } // TestUnionKeyedReset puts a union inside a list, so that we can use the list's reuse of assembler as a test of the assembler's reset feature. // The value inside the union is also more complex than a scalar value so that we test resetting gets passed down, too. func SchemaTestUnionKeyedReset(t *testing.T, engine Engine) { ts := schema.TypeSystem{} ts.Init() ts.Accumulate(schema.SpawnString("String")) ts.Accumulate(schema.SpawnStruct("SmolStruct", []schema.StructField{ schema.SpawnStructField("s", "String", false, false), }, schema.SpawnStructRepresentationMap(map[string]string{ "s": "q", }), )) ts.Accumulate(schema.SpawnUnion("WheeUnion", []schema.TypeName{ "String", "SmolStruct", }, schema.SpawnUnionRepresentationKeyed(map[string]schema.TypeName{ "a": "String", "b": "SmolStruct", }), )) ts.Accumulate(schema.SpawnList("OuterList", "WheeUnion", false, )) engine.Init(t, ts) specs := []testcase{ { typeJson: `[{"SmolStruct":{"s":"one"}}, {"SmolStruct":{"s":"two"}}, {"String":"three"}]`, reprJson: `[{"b":{"q":"one"}}, {"b":{"q":"two"}}, {"a":"three"}]`, typePoints: []testcasePoint{ {"0/SmolStruct/s", "one"}, {"1/SmolStruct/s", "two"}, {"2/String", "three"}, }, reprPoints: []testcasePoint{ {"0/b/q", "one"}, {"1/b/q", "two"}, {"2/a", "three"}, }, }, } np := engine.PrototypeByName("OuterList") nrp := engine.PrototypeByName("OuterList.Repr") for _, tcase := range specs { tcase.Test(t, np, nrp) } } ================================================ FILE: node/tests/schemaUnionsKinded.go ================================================ package tests import ( "testing" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/schema" ) func SchemaTestUnionKinded(t *testing.T, engine Engine) { ts := schema.TypeSystem{} ts.Init() ts.Accumulate(schema.SpawnString("String")) ts.Accumulate(schema.SpawnStruct("SmolStruct", []schema.StructField{ schema.SpawnStructField("s", "String", false, false), }, schema.SpawnStructRepresentationMap(map[string]string{ "s": "q", }), )) ts.Accumulate(schema.SpawnUnion("WheeUnion", []schema.TypeName{ "String", "SmolStruct", }, schema.SpawnUnionRepresentationKinded(map[datamodel.Kind]schema.TypeName{ datamodel.Kind_String: "String", datamodel.Kind_Map: "SmolStruct", }), )) engine.Init(t, ts) // These are the same *type-level* as in TestUnionKeyedComplexChildren, // but (of course) have very different representations. specs := []testcase{ { name: "InhabitantA", typeJson: `{"String":"whee"}`, reprJson: `"whee"`, typePoints: []testcasePoint{ {"", datamodel.Kind_Map}, {"String", "whee"}, //{"SmolStruct", datamodel.ErrNotExists{}}, // TODO: need better error typing from traversal package. }, reprPoints: []testcasePoint{ {"", datamodel.Kind_String}, {"", "whee"}, }, }, { name: "InhabitantB", typeJson: `{"SmolStruct":{"s":"whee"}}`, reprJson: `{"q":"whee"}`, typePoints: []testcasePoint{ {"", datamodel.Kind_Map}, //{"String", datamodel.ErrNotExists{}}, // TODO: need better error typing from traversal package. {"SmolStruct", datamodel.Kind_Map}, {"SmolStruct/s", "whee"}, }, reprPoints: []testcasePoint{ {"", datamodel.Kind_Map}, {"q", "whee"}, }, }, } np := engine.PrototypeByName("WheeUnion") nrp := engine.PrototypeByName("WheeUnion.Repr") for _, tcase := range specs { tcase.Test(t, np, nrp) } } ================================================ FILE: node/tests/schemaUnionsStringprefix.go ================================================ package tests import ( "testing" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/schema" ) func SchemaTestUnionStringprefix(t *testing.T, engine Engine) { ts := schema.TypeSystem{} ts.Init() ts.Accumulate(schema.SpawnString("String")) ts.Accumulate(schema.SpawnStruct("SmolStruct", []schema.StructField{ schema.SpawnStructField("a", "String", false, false), schema.SpawnStructField("b", "String", false, false), }, schema.SpawnStructRepresentationStringjoin(":"), )) ts.Accumulate(schema.SpawnUnion("WheeUnion", []schema.TypeName{ "String", "SmolStruct", }, schema.SpawnUnionRepresentationStringprefix( ":", map[string]schema.TypeName{ "simple": "String", "complex": "SmolStruct", }, ), )) engine.Init(t, ts) // These are the same *type-level* as in TestUnionKeyedComplexChildren, // but (of course) have very different representations. specs := []testcase{ { name: "InhabitantA", typeJson: `{"String":"whee"}`, reprJson: `"simple:whee"`, typePoints: []testcasePoint{ {"", datamodel.Kind_Map}, {"String", "whee"}, //{"SmolStruct", datamodel.ErrNotExists{}}, // TODO: need better error typing from traversal package. }, reprPoints: []testcasePoint{ {"", datamodel.Kind_String}, {"", "simple:whee"}, }, }, { name: "InhabitantB", typeJson: `{"SmolStruct":{"a":"whee","b":"woo"}}`, reprJson: `"complex:whee:woo"`, typePoints: []testcasePoint{ {"", datamodel.Kind_Map}, //{"String", datamodel.ErrNotExists{}}, // TODO: need better error typing from traversal package. {"SmolStruct", datamodel.Kind_Map}, {"SmolStruct/a", "whee"}, {"SmolStruct/b", "woo"}, }, reprPoints: []testcasePoint{ {"", datamodel.Kind_String}, {"", "complex:whee:woo"}, }, }, } np := engine.PrototypeByName("WheeUnion") nrp := engine.PrototypeByName("WheeUnion.Repr") for _, tcase := range specs { tcase.Test(t, np, nrp) } } ================================================ FILE: node/tests/stringSpecs.go ================================================ package tests import ( "testing" qt "github.com/frankban/quicktest" "github.com/ipld/go-ipld-prime/datamodel" ) func SpecTestString(t *testing.T, np datamodel.NodePrototype) { t.Run("string node", func(t *testing.T) { nb := np.NewBuilder() err := nb.AssignString("asdf") qt.Check(t, err, qt.IsNil) n := nb.Build() qt.Check(t, n.Kind(), qt.Equals, datamodel.Kind_String) qt.Check(t, n.IsNull(), qt.IsFalse) x, err := n.AsString() qt.Check(t, err, qt.IsNil) qt.Check(t, x, qt.Equals, "asdf") }) } ================================================ FILE: node/tests/testEngine.go ================================================ package tests import ( "testing" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/schema" ) // Engine describes the interface that can be supplied to run tests on schemas. // // The PrototypeByName function can get its job done using only interface types // that we already know from outside any generated code, so you can write tests // that have no _compile time_ dependency on the generated code. This makes it // easier for IDEs and suchlike to help you write and check the test functions. // // Ask for prototypes using the type name alone (no package prefix); // their representation prototypes can be obtained by appending ".Repr". type Engine interface { Init(t *testing.T, ts schema.TypeSystem) PrototypeByName(name string) datamodel.NodePrototype } ================================================ FILE: node/tests/testcase.go ================================================ package tests import ( "bytes" "errors" "fmt" "strings" "testing" qt "github.com/frankban/quicktest" "github.com/polydawn/refmt/json" "github.com/polydawn/refmt/shared" "github.com/ipld/go-ipld-prime/codec" "github.com/ipld/go-ipld-prime/codec/dagjson" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/schema" "github.com/ipld/go-ipld-prime/traversal" ) // This file introduces a testcase struct and a bunch of functions around it. // This structure can be used to specify many test scenarios easily, using json as a shorthand for the fixtures. // Not everything can be tested this way (in particular, there's some fun details around maps with complex keys, and structs with absent fields), but it covers a lot. /* testcase contains data for directing a sizable number of tests against a NodePrototype (or more specifically, a pair of them -- one for the type-level node, one for the representation), all of which are applied by calling the testcase.Test method: - Creation of values using the type-level builder is tested. - This is done using a json input as a convenient shorthand. - n.b. this is optional, because it won't work for maps with complex keys. - In things that behave as maps: this tests the AssembleEntry path (rather than AssembleKey+AssembleValue; this is the case because this is implemented using unmarshal codepaths). - If this is expected to fail, an expected error may be specified (which will also make all other tests after creation inapplicable to this testcase). - Creation of values using the repr-level builder is tested. - This is (again) done using a json input as a convenient shorthand. - At least *one* of this or the json for type-level must be present. If neither: the testcase spec is broken. - As for the type-level test: in things that behave as maps, this tests the AssembleEntry path. - If this is expected to fail, an expected error may be specified (which will also make all other tests after creation inapplicable to this testcase). - If both forms of creation were exercised: check that the result nodes are deep-equal. - A list of "point" observations may be provided, which can probe positions in the data tree for expected values (or just type kind, etc). - This tests that direct lookups work. (It doesn't test iterators; that'll come in another step, later.) - Pathing (a la traversal.Get) is used for this this, so it's ready to inspect deep structures. - The field for expected value is just `interface{}`; it handles nodes, some primitives, and will also allow asserting an error. - The node is *copied*, and deep-equal checked again. - The purpose of this is to exercise the AssembleKey+AssembleValue path (as opposed to AssembleEntry (which is already exercised by our creation tests, since they use unmarshal codepaths)). - Access of type-level data via iterators is tested in one of two ways: - A list of expected key+values expected of the iterator can be provided explicitly; - If an explicit list isn't provided, but type-level json is provided, the type-level data will be marshalled and compared to the json fixture. - Most things can use the json path -- those that can't (e.g. maps with complex keys; structs with absent values -- neither is marshallable) use the explicit key+value system instead. - Access of the representation-level data via interators is tested via marshalling, and asserting it against the json fixture data (if present). - There's no explicit key+value list alternative here -- it's not needed; there is no data that is unmarshallable, by design! This system should cover a lot of things, but doesn't cover everything. - Good coverage for "reset" pathways is reached somewhat indirectly... - Tests for recursive types containing nontrivial reset methods exercise both the child type's assembler reset method, and that the parent calls it correctly. - Maps with complex keys are tricky to handle, as already noted above. - But you should be able to do it, with some care. - This whole system depends on json parsers and serializers already working. - This is arguably an uncomfortably large and complex dependency for a test system. However, the json systems are tested by using basicnode; there's no cycle here. - "Unhappy paths" in creation are a bit tricky to test. - It can be done, but for map-like things, only for the AssembleEntry path. - PRs welcome if someone's got a clever idea for a good way to exercise AssembleKey+AssembleValue. (A variant of unmarshaller implementation? Would do it; just verbose.) - No support yet for checking properties like Length. - Future: we could add another type-hinted special case to the testcasePoint.expect for this, i suppose. */ type testcase struct { name string // name for the testcase. typeJson string // json that will be fed to unmarshal together with a type-level assembler. marshal output will also be checked for equality. may be absent. reprJson string // json that will be fed to unmarshal together with a representational assembler. marshal output will also be checked for equality. expectUnmarshalFail error // if present, this error will be expected from the unmarshal process (and implicitly, marshal tests will not be applicable for this testcase). typePoints []testcasePoint // inspections that will be made by traversing the type-level nodes. reprPoints []testcasePoint // inspections that will be made by traversing the representation nodes. typeItr []entry // if set, the type will be iterated in this way. The remarshalling and checking against typeJson will not be tested. This is used to probe for correct iteration over Absent values in structs (which needs special handling, because they are unserializable). // there's really no need for an 'expectFail' that applies to marshal, because it shouldn't be possible to create data that's unmarshallable! (excepting data which is not marshallable by some *codec* due to incompleteness of that codec. But that's not what we're testing, here.) // there's no need for a reprItr because the marshalling to reprJson always covers that; unlike with the type level, neither absents nor complex keys can throw a wrench in serialization, so it's always available to us to exercise the iteration code. } type testcasePoint struct { path string expect interface{} // if primitive: we'll AsFoo and assert equal on that; if an error, we'll expect an error and compare error types; if a kind, we'll check that the thing reached simply has that kind. } type entry struct { key interface{} // (mostly string. not yet defined how this will handle maps with complex keys.) value interface{} // same rules as testcasePoint.expect } func (tcase testcase) Test(t *testing.T, np, npr datamodel.NodePrototype) { t.Run(tcase.name, func(t *testing.T) { // We'll produce either one or two nodes, depending on the fixture; if two, we'll be expecting them to be equal. var n, n2 datamodel.Node // Attempt to produce a node by using unmarshal on type-level fixture data and the type-level NodePrototype. // This exercises creating a value using the AssembleEntry path (but note, not AssembleKey+AssembleValue path). // This test section is optional because we can't use it for some types (namely, maps with complex keys -- which simply need custom tests). if tcase.typeJson != "" { t.Run("typed-create", func(t *testing.T) { n = testUnmarshal(t, np, tcase.typeJson, tcase.expectUnmarshalFail) }) } // Attempt to produce a node by using unmarshal on repr-level fixture data and the repr-level NodePrototype. // This exercises creating a value using the AssembleEntry path (but note, not AssembleKey+AssembleValue path). // This test section is optional simply because it's nice to be able to omit it when writing a new system and not wanting to test representation yet. if tcase.reprJson != "" { t.Run("repr-create", func(t *testing.T) { n3 := testUnmarshal(t, npr, tcase.reprJson, tcase.expectUnmarshalFail) if n == nil { n = n3 } else { n2 = n3 } }) } // If unmarshalling was expected to fail, the rest of the tests are inapplicable. if tcase.expectUnmarshalFail != nil { return } // Check the nodes are equal, if there's two of them. (Or holler, if none.) if n == nil { t.Fatalf("invalid fixture: need one of either typeJson or reprJson provided") } if n2 != nil { t.Run("type-create and repr-create match", func(t *testing.T) { qt.Check(t, n, NodeContentEquals, n2) }) } // Perform all the point inspections on the type-level node. if tcase.typePoints != nil { t.Run("type-level inspection", func(t *testing.T) { for _, point := range tcase.typePoints { wishPoint(t, n, point) } }) } // Perform all the point inspections on the repr-level node. if tcase.reprPoints != nil { t.Run("repr-level inspection", func(t *testing.T) { for _, point := range tcase.reprPoints { wishPoint(t, n.(schema.TypedNode).Representation(), point) } }) } // Serialize the type-level node, and check that we get the original json again. // This exercises iterators on the type-level node. // OR, if typeItr is present, do that instead (this is necessary when handling maps with complex keys or handling structs with absent values, since both of those are unserializable). if tcase.typeItr != nil { // This can unconditionally assume we're going to handle maps, // because the only kind of thing that needs this style of testing are some instances of maps and some instances of structs. itr := n.MapIterator() for _, entry := range tcase.typeItr { qt.Check(t, itr.Done(), qt.IsFalse) k, v, err := itr.Next() qt.Check(t, k, closeEnough, entry.key) qt.Check(t, v, closeEnough, entry.value) qt.Check(t, err, qt.IsNil) } qt.Check(t, itr.Done(), qt.IsTrue) k, v, err := itr.Next() qt.Check(t, k, qt.IsNil) qt.Check(t, v, qt.IsNil) qt.Check(t, err, qt.Equals, datamodel.ErrIteratorOverread{}) } else if tcase.typeJson != "" { t.Run("type-marshal", func(t *testing.T) { testMarshal(t, n, tcase.typeJson) }) } // Serialize the repr-level node, and check that we get the original json again. // This exercises iterators on the repr-level node. if tcase.reprJson != "" { t.Run("repr-marshal", func(t *testing.T) { testMarshal(t, n.(schema.TypedNode).Representation(), tcase.reprJson) }) } // Copy the node. If it's a map-like. // This exercises the AssembleKey+AssembleValue path for maps (or things that act as maps, such as structs and unions), // as opposed to the AssembleEntry path (which is what was exercised by the creation via unmarshal). // Assumes that the iterators are working correctly. if n.Kind() == datamodel.Kind_Map { t.Run("type-create with AK+AV", func(t *testing.T) { n3, err := shallowCopyMap(np, n) qt.Check(t, err, qt.IsNil) qt.Check(t, n, NodeContentEquals, n3) }) } // Copy the node, now at repr level. Again, this is for exercising AssembleKey+AssembleValue paths. // Assumes that the iterators are working correctly. if n.(schema.TypedNode).Representation().Kind() == datamodel.Kind_Map { t.Run("repr-create with AK+AV", func(t *testing.T) { n3, err := shallowCopyMap(npr, n.(schema.TypedNode).Representation()) qt.Check(t, err, qt.IsNil) qt.Check(t, n3, NodeContentEquals, n) }) } }) } func shallowCopyMap(np datamodel.NodePrototype, n datamodel.Node) (datamodel.Node, error) { nb := np.NewBuilder() ma, err := nb.BeginMap(n.Length()) if err != nil { return nil, err } for itr := n.MapIterator(); !itr.Done(); { k, v, err := itr.Next() if err != nil { return nil, err } if v.IsAbsent() { continue } if err := ma.AssembleKey().AssignNode(k); err != nil { return nil, err } if err := ma.AssembleValue().AssignNode(v); err != nil { return nil, err } } if err := ma.Finish(); err != nil { return nil, err } return nb.Build(), nil } func testUnmarshal(t *testing.T, np datamodel.NodePrototype, data string, expectFail error) datamodel.Node { t.Helper() nb := np.NewBuilder() err := dagjson.Decode(nb, strings.NewReader(data)) switch { case expectFail == nil && err != nil: t.Fatalf("fixture parse failed: %s", err) case expectFail == nil && err == nil: // carry on case expectFail != nil && err != nil: qt.Check(t, err, qt.ErrorAs, expectFail) case expectFail != nil && err == nil: t.Errorf("expected creation to fail with a %T error, but got no error", expectFail) } return nb.Build() } func testMarshal(t *testing.T, n datamodel.Node, data string) { t.Helper() // We'll marshal with "pretty" linebreaks and indents (and re-format the fixture to the same) for better diffing. prettyprint := json.EncodeOptions{Line: []byte{'\n'}, Indent: []byte{'\t'}} var buf bytes.Buffer err := dagjson.Marshal(n, json.NewEncoder(&buf, prettyprint), dagjson.EncodeOptions{ EncodeLinks: true, EncodeBytes: true, MapSortMode: codec.MapSortMode_Lexical, }) if err != nil { t.Errorf("marshal failed: %s", err) } qt.Check(t, buf.String(), qt.Equals, reformat(data, prettyprint)) } func wishPoint(t *testing.T, n datamodel.Node, point testcasePoint) { t.Helper() reached, err := traversal.Get(n, datamodel.ParsePath(point.path)) switch point.expect.(type) { case error: qt.Check(t, err, qt.ErrorAs, point.expect) qt.Check(t, err, qt.Equals, point.expect) default: qt.Check(t, err, qt.IsNil) if reached == nil { return } qt.Check(t, reached, closeEnough, point.expect) } } // closeEnough conforms to quicktest.Checker (so we can use it in quicktest invocations), // and lets Nodes be compared to primitives in convenient ways. // // If the expected value is a primitive string, it'll AsStrong on the Node; etc. // // Using a datamodel.Kind value is also possible, which will just check the kind and not the value contents. // // If a datamodel.Node is the expected value, a full deep qt.Equals is used as normal. var closeEnough = &closeEnoughChecker{} var _ qt.Checker = (*closeEnoughChecker)(nil) type closeEnoughChecker struct{} func (c *closeEnoughChecker) ArgNames() []string { return []string{"got", "want"} } func (c *closeEnoughChecker) Check(actual interface{}, args []interface{}, note func(key string, value interface{})) (err error) { expected := args[0] if expected == nil { return qt.IsNil.Check(actual, args, note) } a, ok := actual.(datamodel.Node) if !ok { return errors.New("this checker only supports checking datamodel.Node values") } switch expected.(type) { case datamodel.Kind: return qt.Equals.Check(a.Kind(), args, note) case string: if a.Kind() != datamodel.Kind_String { return fmt.Errorf("expected something with kind string, got kind %s", a.Kind()) } x, _ := a.AsString() return qt.Equals.Check(x, args, note) case int: if a.Kind() != datamodel.Kind_Int { return fmt.Errorf("expected something with kind int, got kind %s", a.Kind()) } x, _ := a.AsInt() return qt.Equals.Check(x, args, note) case datamodel.Node: return qt.Equals.Check(actual, args, note) default: return fmt.Errorf("this checker doesn't support an expected value of type %T", expected) } } func reformat(x string, opts json.EncodeOptions) string { var buf bytes.Buffer if err := (shared.TokenPump{ TokenSource: json.NewDecoder(strings.NewReader(x)), TokenSink: json.NewEncoder(&buf, opts), }).Run(); err != nil { panic(err) } return buf.String() } ================================================ FILE: node/tests/testutil.go ================================================ package tests import ( "github.com/ipld/go-ipld-prime/datamodel" ) // This file is full of helper functions. Most are moderately embarassing. // // We should probably turn half of this into Wish Checkers; // they'd probably be much less fragile and give better error messages that way. // On the other hand, the functions for condensing two-arg returns wouldn't go away anyway. // various benchmarks assign their final result here, // in order to defuse the possibility of their work being elided. var sink interface{} //lint:ignore U1000 used by benchmarks // purely to syntactically flip large inline closures so we can see the argument at the top rather than the bottom of the block. func withNode(n datamodel.Node, cb func(n datamodel.Node)) { cb(n) } ================================================ FILE: node/tests/traversalBenchmarks.go ================================================ package tests import ( "fmt" "testing" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/node/tests/corpus" "github.com/ipld/go-ipld-prime/traversal" ) func BenchmarkSpec_Walk_Map3StrInt(b *testing.B, np datamodel.NodePrototype) { node := mustNodeFromJsonString(np, corpus.Map3StrInt()) sel := mustSelectorFromJsonString(np, `{"a":{">":{".":{}}}}`) b.ResetTimer() var visitCountSanityCheck int for i := 0; i < b.N; i++ { visitCountSanityCheck = 0 traversal.WalkMatching(node, sel, func(tp traversal.Progress, n datamodel.Node) error { visitCountSanityCheck++ // this sanity check is sufficiently cheap to be worth it return nil // no need to do anything here; just care about exercising the walk internals. }) } if visitCountSanityCheck != 3 { b.Fatalf("visitCountSanityCheck should be 3, got %d", visitCountSanityCheck) } } func BenchmarkSpec_Walk_MapNStrMap3StrInt(b *testing.B, np datamodel.NodePrototype) { sel := mustSelectorFromJsonString(np, `{"a":{">":{"a":{">":{".":{}}}}}}`) for _, n := range []int{0, 1, 2, 4, 8, 16, 32} { b.Run(fmt.Sprintf("n=%d", n), func(b *testing.B) { node := mustNodeFromJsonString(np, corpus.MapNStrMap3StrInt(n)) b.ResetTimer() var visitCountSanityCheck int for i := 0; i < b.N; i++ { visitCountSanityCheck = 0 traversal.WalkMatching(node, sel, func(tp traversal.Progress, n datamodel.Node) error { visitCountSanityCheck++ // this sanity check is sufficiently cheap to be worth it return nil // no need to do anything here; just care about exercising the walk internals. }) } if visitCountSanityCheck != 3*n { b.Fatalf("visitCountSanityCheck should be %d, got %d", n*3, visitCountSanityCheck) } }) } } ================================================ FILE: node/tests/unmarshalBenchmarks.go ================================================ package tests import ( "bytes" "fmt" "strings" "testing" "github.com/ipld/go-ipld-prime/codec/json" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/node/tests/corpus" ) // All of the marshalling and unmarshalling benchmark specs use JSON. // This does mean we're measuring a bunch of stuff that has nothing to do // with the core operations of the Node/NodeBuilder interface. // We do this so that: // - we get a reasonable picture of how much time is spent in the IPLD Data Model // versus how much time is spent in the serialization efforts; // - we can make direct comparisons to the standard library json marshalling // and unmarshalling, thus having a back-of-the-envelope baseline to compare. func BenchmarkSpec_Unmarshal_Map3StrInt(b *testing.B, np datamodel.NodePrototype) { var err error for i := 0; i < b.N; i++ { nb := np.NewBuilder() err = json.Decode(nb, strings.NewReader(`{"whee":1,"woot":2,"waga":3}`)) sink = nb.Build() } if err != nil { panic(err) } } func BenchmarkSpec_Unmarshal_MapNStrMap3StrInt(b *testing.B, np datamodel.NodePrototype) { for _, n := range []int{0, 1, 2, 4, 8, 16, 32} { b.Run(fmt.Sprintf("n=%d", n), func(b *testing.B) { msg := corpus.MapNStrMap3StrInt(n) b.ResetTimer() var node datamodel.Node var err error nb := np.NewBuilder() for i := 0; i < b.N; i++ { err = json.Decode(nb, strings.NewReader(msg)) node = nb.Build() nb.Reset() } b.StopTimer() if err != nil { b.Fatalf("decode errored: %s", err) } var buf bytes.Buffer json.Encode(node, &buf) if buf.String() != msg { b.Fatalf("re-encode result didn't match corpus") } }) } } ================================================ FILE: node/tests/util.go ================================================ package tests import ( "strings" "github.com/ipld/go-ipld-prime/codec/json" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/must" "github.com/ipld/go-ipld-prime/traversal/selector" ) func mustNodeFromJsonString(np datamodel.NodePrototype, str string) datamodel.Node { nb := np.NewBuilder() must.NotError(json.Decode(nb, strings.NewReader(str))) return nb.Build() } func mustSelectorFromJsonString(np datamodel.NodePrototype, str string) selector.Selector { // Needing an 'ns' parameter here is sort of off-topic, to be honest. // Someday the selector package will probably contain codegen'd nodes of its own schema, and we'll use those unconditionally. // For now... we'll just use whatever node you're already testing, because it oughta work // (and because it avoids hardcoding any other implementation that might cause import cycle funtimes.). seldefn := mustNodeFromJsonString(np, str) sel, err := selector.ParseSelector(seldefn) must.NotError(err) return sel } ================================================ FILE: operations.go ================================================ package ipld import ( "github.com/ipld/go-ipld-prime/datamodel" ) // DeepEqual reports whether x and y are "deeply equal" as IPLD nodes. // This is similar to reflect.DeepEqual, but based around the Node interface. // // This is exactly equivalent to the datamodel.DeepEqual function. func DeepEqual(x, y Node) bool { return datamodel.DeepEqual(x, y) } ================================================ FILE: printer/doc.go ================================================ /* Printer provides features for printing out IPLD nodes and their contained data in a human-readable diagnostic format. Outputs should look like... map{ string{"foo"}: string{"bar"} string{"zot"}: struct{ someFieldName: list{ 0: string{"this list is untyped"} 1: string{"and contains a mixture of kinds of values"} 2: int{400} 3: bool{true} } otherField: list{ 0: string{"mind you: 'ANamedListType' is the name of the *list type*."} 1: string{"it is not the name of the types of the value."} 2: string{"you'd have to look at the schema for that information."} 3: string{"or, of course, you can see it at the start of each of these entries, since they are also each annotated."} } moreField: list<[nullable String]>{ 0: string{"this is a typed list"} 1: string{"but anonymous (meaning you see the value type in the 'name' of it)"} 2: null } } string{"frog"}: map<{String:String}>{ string{"as you have probably imagined"}: string{"this is a typed (but anonymous type) map"} } string{"numbers"}: int{1} string{"binary"}: bytes{ABCDEF0123456789} string{"typed numbers"}: int{9000} string{"typed string"}: string{"okay, this one needed some marker prefixes."} string{"map with typed keys"}: map<{MyNamedTypeString:MyNamedTypeString}>{ string{"work just fine"}: string{"there's no ambiguity"} string{"you could elide key type info"}: string{"as long as its a string kind, anyway"} string{"but we don't by default"}: string{"explicit is good, especially in a debug tool!"} } string{"structs"}: struct{ foo: string{"do not need to have quoted field names"} bar: string{"because (unlike map keys) their character range is already restricted"} } string{"unit types"}: unit string{"notice unit types"}: string{"have no braces at all, because they have literally no further details. they're all type info."} string{"unions"}: union{string{ "that was wild, wasn't it. Check out these double closing braces, coming up, too! also the string got forced to a new line, even though it usually would've clung closer to its type and kind marker." }} string{"enums"}: enum{"inhabitant name"} string{"typed bools"}: bool{true} string{"map with struct keys"{: map<{FooBar:String}>{ struct{foo:"foo", bar:"bar"}: string{"that one probably surprised you, didn't it?"} struct{foo:"hmmm", bar:"maybe"}: string{"we might be able to get away without the kind+type marker, actually. but we need the one-liner struct content printing, at least, for sure."} } string{"map with really wicked keys"}: map<{WickedNestedUnion:String}>{ union{union{string{"wow"}}}: "yeah, that happens sometimes" } } The pattern is a preamble saying what kind the value is (and what type, if applicable), followed by the actual value content, in braces. For untyped nodes, this means `kindname{"value"}` (so: `string{"foo"}` and `int{12}` and `bool{true}` etc), or for typed nodes, we get `typekindname{"value"}`. In addition to the example above, you can check out the tests for a few more examples of how it looks. Some configuration options are available to elide some information. For example, some configuration can reduce the amount of annotational weight around strings (which is possible to do without getting completely vague because the quotation markings for strings already are syntactically distinctive). Not all things can be configured for elision, however. For example, for the various number kinds, the kind preambles are always required. (Number parsers are otherwise often an annoying lookahead problem.) Similarly, for bytes, the kind preamble is always required. (Among other things, the hexadecimal up until the first letter could be confused with an integer, if we didn't label both of them.) Anything that's typed also gets a preamble with the type and kind information, even if its kind is something we'd otherwise elide, like string. Notice that struct fields aren't quoted. (It's not necessary, because field names are already constrained.) But map keys are. (They need quoting because they can be any string.) Note that the output of printer is NOT INTENDED TO BE PARSABLE. It is NOT an IPLD codec! It is a diagnostic format only. Much of the information included (especially about schema type information) is _more_ information than the IPLD data model holds alone, so trying to re-parse the printer output would be a strange choice. The diagnostic format emitted by printer is not formally specified, and is not necessarily language-agnostic. It may not even remain stable across releases of this library. It is intended to be used for diagnostics only. */ package printer /* How to print ADLs is not yet clear. Perhaps something like `` will do; this would also stack reasonably clearly with types as ``; this style would have the downside of making ADLs look *very* different than other mere representation strategies, which may be totally reasonable or mildly questionable depending on how purist you feel about that. */ ================================================ FILE: printer/printer.go ================================================ package printer import ( "encoding/hex" "io" "os" "strconv" "strings" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/schema" ) // Print emits a textual description of the node tree straight to stdout. // All printer configuration will be the default; // links will be printed, and will not be traversed. func Print(n datamodel.Node) { Config{}.Print(n) } // Sprint returns a textual description of the node tree. // All printer configuration will be the default; // links will be printed, and will not be traversed. func Sprint(n datamodel.Node) string { return Config{}.Sprint(n) } // Fprint accepts an io.Writer to which a textual description of the node tree will be written. // All printer configuration will be the default; // links will be printed, and will not be traversed. func Fprint(w io.Writer, n datamodel.Node) { Config{}.Fprint(w, n) } // Print emits a textual description of the node tree straight to stdout. // The configuration structure this method is attached to can be used to specified details for how the printout will be formatted. func (cfg Config) Print(n datamodel.Node) { cfg.Fprint(os.Stdout, n) } // Sprint returns a textual description of the node tree. // The configuration structure this method is attached to can be used to specified details for how the printout will be formatted. func (cfg Config) Sprint(n datamodel.Node) string { var buf strings.Builder cfg.Fprint(&buf, n) return buf.String() } // Fprint accepts an io.Writer to which a textual description of the node tree will be written. // The configuration structure this method is attached to can be used to specified details for how the printout will be formatted. func (cfg Config) Fprint(w io.Writer, n datamodel.Node) { pr := printBuf{w, cfg} pr.Config.init() pr.doString(0, printState_normal, n) } type Config struct { // If true, long strings and long byte sequences will truncated, and will include ellipses instead. // // Not yet supported. Abbreviate bool // If set, the indentation to use. // If nil, it will be treated as a default "\t". Indentation []byte // Probably does exactly what you think it does. StartingIndent []byte // Set to true if you like verbosity, I guess. // If false, strings will only have kind+type markings if they're typed. // // Not yet supported. AlwaysMarkStrings bool // Set to true if you want type info to be skipped for any type that's in the Prelude // (e.g. instead of `string{` seeing only `string{` is preferred, etc). // // Not yet supported. ElidePreludeTypeInfo bool // Set to true if you want maps to use "complex"-style printouts: // meaning they will print their keys on separate lines than their values, // and keys may spread across multiple lines if appropriate. // // If not set, a heuristic will be used based on if the map is known to // have keys that are complex enough that rendering them as oneline seems likely to overload. // See Config.useCmplxKeys for exactly how that's determined. UseMapComplexStyleAlways bool // For maps to use "complex"-style printouts (or not) per type. // See docs on UseMapComplexStyleAlways for the overview of what "complex"-style means. UseMapComplexStyleOnType map[schema.TypeName]bool } func (cfg *Config) init() { if cfg.Indentation == nil { cfg.Indentation = []byte{'\t'} } } // oneline decides if a value should be flatted into printing on a single, // or if it's allowed to spread out over multiple lines. // Note that this will not be asked if something outside of a value has already declared it's // doing a oneline rendering; that railroads everything within it into that mode too. func (cfg Config) oneline(typ schema.Type, isInKey bool) bool { return isInKey // Future: this could become customizable, with some kind of Always|OnlyInKeys|Never option enum per type. } /* TODO: not implemented or used // useRepr decides if a value should be printed using its representation. // Sometimes configuring this to be true for structs or unions with stringy representations // will cause map printouts using them as keys to become drastically more readable // (if with some loss of informativeness, or at least loss of explicitness). func (cfg Config) useRepr(typ schema.Type, isInKey bool) bool { return false } */ // useCmplxKeys decides if a map should print itself using a multi-line and extra-indented style for keys. func (cfg Config) useCmplxKeys(mapn datamodel.Node) bool { if cfg.UseMapComplexStyleAlways { return true } tn, ok := mapn.(schema.TypedNode) if !ok { return false } tnt := tn.Type() if tnt == nil { return false } force, ok := cfg.UseMapComplexStyleOnType[tnt.Name()] if ok { return force } ti, ok := tnt.(*schema.TypeMap) if !ok { // Probably should never even have been asked, then? panic("how did you get here?") } return !cfg.oneline(ti.KeyType(), true) } // FUTURE: one could imagine putting an optional LinkSystem param into the Config, too, and some recursion control. // It's definitely going to be the default to do zero recursion across links, though, // as doing that requires creating graph visualizations, and that is both possible, yet to do well becomes rather nontrivial. // Also, often a single node's tree visualization has been enough to get started debugging whatever I need to debug so far. type printBuf struct { wr io.Writer Config } func (z *printBuf) writeString(s string) { z.wr.Write([]byte(s)) } func (z *printBuf) doIndent(indentLevel int) { z.wr.Write(z.Config.StartingIndent) for i := 0; i < indentLevel; i++ { z.wr.Write(z.Config.Indentation) } } const ( printState_normal uint8 = iota printState_isKey // may sometimes entersen or stringify things harder. printState_isValue // signals that we're continuing a line that started with a key (so, don't emit indent). printState_isCmplxKey // used to ask something to use multiline form, and an extra indent -- the opposite of what isKey does. printState_isCmplxValue // we're continuing a line (so don't emit indent), and we're stuck in complex mode (so keep telling your children to stay in this state too). ) func (z *printBuf) doString(indentLevel int, printState uint8, n datamodel.Node) { // First: indent. switch printState { case printState_normal, printState_isKey, printState_isCmplxKey: z.doIndent(indentLevel) } // Second: the typekind and type name; or, just the kind, if there's no type. // Note: this can be somewhat overbearing -- for example, typed strings are going to get called out as `string{"value"}`. // This is rather agonizingly verbose, but also accurate; I'm not sure if we'd want to elide information about typed-vs-untyped entirely. if tn, ok := n.(schema.TypedNode); ok { var tnk schema.TypeKind var tntName string // Defensively check for nil node type if tnt := tn.Type(); tnt == nil { tntName = "?!nil" tnk = schema.TypeKind_Invalid } else { tntName = tnt.Name() tnk = tnt.TypeKind() } z.writeString(tnk.String()) z.writeString("<") z.writeString(tntName) z.writeString(">") switch tnk { case schema.TypeKind_Invalid: z.writeString("{?!}") case schema.TypeKind_Map: // continue -- the data-model driven behavior is sufficient to handle the content. case schema.TypeKind_List: // continue -- the data-model driven behavior is sufficient to handle the content. case schema.TypeKind_Unit: return // that's it! there's no content data for a unit type. case schema.TypeKind_Bool: // continue -- the data-model driven behavior is sufficient to handle the content. case schema.TypeKind_Int: // continue -- the data-model driven behavior is sufficient to handle the content. case schema.TypeKind_Float: // continue -- the data-model driven behavior is sufficient to handle the content. case schema.TypeKind_String: // continue -- the data-model driven behavior is sufficient to handle the content. case schema.TypeKind_Bytes: // continue -- the data-model driven behavior is sufficient to handle the content. case schema.TypeKind_Link: // continue -- the data-model driven behavior is sufficient to handle the content. case schema.TypeKind_Struct: // Very similar to a map, but keys aren't quoted. // Also, because it's possible for structs to be keys in a map themselves, they potentially need oneline emission. // Or, to customize emission in another direction if being a key in a map that's printing in "complex" mode. // FUTURE: there should also probably be some way to configure instructions to use their representation form instead. oneline := printState == printState_isCmplxValue || printState != printState_isCmplxKey && z.Config.oneline(tn.Type(), printState == printState_isKey) deepen := 1 if printState == printState_isCmplxKey { deepen = 2 } childState := printState_isValue if oneline { childState = printState_isCmplxValue } z.writeString("{") if !oneline && n.Length() > 0 { z.writeString("\n") } for itr := n.MapIterator(); !itr.Done(); { k, v, _ := itr.Next() if !oneline { z.doIndent(indentLevel + deepen) } fn, _ := k.AsString() z.writeString(fn) z.writeString(": ") z.doString(indentLevel+deepen, childState, v) if oneline { if !itr.Done() { z.writeString(", ") } } else { z.writeString("\n") } } if !oneline { z.doIndent(indentLevel) } z.writeString("}") return case schema.TypeKind_Union: // There will only be one thing in it, but we still have to use an iterator // to figure out what that is if we're doing this generically. // We can ignore the key and just look at the value type again though (even though those are the same in practice). _, v, _ := n.MapIterator().Next() z.writeString("{") z.doString(indentLevel, printState_isValue, v) z.writeString("}") return case schema.TypeKind_Enum: panic("TODO") default: panic("unreachable") } } else { if n.IsAbsent() { z.writeString("absent") return } z.writeString(n.Kind().String()) } // Third: all the actual content. // FUTURE: this is probably gonna become... somewhat more conditional, and may end up being a sub-function to be reasonably wieldy. switch n.Kind() { case datamodel.Kind_Map: // Maps have to decide if they have complex keys and want to use an additionally-intended pattern to make that readable. // "Complex" here means roughly: if you try to cram them into one line, it doesn't look good. // This choice starts at the map but is mostly executed during the printing of the key: // the key will start itself at normal indentation, // but should then doubly indent all its nested values (assuming it has any). cmplxKeys := z.Config.useCmplxKeys(n) childKeyState := printState_isKey if cmplxKeys { childKeyState = printState_isCmplxKey } z.writeString("{") if n.Length() > 0 { z.writeString("\n") } else { z.writeString("}") return } for itr := n.MapIterator(); !itr.Done(); { k, v, err := itr.Next() if err != nil { z.doIndent(indentLevel + 1) z.writeString("!! map iteration step yielded error: ") z.writeString(err.Error()) z.writeString("\n") break } z.doString(indentLevel+1, childKeyState, k) z.writeString(": ") z.doString(indentLevel+1, printState_isValue, v) z.writeString("\n") } z.doIndent(indentLevel) z.writeString("}") case datamodel.Kind_List: z.writeString("{") if n.Length() > 0 { z.writeString("\n") } else { z.writeString("}") return } for itr := n.ListIterator(); !itr.Done(); { idx, v, err := itr.Next() if err != nil { z.doIndent(indentLevel + 1) z.writeString("!! list iteration step yielded error: ") z.writeString(err.Error()) z.writeString("\n") break } z.doIndent(indentLevel + 1) z.writeString(strconv.FormatInt(idx, 10)) z.writeString(": ") z.doString(indentLevel+1, printState_isValue, v) z.writeString("\n") } z.doIndent(indentLevel) z.writeString("}") case datamodel.Kind_Null: // nothing: we already wrote the word "null" when we wrote the kind info prefix. case datamodel.Kind_Bool: z.writeString("{") if b, _ := n.AsBool(); b { z.writeString("true") } else { z.writeString("false") } z.writeString("}") case datamodel.Kind_Int: x, _ := n.AsInt() z.writeString("{") z.writeString(strconv.FormatInt(x, 10)) z.writeString("}") case datamodel.Kind_Float: x, _ := n.AsFloat() z.writeString("{") z.writeString(strconv.FormatFloat(x, 'f', -1, 64)) z.writeString("}") case datamodel.Kind_String: x, _ := n.AsString() z.writeString("{") z.writeString(strconv.QuoteToGraphic(x)) z.writeString("}") case datamodel.Kind_Bytes: x, _ := n.AsBytes() z.writeString("{") dst := make([]byte, hex.EncodedLen(len(x))) hex.Encode(dst, x) z.writeString(string(dst)) z.writeString("}") case datamodel.Kind_Link: x, _ := n.AsLink() z.writeString("{") z.writeString(x.String()) z.writeString("}") } } ================================================ FILE: printer/printer_test.go ================================================ package printer import ( "testing" qt "github.com/frankban/quicktest" "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/fluent/qp" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/node/bindnode" "github.com/ipld/go-ipld-prime/schema" "github.com/ipld/go-ipld-prime/testutil" ) var testLink = func() datamodel.Link { someCid, _ := cid.Cast([]byte{1, 85, 0, 5, 0, 1, 2, 3, 4}) return cidlink.Link{Cid: someCid} }() func TestSimpleData(t *testing.T) { t.Run("nested-maps", func(t *testing.T) { n, _ := qp.BuildMap(basicnode.Prototype.Any, -1, func(ma datamodel.MapAssembler) { qp.MapEntry(ma, "some key", qp.String("some value")) qp.MapEntry(ma, "another key", qp.String("another value")) qp.MapEntry(ma, "nested map", qp.Map(2, func(ma datamodel.MapAssembler) { qp.MapEntry(ma, "deeper entries", qp.String("deeper values")) qp.MapEntry(ma, "more deeper entries", qp.String("more deeper values")) })) qp.MapEntry(ma, "nested list", qp.List(2, func(la datamodel.ListAssembler) { qp.ListEntry(la, qp.Int(1)) qp.ListEntry(la, qp.Int(2)) })) qp.MapEntry(ma, "list with float", qp.List(1, func(la datamodel.ListAssembler) { qp.ListEntry(la, qp.Float(3.4)) })) }) qt.Check(t, Sprint(n), qt.CmpEquals(), testutil.Dedent(` map{ string{"some key"}: string{"some value"} string{"another key"}: string{"another value"} string{"nested map"}: map{ string{"deeper entries"}: string{"deeper values"} string{"more deeper entries"}: string{"more deeper values"} } string{"nested list"}: list{ 0: int{1} 1: int{2} } string{"list with float"}: list{ 0: float{3.4} } }`, )) }) t.Run("map-with-link-and-bytes", func(t *testing.T) { n, _ := qp.BuildMap(basicnode.Prototype.Any, -1, func(ma datamodel.MapAssembler) { qp.MapEntry(ma, "some key", qp.Link(testLink)) qp.MapEntry(ma, "another key", qp.String("another value")) qp.MapEntry(ma, "nested map", qp.Map(2, func(ma datamodel.MapAssembler) { qp.MapEntry(ma, "deeper entries", qp.String("deeper values")) qp.MapEntry(ma, "more deeper entries", qp.Link(testLink)) qp.MapEntry(ma, "yet another deeper entries", qp.Bytes([]byte("fish"))) })) qp.MapEntry(ma, "nested list", qp.List(2, func(la datamodel.ListAssembler) { qp.ListEntry(la, qp.Bytes([]byte("ghoti"))) qp.ListEntry(la, qp.Int(1)) qp.ListEntry(la, qp.Link(testLink)) })) }) qt.Check(t, Sprint(n), qt.CmpEquals(), testutil.Dedent(` map{ string{"some key"}: link{bafkqabiaaebagba} string{"another key"}: string{"another value"} string{"nested map"}: map{ string{"deeper entries"}: string{"deeper values"} string{"more deeper entries"}: link{bafkqabiaaebagba} string{"yet another deeper entries"}: bytes{66697368} } string{"nested list"}: list{ 0: bytes{67686f7469} 1: int{1} 2: link{bafkqabiaaebagba} } }`, )) }) } func TestTypedData(t *testing.T) { t.Run("structs", func(t *testing.T) { type FooBar struct { Foo string Bar string Baz []byte Jazz datamodel.Link } ts := schema.MustTypeSystem( schema.SpawnString("String"), schema.SpawnBytes("Bytes"), schema.SpawnLink("Link"), schema.SpawnStruct("FooBar", []schema.StructField{ schema.SpawnStructField("foo", "String", false, false), schema.SpawnStructField("bar", "String", false, false), schema.SpawnStructField("baz", "Bytes", false, false), schema.SpawnStructField("jazz", "Link", false, false), }, nil), ) n := bindnode.Wrap(&FooBar{"x", "y", []byte("zed"), testLink}, ts.TypeByName("FooBar")) qt.Check(t, Sprint(n), qt.CmpEquals(), testutil.Dedent(` struct{ foo: string{"x"} bar: string{"y"} baz: bytes{7a6564} jazz: link{bafkqabiaaebagba} }`, )) }) t.Run("map-with-struct-keys", func(t *testing.T) { type FooBar struct { Foo string Bar string } type WowMap struct { Keys []FooBar Values map[FooBar]string } ts := schema.MustTypeSystem( schema.SpawnString("String"), schema.SpawnStruct("FooBar", []schema.StructField{ schema.SpawnStructField("foo", "String", false, false), schema.SpawnStructField("bar", "String", false, false), }, schema.SpawnStructRepresentationStringjoin(":")), schema.SpawnMap("WowMap", "FooBar", "String", false), ) n := bindnode.Wrap(&WowMap{ Keys: []FooBar{{"x", "y"}, {"z", "z"}}, Values: map[FooBar]string{ {"x", "y"}: "a", {"z", "z"}: "b", }, }, ts.TypeByName("WowMap")) qt.Check(t, Sprint(n), qt.CmpEquals(), testutil.Dedent(` map{ struct{foo: string{"x"}, bar: string{"y"}}: string{"a"} struct{foo: string{"z"}, bar: string{"z"}}: string{"b"} }`, )) }) t.Run("map-with-nested-struct-keys", func(t *testing.T) { type Baz struct { Baz string } type FooBar struct { Foo string Bar Baz Baz Baz } type WowMap struct { Keys []FooBar Values map[FooBar]Baz } ts := schema.MustTypeSystem( schema.SpawnString("String"), schema.SpawnStruct("FooBar", []schema.StructField{ schema.SpawnStructField("foo", "String", false, false), schema.SpawnStructField("bar", "Baz", false, false), schema.SpawnStructField("baz", "Baz", false, false), }, schema.SpawnStructRepresentationStringjoin(":")), schema.SpawnStruct("Baz", []schema.StructField{ schema.SpawnStructField("baz", "String", false, false), }, schema.SpawnStructRepresentationStringjoin(":")), schema.SpawnMap("WowMap", "FooBar", "Baz", false), ) n := bindnode.Wrap(&WowMap{ Keys: []FooBar{{"x", Baz{"y"}, Baz{"y"}}, {"z", Baz{"z"}, Baz{"z"}}}, Values: map[FooBar]Baz{ {"x", Baz{"y"}, Baz{"y"}}: {"a"}, {"z", Baz{"z"}, Baz{"z"}}: {"b"}, }, }, ts.TypeByName("WowMap")) t.Run("complex-keys-in-effect", func(t *testing.T) { cfg := Config{ UseMapComplexStyleAlways: true, } qt.Check(t, cfg.Sprint(n), qt.CmpEquals(), testutil.Dedent(` map{ struct{ foo: string{"x"} bar: struct{ baz: string{"y"} } baz: struct{ baz: string{"y"} } }: struct{ baz: string{"a"} } struct{ foo: string{"z"} bar: struct{ baz: string{"z"} } baz: struct{ baz: string{"z"} } }: struct{ baz: string{"b"} } }`, )) }) t.Run("complex-keys-in-disabled", func(t *testing.T) { cfg := Config{ UseMapComplexStyleOnType: map[schema.TypeName]bool{ "WowMap": false, }, } qt.Check(t, cfg.Sprint(n), qt.CmpEquals(), testutil.Dedent(` map{ struct{foo: string{"x"}, bar: struct{baz: string{"y"}}, baz: struct{baz: string{"y"}}}: struct{ baz: string{"a"} } struct{foo: string{"z"}, bar: struct{baz: string{"z"}}, baz: struct{baz: string{"z"}}}: struct{ baz: string{"b"} } }`, )) }) }) t.Run("invalid-nil-typed-node", func(t *testing.T) { qt.Check(t, Sprint(&nilTypedNode{datamodel.Kind_Invalid}), qt.CmpEquals(), "invalid{?!}") }) t.Run("invalid-nil-typed-node-with-map-kind", func(t *testing.T) { qt.Check(t, Sprint(&nilTypedNode{datamodel.Kind_Map}), qt.CmpEquals(), "invalid{?!}{}") }) } var _ schema.TypedNode = (*nilTypedNode)(nil) type nilTypedNode struct { kind datamodel.Kind } func (n *nilTypedNode) Kind() datamodel.Kind { return n.kind } func (n nilTypedNode) LookupByString(key string) (datamodel.Node, error) { return nil, nil } func (n nilTypedNode) LookupByNode(key datamodel.Node) (datamodel.Node, error) { return nil, nil } func (n nilTypedNode) LookupByIndex(idx int64) (datamodel.Node, error) { return nil, nil } func (n nilTypedNode) LookupBySegment(seg datamodel.PathSegment) (datamodel.Node, error) { return nil, nil } func (n nilTypedNode) MapIterator() datamodel.MapIterator { return nil } func (n nilTypedNode) ListIterator() datamodel.ListIterator { return nil } func (n nilTypedNode) Length() int64 { return 0 } func (n nilTypedNode) IsAbsent() bool { return false } func (n nilTypedNode) IsNull() bool { return false } func (n nilTypedNode) AsBool() (bool, error) { panic("nil-typed-node") } func (n nilTypedNode) AsInt() (int64, error) { panic("nil-typed-node") } func (n nilTypedNode) AsFloat() (float64, error) { panic("nil-typed-node") } func (n nilTypedNode) AsString() (string, error) { panic("nil-typed-node") } func (n nilTypedNode) AsBytes() ([]byte, error) { panic("nil-typed-node") } func (n nilTypedNode) AsLink() (datamodel.Link, error) { panic("nil-typed-node") } func (n nilTypedNode) Prototype() datamodel.NodePrototype { return nil } func (n nilTypedNode) Type() schema.Type { return nil } func (n nilTypedNode) Representation() datamodel.Node { return nil } ================================================ FILE: schema/dmt/compile.go ================================================ package schemadmt import ( "fmt" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/schema" ) // Compile transforms a description of a schema in raw data model ("dmt") form // into a compiled schema.TypeSystem, which is the ready-to-use form. // // The first parameter is mutated by this process, // and the second parameter is the data source. // // The compilation process includes first inserting the "prelude" types into the // schema.TypeSystem -- that is, the "type Bool bool" and "type String string", etc, // which are generally presumed to be present in any type system. // // The compilation process attempts to check the validity of the schema at a logical level as it goes. // For example, references to type names not present elsewhere in the same schema are now an error // (even though that has been easily representable in the dmt.Schema form up until this point). // // Note that this API is EXPERIMENTAL and will likely change. // It supports many features of IPLD Schemas, // but it may yet not support all of them. // It supports several validations for logical coherency of schemas, // but may not yet successfully reject all invalid schemas. func Compile(ts *schema.TypeSystem, node *Schema) error { // Add basic types schema.SpawnDefaultBasicTypes(ts) // Add the schema types err := SpawnSchemaTypes(ts, node) if err != nil { return err } // TODO: if this fails and the user forgot to check Compile's returned error, // we can leave the TypeSystem in an unfortunate broken state: // they can obtain types out of the TypeSystem and they are non-nil, // but trying to use them in any way may result in panics. // Consider making that less prone to misuse, such as making it illegal to // call TypeByName until ValidateGraph is happy. if errs := ts.ValidateGraph(); errs != nil { // Return the first error. for _, err := range errs { return err } } return nil } // SpawnSchemaTypes is a lighter verion of compile that doesn't add basic types and doesn't validate the graph -- // for use when you want to build a type system from multiple sources and validate later func SpawnSchemaTypes(ts *schema.TypeSystem, node *Schema) error { for _, name := range node.Types.Keys { defn := node.Types.Values[name] // TODO: once ./schema supports anonymous/inline types, remove the ts argument. typ, err := spawnType(ts, name, defn) if err != nil { return err } ts.Accumulate(typ) } return nil } // Note that the parser and compiler support defaults. We're lacking support in bindnode. func todoFromImplicitlyFalseBool(b *bool) bool { if b == nil { return false } return *b } func anonTypeName(nameOrDefn TypeNameOrInlineDefn) string { if nameOrDefn.TypeName != nil { return *nameOrDefn.TypeName } defn := *nameOrDefn.InlineDefn switch { case defn.TypeDefnMap != nil: defn := defn.TypeDefnMap return fmt.Sprintf("Map__%s__%s", defn.KeyType, anonTypeName(defn.ValueType)) case defn.TypeDefnList != nil: defn := defn.TypeDefnList return fmt.Sprintf("List__%s", anonTypeName(defn.ValueType)) case defn.TypeDefnLink != nil: return anonLinkName(*defn.TypeDefnLink) default: panic(fmt.Errorf("%#v", defn)) } } func anonLinkName(defn TypeDefnLink) string { if defn.ExpectedType != nil { return fmt.Sprintf("Link__%s", *defn.ExpectedType) } return "Link__Link" } func parseKind(s string) datamodel.Kind { switch s { case "map": return datamodel.Kind_Map case "list": return datamodel.Kind_List case "null": return datamodel.Kind_Null case "bool": return datamodel.Kind_Bool case "int": return datamodel.Kind_Int case "float": return datamodel.Kind_Float case "string": return datamodel.Kind_String case "bytes": return datamodel.Kind_Bytes case "link": return datamodel.Kind_Link default: return datamodel.Kind_Invalid } } func spawnType(ts *schema.TypeSystem, name schema.TypeName, defn TypeDefn) (schema.Type, error) { switch { // Scalar types without parameters. case defn.TypeDefnBool != nil: return schema.SpawnBool(name), nil case defn.TypeDefnString != nil: return schema.SpawnString(name), nil case defn.TypeDefnBytes != nil: return schema.SpawnBytes(name), nil case defn.TypeDefnInt != nil: return schema.SpawnInt(name), nil case defn.TypeDefnFloat != nil: return schema.SpawnFloat(name), nil case defn.TypeDefnList != nil: typ := defn.TypeDefnList tname := "" if typ.ValueType.TypeName != nil { tname = *typ.ValueType.TypeName } else if tname = anonTypeName(typ.ValueType); ts.TypeByName(tname) == nil { anonDefn := TypeDefn{ TypeDefnMap: typ.ValueType.InlineDefn.TypeDefnMap, TypeDefnList: typ.ValueType.InlineDefn.TypeDefnList, TypeDefnLink: typ.ValueType.InlineDefn.TypeDefnLink, } anonType, err := spawnType(ts, tname, anonDefn) if err != nil { return nil, err } ts.Accumulate(anonType) } switch { case typ.Representation == nil || typ.Representation.ListRepresentation_List != nil: // default behavior default: return nil, fmt.Errorf("TODO: support other list repr in schema package") } return schema.SpawnList(name, tname, todoFromImplicitlyFalseBool(typ.ValueNullable), ), nil case defn.TypeDefnMap != nil: typ := defn.TypeDefnMap tname := "" if typ.ValueType.TypeName != nil { tname = *typ.ValueType.TypeName } else if tname = anonTypeName(typ.ValueType); ts.TypeByName(tname) == nil { anonDefn := TypeDefn{ TypeDefnMap: typ.ValueType.InlineDefn.TypeDefnMap, TypeDefnList: typ.ValueType.InlineDefn.TypeDefnList, TypeDefnLink: typ.ValueType.InlineDefn.TypeDefnLink, } anonType, err := spawnType(ts, tname, anonDefn) if err != nil { return nil, err } ts.Accumulate(anonType) } switch { case typ.Representation == nil || typ.Representation.MapRepresentation_Map != nil: // default behavior case typ.Representation.MapRepresentation_Stringpairs != nil: return nil, fmt.Errorf("TODO: support stringpairs map repr in schema package") default: return nil, fmt.Errorf("TODO: support other map repr in schema package") } return schema.SpawnMap(name, typ.KeyType, tname, todoFromImplicitlyFalseBool(typ.ValueNullable), ), nil case defn.TypeDefnStruct != nil: typ := defn.TypeDefnStruct var fields []schema.StructField for _, fname := range typ.Fields.Keys { field := typ.Fields.Values[fname] tname := "" if field.Type.TypeName != nil { tname = *field.Type.TypeName } else if tname = anonTypeName(field.Type); ts.TypeByName(tname) == nil { // Note that TypeDefn and InlineDefn aren't the same enum. anonDefn := TypeDefn{ TypeDefnMap: field.Type.InlineDefn.TypeDefnMap, TypeDefnList: field.Type.InlineDefn.TypeDefnList, TypeDefnLink: field.Type.InlineDefn.TypeDefnLink, } anonType, err := spawnType(ts, tname, anonDefn) if err != nil { return nil, err } ts.Accumulate(anonType) } fields = append(fields, schema.SpawnStructField(fname, tname, todoFromImplicitlyFalseBool(field.Optional), todoFromImplicitlyFalseBool(field.Nullable), )) } var repr schema.StructRepresentation switch { case typ.Representation.StructRepresentation_Map != nil: rp := typ.Representation.StructRepresentation_Map if rp.Fields == nil { repr = schema.SpawnStructRepresentationMap2(nil, nil) break } renames := make(map[string]string, len(rp.Fields.Keys)) implicits := make(map[string]schema.ImplicitValue, len(rp.Fields.Keys)) for _, name := range rp.Fields.Keys { details := rp.Fields.Values[name] if details.Rename != nil { renames[name] = *details.Rename } if imp := details.Implicit; imp != nil { var sumVal schema.ImplicitValue switch { case imp.Bool != nil: sumVal = schema.ImplicitValue_Bool(*imp.Bool) case imp.String != nil: sumVal = schema.ImplicitValue_String(*imp.String) case imp.Int != nil: sumVal = schema.ImplicitValue_Int(*imp.Int) default: panic("TODO: implicit value kind") } implicits[name] = sumVal } } repr = schema.SpawnStructRepresentationMap2(renames, implicits) case typ.Representation.StructRepresentation_Tuple != nil: rp := typ.Representation.StructRepresentation_Tuple if rp.FieldOrder == nil { repr = schema.SpawnStructRepresentationTuple() break } return nil, fmt.Errorf("TODO: support for tuples with field orders in the schema package") case typ.Representation.StructRepresentation_Stringjoin != nil: join := typ.Representation.StructRepresentation_Stringjoin.Join if join == "" { return nil, fmt.Errorf("stringjoin has empty join value") } repr = schema.SpawnStructRepresentationStringjoin(join) case typ.Representation.StructRepresentation_Listpairs != nil: repr = schema.SpawnStructRepresentationListPairs() default: return nil, fmt.Errorf("TODO: support other struct repr in schema package") } return schema.SpawnStruct(name, fields, repr, ), nil case defn.TypeDefnUnion != nil: typ := defn.TypeDefnUnion var members []schema.TypeName for _, member := range typ.Members { if member.TypeName != nil { members = append(members, *member.TypeName) } else { tname := anonLinkName(*member.UnionMemberInlineDefn.TypeDefnLink) members = append(members, tname) if ts.TypeByName(tname) == nil { anonDefn := TypeDefn{ TypeDefnLink: member.UnionMemberInlineDefn.TypeDefnLink, } anonType, err := spawnType(ts, tname, anonDefn) if err != nil { return nil, err } ts.Accumulate(anonType) } } } remainingMembers := make(map[string]bool) for _, memberName := range members { remainingMembers[memberName] = true } validMember := func(memberName string) error { switch remaining, known := remainingMembers[memberName]; { case remaining: remainingMembers[memberName] = false return nil case !known: return fmt.Errorf("%q is not a valid member of union %q", memberName, name) default: return fmt.Errorf("%q is duplicate in the union repr of %q", memberName, name) } } var repr schema.UnionRepresentation switch { case typ.Representation.UnionRepresentation_Kinded != nil: rp := typ.Representation.UnionRepresentation_Kinded table := make(map[datamodel.Kind]schema.TypeName, len(rp.Keys)) for _, kindStr := range rp.Keys { kind := parseKind(kindStr) member := rp.Values[kindStr] switch { case member.TypeName != nil: memberName := *member.TypeName if err := validMember(memberName); err != nil { return nil, err } table[kind] = memberName case member.UnionMemberInlineDefn != nil: tname := anonLinkName(*member.UnionMemberInlineDefn.TypeDefnLink) if err := validMember(tname); err != nil { return nil, err } table[kind] = tname } } repr = schema.SpawnUnionRepresentationKinded(table) case typ.Representation.UnionRepresentation_Keyed != nil: rp := typ.Representation.UnionRepresentation_Keyed table := make(map[string]schema.TypeName, len(rp.Keys)) for _, key := range rp.Keys { member := rp.Values[key] switch { case member.TypeName != nil: memberName := *member.TypeName if err := validMember(memberName); err != nil { return nil, err } table[key] = memberName case member.UnionMemberInlineDefn != nil: tname := anonLinkName(*member.UnionMemberInlineDefn.TypeDefnLink) if err := validMember(tname); err != nil { return nil, err } table[key] = tname } } repr = schema.SpawnUnionRepresentationKeyed(table) case typ.Representation.UnionRepresentation_StringPrefix != nil: prefixes := typ.Representation.UnionRepresentation_StringPrefix.Prefixes for _, key := range prefixes.Keys { if err := validMember(prefixes.Values[key]); err != nil { return nil, err } } repr = schema.SpawnUnionRepresentationStringprefix("", prefixes.Values) case typ.Representation.UnionRepresentation_Inline != nil: rp := typ.Representation.UnionRepresentation_Inline if rp.DiscriminantKey == "" { return nil, fmt.Errorf("inline union has empty discriminantKey value") } if rp.DiscriminantTable.Keys == nil || rp.DiscriminantTable.Values == nil { return nil, fmt.Errorf("inline union has empty discriminantTable") } for _, key := range rp.DiscriminantTable.Keys { if err := validMember(rp.DiscriminantTable.Values[key]); err != nil { return nil, err } } repr = schema.SpawnUnionRepresentationInline(rp.DiscriminantKey, rp.DiscriminantTable.Values) default: return nil, fmt.Errorf("TODO: support other union repr in schema package") } for memberName, remaining := range remainingMembers { if remaining { return nil, fmt.Errorf("%q is not present in the union repr of %q", memberName, name) } } return schema.SpawnUnion(name, members, repr, ), nil case defn.TypeDefnEnum != nil: typ := defn.TypeDefnEnum var repr schema.EnumRepresentation // TODO: we should probably also reject duplicates. validMember := func(name string) bool { for _, memberName := range typ.Members { if memberName == name { return true } } return false } switch { case typ.Representation.EnumRepresentation_String != nil: rp := typ.Representation.EnumRepresentation_String for memberName := range rp.Values { if !validMember(memberName) { return nil, fmt.Errorf("%q is not a valid member of enum %q", memberName, name) } } repr = schema.EnumRepresentation_String(rp.Values) case typ.Representation.EnumRepresentation_Int != nil: rp := typ.Representation.EnumRepresentation_Int for memberName := range rp.Values { if !validMember(memberName) { return nil, fmt.Errorf("%q is not a valid member of enum %q", memberName, name) } } repr = schema.EnumRepresentation_Int(rp.Values) default: return nil, fmt.Errorf("TODO: support other enum repr in schema package") } return schema.SpawnEnum(name, typ.Members, repr, ), nil case defn.TypeDefnLink != nil: typ := defn.TypeDefnLink if typ.ExpectedType == nil { return schema.SpawnLink(name), nil } return schema.SpawnLinkReference(name, *typ.ExpectedType), nil case defn.TypeDefnAny != nil: return schema.SpawnAny(name), nil default: panic(fmt.Errorf("%#v", defn)) } } ================================================ FILE: schema/dmt/doc.go ================================================ /* Package schema/dmt contains types and functions for dealing with the data model form of IPLD Schemas. (DMT is short for "data model tree" -- see https://ipld.io/glossary/#dmt .) As with anything that's IPLD data model, this data can be serialized or deserialized into a wide variety of codecs. To contrast this package with some of its neighbors and with some various formats for the data this package describes: Schemas also have a DSL (a domain-specific language -- something that's meant to look nice, and be easy for humans to read and write), which are parsed by the `schema/dsl` package, and produce a DMT form (defined by and handled by this package). Schemas also have a compiled form, which is the in-memory structure that this library uses when working with them; this compiled form differs from the DMT because it can use pointers (and that includes cyclic pointers, which is something the DMT form cannot contain). We use the DMT form (this package) to produce the compiled form (which is the `schema` package). Creating a Compiled schema either flows from DSL(text)->`schema/dsl`->`schema/dmt`->`schema`, or just (some codec, e.g. JSON or CBOR or etc)->`schema/dmt`->`schema`. The `dmt.Schema` type describes the data found at the root of an IPLD Schema document. The `Compile` function turns such data into a `schema.TypeSystem` that is ready to be used. The `dmt.Prototype.Schema` value is a NodePrototype that can be used to handle IPLD Schemas in DMT form as regular IPLD Nodes. Typically this package is imported aliased as "schemadmt", since "dmt" is a fairly generic term in the IPLD ecosystem (see https://ipld.io/glossary/#dmt ). Many types in this package lack documentation directly on the type; generally, these are structs that match the IPLD schema-schema, and so you can find descriptions of them in documentation for the schema-schema. */ package schemadmt ================================================ FILE: schema/dmt/gen.go ================================================ //go:build ignore package main import ( "fmt" "os" "github.com/ipld/go-ipld-prime/node/bindnode" schemadmt "github.com/ipld/go-ipld-prime/schema/dmt" ) func main() { f, err := os.Create("types.go") if err != nil { panic(err) } fmt.Fprintf(f, "package schemadmt\n\n") if err := bindnode.ProduceGoTypes(f, schemadmt.TypeSystem); err != nil { panic(err) } if err := f.Close(); err != nil { panic(err) } } ================================================ FILE: schema/dmt/operations.go ================================================ package schemadmt import ( "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/node/bindnode" ) // ConcatenateSchemas returns a new schema DMT object containing the // type declarations from both. // // As is usual for DMT form data, there is no check about the validity // of the result yet; you'll need to apply `Compile` on the produced value // to produce a usable compiled typesystem or to become certain that // all references in the DMT are satisfied, etc. func ConcatenateSchemas(a, b *Schema) *Schema { // The joy of having an intermediate form that's just regular data model: // we can implement this by simply using data model "copy" operations, // and the result is correct. nb := Prototypes.Schema.NewBuilder() if err := datamodel.Copy(bindnode.Wrap(a, Prototypes.Schema.Type()), nb); err != nil { panic(err) } if err := datamodel.Copy(bindnode.Wrap(b, Prototypes.Schema.Type()), nb); err != nil { panic(err) } return bindnode.Unwrap(nb.Build()).(*Schema) } ================================================ FILE: schema/dmt/roundtrip_test.go ================================================ package schemadmt_test import ( "bytes" "os" "regexp" "strings" "testing" ipldjson "github.com/ipld/go-ipld-prime/codec/json" "github.com/ipld/go-ipld-prime/node/bindnode" "github.com/ipld/go-ipld-prime/schema" schemadmt "github.com/ipld/go-ipld-prime/schema/dmt" qt "github.com/frankban/quicktest" ) func TestRoundtripSchemaSchema(t *testing.T) { t.Parallel() input := "../../.ipld/specs/schemas/schema-schema.ipldsch.json" src, err := os.ReadFile(input) qt.Assert(t, err, qt.IsNil) testRoundtrip(t, string(src), func(updated string) { err := os.WriteFile(input, []byte(updated), 0o777) qt.Assert(t, err, qt.IsNil) }) } func testRoundtrip(t *testing.T, want string, updateFn func(string)) { t.Helper() crre := regexp.MustCompile(`\r?\n`) want = crre.ReplaceAllString(want, "\n") nb := schemadmt.Prototypes.Schema.Representation().NewBuilder() err := ipldjson.Decode(nb, strings.NewReader(want)) qt.Assert(t, err, qt.IsNil) node := nb.Build().(schema.TypedNode) // Ensure the decoded schema compiles as expected. { sch := bindnode.Unwrap(node).(*schemadmt.Schema) var ts schema.TypeSystem ts.Init() err := schemadmt.Compile(&ts, sch) qt.Assert(t, err, qt.IsNil) typeStruct := ts.TypeByName("TypeDefnStruct") if typeStruct == nil { t.Fatal("TypeStruct not found") } } // Ensure we can re-encode the schema as dag-json, // and that it results in the same bytes as prettified by encoding/json. { var buf bytes.Buffer err := ipldjson.Encode(node.Representation(), &buf) qt.Assert(t, err, qt.IsNil) got := buf.String() qt.Assert(t, got, qt.Equals, want) } // For the sake of completeness, check that we can encode the non-repr node. // This just ensures we don't panic or error. { var buf bytes.Buffer err := ipldjson.Encode(node, &buf) qt.Assert(t, err, qt.IsNil) } } ================================================ FILE: schema/dmt/schema.go ================================================ package schemadmt import ( "fmt" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/node/bindnode" "github.com/ipld/go-ipld-prime/schema" ) // Prototypes contains some schema.TypedPrototype values which match // the IPLD schema-schema -- that is, the schema that describes IPLD schemas. // These prototypes create an in-memory representation that is backed by // structs in this package and bindnode. var Prototypes struct { Schema schema.TypedPrototype } //go:generate go run -tags=schemadmtgen gen.go // TypeSystem is a compiled equivalent of the IPLD schema-schema -- that is, the schema that describes IPLD schemas. // // The IPLD schema-schema can be found at https://ipld.io/specs/schemas/schema-schema.ipldsch . var TypeSystem schema.TypeSystem // In this init function, we manually create a type system that *matches* the IPLD schema-schema. // This manual work is unfortunate, and also must be kept in-sync manually, // but is important because breaks a cyclic dependency -- // we use the compiled schema-schema produced by this to parse other schema documents. // We would also use it to parse... the IPLD schema-schema... if that weren't a cyclic dependency. func init() { var ts schema.TypeSystem ts.Init() // I've elided all references to Advancedlayouts stuff for the moment. // (Not because it's particularly hard or problematic; I just want to draw a slightly smaller circle first.) // Prelude ts.Accumulate(schema.SpawnString("String")) ts.Accumulate(schema.SpawnBool("Bool")) ts.Accumulate(schema.SpawnInt("Int")) ts.Accumulate(schema.SpawnFloat("Float")) ts.Accumulate(schema.SpawnBytes("Bytes")) // Schema-schema! // In the same order as the spec's ipldsch file. // Note that ADL stuff is excluded for now, as per above. ts.Accumulate(schema.SpawnString("TypeName")) ts.Accumulate(schema.SpawnStruct("Schema", []schema.StructField{ schema.SpawnStructField("types", "Map__TypeName__TypeDefn", false, false), // also: `advanced AdvancedDataLayoutMap`, but as commented above, we'll pursue this later. }, schema.StructRepresentation_Map{}, )) ts.Accumulate(schema.SpawnMap("Map__TypeName__TypeDefn", "TypeName", "TypeDefn", false, )) ts.Accumulate(schema.SpawnUnion("TypeDefn", []schema.TypeName{ "TypeDefnBool", "TypeDefnString", "TypeDefnBytes", "TypeDefnInt", "TypeDefnFloat", "TypeDefnMap", "TypeDefnList", "TypeDefnLink", "TypeDefnUnion", "TypeDefnStruct", "TypeDefnEnum", "TypeDefnUnit", "TypeDefnAny", "TypeDefnCopy", }, // TODO: spec uses inline repr. schema.SpawnUnionRepresentationKeyed(map[string]schema.TypeName{ "bool": "TypeDefnBool", "string": "TypeDefnString", "bytes": "TypeDefnBytes", "int": "TypeDefnInt", "float": "TypeDefnFloat", "map": "TypeDefnMap", "list": "TypeDefnList", "link": "TypeDefnLink", "union": "TypeDefnUnion", "struct": "TypeDefnStruct", "enum": "TypeDefnEnum", "unit": "TypeDefnUnit", "any": "TypeDefnAny", "copy": "TypeDefnCopy", }), )) ts.Accumulate(schema.SpawnUnion("TypeNameOrInlineDefn", []schema.TypeName{ "TypeName", "InlineDefn", }, schema.SpawnUnionRepresentationKinded(map[datamodel.Kind]schema.TypeName{ datamodel.Kind_String: "TypeName", datamodel.Kind_Map: "InlineDefn", }), )) ts.Accumulate(schema.SpawnUnion("InlineDefn", []schema.TypeName{ "TypeDefnMap", "TypeDefnList", "TypeDefnLink", }, schema.SpawnUnionRepresentationKeyed(map[string]schema.TypeName{ "map": "TypeDefnMap", "list": "TypeDefnList", "link": "TypeDefnLink", }), )) ts.Accumulate(schema.SpawnStruct("TypeDefnBool", []schema.StructField{}, schema.StructRepresentation_Map{}, )) ts.Accumulate(schema.SpawnStruct("TypeDefnString", []schema.StructField{}, schema.StructRepresentation_Map{}, )) ts.Accumulate(schema.SpawnStruct("TypeDefnBytes", []schema.StructField{}, // No BytesRepresentation, since we omit ADL stuff. schema.StructRepresentation_Map{}, )) ts.Accumulate(schema.SpawnStruct("TypeDefnInt", []schema.StructField{}, schema.StructRepresentation_Map{}, )) ts.Accumulate(schema.SpawnStruct("TypeDefnFloat", []schema.StructField{}, schema.StructRepresentation_Map{}, )) ts.Accumulate(schema.SpawnStruct("TypeDefnMap", []schema.StructField{ schema.SpawnStructField("keyType", "TypeName", false, false), schema.SpawnStructField("valueType", "TypeNameOrInlineDefn", false, false), schema.SpawnStructField("valueNullable", "Bool", true, false), // TODO: wants to use the "implicit" feature, but not supported yet schema.SpawnStructField("representation", "MapRepresentation", true, false), // XXXXXX }, schema.StructRepresentation_Map{}, )) ts.Accumulate(schema.SpawnUnion("MapRepresentation", []schema.TypeName{ "MapRepresentation_Map", "MapRepresentation_Stringpairs", "MapRepresentation_Listpairs", }, schema.SpawnUnionRepresentationKeyed(map[string]schema.TypeName{ "map": "MapRepresentation_Map", "stringpairs": "MapRepresentation_Stringpairs", "listpairs": "MapRepresentation_Listpairs", }), )) ts.Accumulate(schema.SpawnStruct("MapRepresentation_Map", []schema.StructField{}, schema.StructRepresentation_Map{}, )) ts.Accumulate(schema.SpawnStruct("MapRepresentation_Stringpairs", []schema.StructField{ schema.SpawnStructField("innerDelim", "String", false, false), schema.SpawnStructField("entryDelim", "String", false, false), }, schema.StructRepresentation_Map{}, )) ts.Accumulate(schema.SpawnStruct("MapRepresentation_Listpairs", []schema.StructField{}, schema.StructRepresentation_Map{}, )) ts.Accumulate(schema.SpawnStruct("TypeDefnList", []schema.StructField{ schema.SpawnStructField("valueType", "TypeNameOrInlineDefn", false, false), schema.SpawnStructField("valueNullable", "Bool", true, false), // TODO: wants to use the "implicit" feature, but not supported yet schema.SpawnStructField("representation", "ListRepresentation", true, false), // XXXXXX }, schema.StructRepresentation_Map{}, )) ts.Accumulate(schema.SpawnUnion("ListRepresentation", []schema.TypeName{ "ListRepresentation_List", }, schema.SpawnUnionRepresentationKeyed(map[string]schema.TypeName{ "list": "ListRepresentation_List", }), )) ts.Accumulate(schema.SpawnStruct("ListRepresentation_List", []schema.StructField{}, schema.StructRepresentation_Map{}, )) ts.Accumulate(schema.SpawnStruct("TypeDefnUnion", []schema.StructField{ // n.b. we could conceivably allow TypeNameOrInlineDefn here rather than just TypeName. but... we'd rather not: imagine what that means about the type-level behavior of the union: the name munge for the anonymous type would suddenly become load-bearing. would rather not. schema.SpawnStructField("members", "List__UnionMember", false, false), schema.SpawnStructField("representation", "UnionRepresentation", false, false), }, schema.StructRepresentation_Map{}, )) ts.Accumulate(schema.SpawnList("List__UnionMember", "UnionMember", false, )) ts.Accumulate(schema.SpawnUnion("UnionMember", []schema.TypeName{ "TypeName", "UnionMemberInlineDefn", }, schema.SpawnUnionRepresentationKinded(map[datamodel.Kind]schema.TypeName{ datamodel.Kind_String: "TypeName", datamodel.Kind_Map: "UnionMemberInlineDefn", }), )) ts.Accumulate(schema.SpawnUnion("UnionMemberInlineDefn", []schema.TypeName{ "TypeDefnLink", }, schema.SpawnUnionRepresentationKeyed(map[string]schema.TypeName{ "link": "TypeDefnLink", }), )) ts.Accumulate(schema.SpawnList("List__TypeName", // todo: this is a slight hack: should be an anon inside TypeDefnUnion.members. "TypeName", false, )) ts.Accumulate(schema.SpawnStruct("TypeDefnLink", []schema.StructField{ schema.SpawnStructField("expectedType", "TypeName", true, false), // todo: this uses an implicit with a value of 'any' in the schema-schema, but that's been questioned before. maybe it should simply be an optional. }, schema.StructRepresentation_Map{}, )) ts.Accumulate(schema.SpawnUnion("UnionRepresentation", []schema.TypeName{ "UnionRepresentation_Kinded", "UnionRepresentation_Keyed", "UnionRepresentation_Envelope", "UnionRepresentation_Inline", "UnionRepresentation_StringPrefix", "UnionRepresentation_BytesPrefix", }, schema.SpawnUnionRepresentationKeyed(map[string]schema.TypeName{ "kinded": "UnionRepresentation_Kinded", "keyed": "UnionRepresentation_Keyed", "envelope": "UnionRepresentation_Envelope", "inline": "UnionRepresentation_Inline", "stringprefix": "UnionRepresentation_StringPrefix", "byteprefix": "UnionRepresentation_BytesPrefix", }), )) ts.Accumulate(schema.SpawnMap("UnionRepresentation_Kinded", "RepresentationKind", "UnionMember", false, )) ts.Accumulate(schema.SpawnMap("UnionRepresentation_Keyed", "String", "UnionMember", false, )) ts.Accumulate(schema.SpawnMap("Map__String__UnionMember", "TypeName", "TypeDefn", false, )) ts.Accumulate(schema.SpawnStruct("UnionRepresentation_Envelope", []schema.StructField{ schema.SpawnStructField("discriminantKey", "String", false, false), schema.SpawnStructField("contentKey", "String", false, false), schema.SpawnStructField("discriminantTable", "Map__String__UnionMember", false, false), }, schema.StructRepresentation_Map{}, )) ts.Accumulate(schema.SpawnStruct("UnionRepresentation_Inline", []schema.StructField{ schema.SpawnStructField("discriminantKey", "String", false, false), schema.SpawnStructField("discriminantTable", "Map__String__TypeName", false, false), }, schema.StructRepresentation_Map{}, )) ts.Accumulate(schema.SpawnStruct("UnionRepresentation_StringPrefix", []schema.StructField{ schema.SpawnStructField("prefixes", "Map__String__TypeName", false, false), }, schema.StructRepresentation_Map{}, )) ts.Accumulate(schema.SpawnStruct("UnionRepresentation_BytesPrefix", []schema.StructField{ schema.SpawnStructField("prefixes", "Map__HexString__TypeName", false, false), }, schema.StructRepresentation_Map{}, )) ts.Accumulate(schema.SpawnMap("Map__HexString__TypeName", "String", "TypeName", false, )) ts.Accumulate(schema.SpawnString("HexString")) ts.Accumulate(schema.SpawnMap("Map__String__TypeName", "String", "TypeName", false, )) ts.Accumulate(schema.SpawnMap("Map__TypeName__Int", "String", "Int", false, )) ts.Accumulate(schema.SpawnString("RepresentationKind")) // todo: RepresentationKind is supposed to be an enum, but we're putting it to a string atm. ts.Accumulate(schema.SpawnStruct("TypeDefnStruct", []schema.StructField{ schema.SpawnStructField("fields", "Map__FieldName__StructField", false, false), // todo: dodging inline defn's again. schema.SpawnStructField("representation", "StructRepresentation", false, false), }, schema.StructRepresentation_Map{}, )) ts.Accumulate(schema.SpawnMap("Map__FieldName__StructField", "FieldName", "StructField", false, )) ts.Accumulate(schema.SpawnString("FieldName")) ts.Accumulate(schema.SpawnStruct("StructField", []schema.StructField{ schema.SpawnStructField("type", "TypeNameOrInlineDefn", false, false), schema.SpawnStructField("optional", "Bool", true, false), // todo: wants to use the "implicit" feature, but not supported yet schema.SpawnStructField("nullable", "Bool", true, false), // todo: wants to use the "implicit" feature, but not supported yet }, schema.StructRepresentation_Map{}, )) ts.Accumulate(schema.SpawnUnion("StructRepresentation", []schema.TypeName{ "StructRepresentation_Map", "StructRepresentation_Tuple", "StructRepresentation_Stringpairs", "StructRepresentation_Stringjoin", "StructRepresentation_Listpairs", }, schema.SpawnUnionRepresentationKeyed(map[string]schema.TypeName{ "map": "StructRepresentation_Map", "tuple": "StructRepresentation_Tuple", "stringpairs": "StructRepresentation_Stringpairs", "stringjoin": "StructRepresentation_Stringjoin", "listpairs": "StructRepresentation_Listpairs", }), )) ts.Accumulate(schema.SpawnStruct("StructRepresentation_Map", []schema.StructField{ schema.SpawnStructField("fields", "Map__FieldName__StructRepresentation_Map_FieldDetails", true, false), // todo: dodging inline defn's again. }, schema.StructRepresentation_Map{}, )) ts.Accumulate(schema.SpawnMap("Map__FieldName__StructRepresentation_Map_FieldDetails", "FieldName", "StructRepresentation_Map_FieldDetails", false, )) ts.Accumulate(schema.SpawnStruct("StructRepresentation_Map_FieldDetails", []schema.StructField{ schema.SpawnStructField("rename", "String", true, false), schema.SpawnStructField("implicit", "AnyScalar", true, false), }, schema.StructRepresentation_Map{}, )) ts.Accumulate(schema.SpawnStruct("StructRepresentation_Tuple", []schema.StructField{ schema.SpawnStructField("fieldOrder", "List__FieldName", true, false), // todo: dodging inline defn's again. }, schema.StructRepresentation_Map{}, )) ts.Accumulate(schema.SpawnList("List__FieldName", "FieldName", false, )) ts.Accumulate(schema.SpawnStruct("StructRepresentation_Stringpairs", []schema.StructField{ schema.SpawnStructField("innerDelim", "String", false, false), schema.SpawnStructField("entryDelim", "String", false, false), }, schema.StructRepresentation_Map{}, )) ts.Accumulate(schema.SpawnStruct("StructRepresentation_Stringjoin", []schema.StructField{ schema.SpawnStructField("join", "String", false, false), // review: "delim" would seem more consistent with others -- but this is currently what the schema-schema says. schema.SpawnStructField("fieldOrder", "List__FieldName", true, false), // todo: dodging inline defn's again. }, schema.StructRepresentation_Map{}, )) ts.Accumulate(schema.SpawnStruct("StructRepresentation_Listpairs", []schema.StructField{}, schema.StructRepresentation_Map{}, )) ts.Accumulate(schema.SpawnStruct("TypeDefnEnum", []schema.StructField{ schema.SpawnStructField("members", "List__EnumMember", false, false), schema.SpawnStructField("representation", "EnumRepresentation", false, false), }, schema.StructRepresentation_Map{}, )) ts.Accumulate(schema.SpawnStruct("Unit", // todo: we should formalize the introdution of unit as first class type kind. []schema.StructField{}, schema.StructRepresentation_Map{}, )) ts.Accumulate(schema.SpawnList("List__EnumMember", "EnumMember", false, )) ts.Accumulate(schema.SpawnString("EnumMember")) ts.Accumulate(schema.SpawnUnion("EnumRepresentation", []schema.TypeName{ "EnumRepresentation_String", "EnumRepresentation_Int", }, schema.SpawnUnionRepresentationKeyed(map[string]schema.TypeName{ "string": "EnumRepresentation_String", "int": "EnumRepresentation_Int", }), )) ts.Accumulate(schema.SpawnMap("EnumRepresentation_String", "EnumMember", "String", false, )) ts.Accumulate(schema.SpawnMap("EnumRepresentation_Int", "EnumMember", "Int", false, )) ts.Accumulate(schema.SpawnStruct("TypeDefnUnit", []schema.StructField{ schema.SpawnStructField("representation", "UnitRepresentation", false, false), }, schema.StructRepresentation_Map{}, )) ts.Accumulate(schema.SpawnString("UnitRepresentation")) // TODO: enum ts.Accumulate(schema.SpawnStruct("TypeDefnAny", []schema.StructField{}, schema.StructRepresentation_Map{}, )) ts.Accumulate(schema.SpawnStruct("TypeDefnCopy", []schema.StructField{ schema.SpawnStructField("fromType", "TypeName", false, false), }, schema.StructRepresentation_Map{}, )) ts.Accumulate(schema.SpawnUnion("AnyScalar", []schema.TypeName{ "Bool", "String", "Bytes", "Int", "Float", }, schema.SpawnUnionRepresentationKinded(map[datamodel.Kind]schema.TypeName{ datamodel.Kind_Bool: "Bool", datamodel.Kind_String: "String", datamodel.Kind_Bytes: "Bytes", datamodel.Kind_Int: "Int", datamodel.Kind_Float: "Float", }), )) if errs := ts.ValidateGraph(); errs != nil { for _, err := range errs { fmt.Printf("- %s\n", err) } panic("not happening") } TypeSystem = ts Prototypes.Schema = bindnode.Prototype( (*Schema)(nil), TypeSystem.TypeByName("Schema"), ) } ================================================ FILE: schema/dmt/types.go ================================================ package schemadmt type Schema struct { Types Map__TypeName__TypeDefn } type Map__TypeName__TypeDefn struct { Keys []string Values map[string]TypeDefn } type TypeDefn struct { TypeDefnBool *TypeDefnBool TypeDefnString *TypeDefnString TypeDefnBytes *TypeDefnBytes TypeDefnInt *TypeDefnInt TypeDefnFloat *TypeDefnFloat TypeDefnMap *TypeDefnMap TypeDefnList *TypeDefnList TypeDefnLink *TypeDefnLink TypeDefnUnion *TypeDefnUnion TypeDefnStruct *TypeDefnStruct TypeDefnEnum *TypeDefnEnum TypeDefnUnit *TypeDefnUnit TypeDefnAny *TypeDefnAny TypeDefnCopy *TypeDefnCopy } type TypeNameOrInlineDefn struct { TypeName *string InlineDefn *InlineDefn } type InlineDefn struct { TypeDefnMap *TypeDefnMap TypeDefnList *TypeDefnList TypeDefnLink *TypeDefnLink } type TypeDefnBool struct { } type TypeDefnString struct { } type TypeDefnBytes struct { } type TypeDefnInt struct { } type TypeDefnFloat struct { } type TypeDefnMap struct { KeyType string ValueType TypeNameOrInlineDefn ValueNullable *bool Representation *MapRepresentation } type MapRepresentation struct { MapRepresentation_Map *MapRepresentation_Map MapRepresentation_Stringpairs *MapRepresentation_Stringpairs MapRepresentation_Listpairs *MapRepresentation_Listpairs } type MapRepresentation_Map struct { } type MapRepresentation_Stringpairs struct { InnerDelim string EntryDelim string } type MapRepresentation_Listpairs struct { } type TypeDefnList struct { ValueType TypeNameOrInlineDefn ValueNullable *bool Representation *ListRepresentation } type ListRepresentation struct { ListRepresentation_List *ListRepresentation_List } type ListRepresentation_List struct { } type TypeDefnUnion struct { Members List__UnionMember Representation UnionRepresentation } type List__UnionMember []UnionMember type UnionMember struct { TypeName *string UnionMemberInlineDefn *UnionMemberInlineDefn } type UnionMemberInlineDefn struct { TypeDefnLink *TypeDefnLink } type List__TypeName []string type TypeDefnLink struct { ExpectedType *string } type UnionRepresentation struct { UnionRepresentation_Kinded *UnionRepresentation_Kinded UnionRepresentation_Keyed *UnionRepresentation_Keyed UnionRepresentation_Envelope *UnionRepresentation_Envelope UnionRepresentation_Inline *UnionRepresentation_Inline UnionRepresentation_StringPrefix *UnionRepresentation_StringPrefix UnionRepresentation_BytesPrefix *UnionRepresentation_BytesPrefix } type UnionRepresentation_Kinded struct { Keys []string Values map[string]UnionMember } type UnionRepresentation_Keyed struct { Keys []string Values map[string]UnionMember } type Map__String__UnionMember struct { Keys []string Values map[string]TypeDefn } type UnionRepresentation_Envelope struct { DiscriminantKey string ContentKey string DiscriminantTable Map__String__UnionMember } type UnionRepresentation_Inline struct { DiscriminantKey string DiscriminantTable Map__String__TypeName } type UnionRepresentation_StringPrefix struct { Prefixes Map__String__TypeName } type UnionRepresentation_BytesPrefix struct { Prefixes Map__HexString__TypeName } type Map__HexString__TypeName struct { Keys []string Values map[string]string } type Map__String__TypeName struct { Keys []string Values map[string]string } type Map__TypeName__Int struct { Keys []string Values map[string]int } type TypeDefnStruct struct { Fields Map__FieldName__StructField Representation StructRepresentation } type Map__FieldName__StructField struct { Keys []string Values map[string]StructField } type StructField struct { Type TypeNameOrInlineDefn Optional *bool Nullable *bool } type StructRepresentation struct { StructRepresentation_Map *StructRepresentation_Map StructRepresentation_Tuple *StructRepresentation_Tuple StructRepresentation_Stringpairs *StructRepresentation_Stringpairs StructRepresentation_Stringjoin *StructRepresentation_Stringjoin StructRepresentation_Listpairs *StructRepresentation_Listpairs } type StructRepresentation_Map struct { Fields *Map__FieldName__StructRepresentation_Map_FieldDetails } type Map__FieldName__StructRepresentation_Map_FieldDetails struct { Keys []string Values map[string]StructRepresentation_Map_FieldDetails } type StructRepresentation_Map_FieldDetails struct { Rename *string Implicit *AnyScalar } type StructRepresentation_Tuple struct { FieldOrder *List__FieldName } type List__FieldName []string type StructRepresentation_Stringpairs struct { InnerDelim string EntryDelim string } type StructRepresentation_Stringjoin struct { Join string FieldOrder *List__FieldName } type StructRepresentation_Listpairs struct { } type TypeDefnEnum struct { Members List__EnumMember Representation EnumRepresentation } type Unit struct { } type List__EnumMember []string type EnumRepresentation struct { EnumRepresentation_String *EnumRepresentation_String EnumRepresentation_Int *EnumRepresentation_Int } type EnumRepresentation_String struct { Keys []string Values map[string]string } type EnumRepresentation_Int struct { Keys []string Values map[string]int } type TypeDefnUnit struct { Representation string } type TypeDefnAny struct { } type TypeDefnCopy struct { FromType string } type AnyScalar struct { Bool *bool String *string Bytes *[]uint8 Int *int Float *float64 } ================================================ FILE: schema/dsl/parse.go ================================================ package schemadsl import ( "bufio" "bytes" "fmt" "io" "os" "reflect" "strconv" "strings" dmt "github.com/ipld/go-ipld-prime/schema/dmt" ) var globalTrue = true // TODO: fuzz testing func ParseBytes(src []byte) (*dmt.Schema, error) { return Parse("", bytes.NewReader(src)) } func ParseFile(path string) (*dmt.Schema, error) { f, err := os.Open(path) if err != nil { return nil, err } defer f.Close() return Parse(path, f) } func Parse(name string, r io.Reader) (*dmt.Schema, error) { p := &parser{ path: name, br: bufio.NewReader(r), line: 1, col: 1, } sch := &dmt.Schema{} sch.Types.Values = make(map[string]dmt.TypeDefn) for { tok, err := p.consumeToken() if err == io.EOF { break } switch tok { case "type": name, err := p.consumeName() if err != nil { return nil, err } defn, err := p.typeDefn() if err != nil { return nil, err } mapAppend(&sch.Types, name, defn) case "advanced": return nil, p.errf("TODO: advanced") default: return nil, p.errf("unexpected token: %q", tok) } } return sch, nil } func mapAppend(mapPtr, k, v interface{}) { // TODO: delete with generics // TODO: error on dupes mval := reflect.ValueOf(mapPtr).Elem() kval := reflect.ValueOf(k) vval := reflect.ValueOf(v) keys := mval.FieldByName("Keys") keys.Set(reflect.Append(keys, kval)) values := mval.FieldByName("Values") if values.IsNil() { values.Set(reflect.MakeMap(values.Type())) } values.SetMapIndex(kval, vval) } type parser struct { path string br *bufio.Reader peekedToken string line, col int } func (p *parser) forwardError(err error) error { var prefix string if p.path != "" { prefix = p.path + ":" } return fmt.Errorf("%s%d:%d: %s", prefix, p.line, p.col, err) } func (p *parser) errf(format string, args ...interface{}) error { return p.forwardError(fmt.Errorf(format, args...)) } func (p *parser) consumeToken() (string, error) { if tok := p.peekedToken; tok != "" { p.peekedToken = "" return tok, nil } for { // TODO: use runes for better unicode support b, err := p.br.ReadByte() if err == io.EOF { return "", err // TODO: ErrUnexpectedEOF? } if err != nil { return "", p.forwardError(err) } p.col++ switch b { case ' ', '\t', '\r': // skip whitespace continue case '\n': // skip newline // TODO: should we require a newline after each type def, struct field, etc? p.line++ p.col = 1 continue case '"': // quoted string quoted, err := p.br.ReadString('"') if err != nil { return "", p.forwardError(err) } return "\"" + quoted, nil case '{', '}', '[', ']', '(', ')', ':', '&': // simple token return string(b), nil case '#': // comment _, err := p.br.ReadString('\n') if err != nil { return "", p.forwardError(err) } // tokenize the newline if err := p.br.UnreadByte(); err != nil { panic(err) // should never happen } continue default: // string token or name var sb strings.Builder sb.WriteByte(b) for { b, err := p.br.ReadByte() if err == io.EOF { // Token ends at the end of the whole input. return sb.String(), nil } if err != nil { return "", p.forwardError(err) } // TODO: should probably allow unicode letters and numbers, like Go? switch { case b >= 'a' && b <= 'z', b >= 'A' && b <= 'Z': case b >= '0' && b <= '9': case b == '_': default: if err := p.br.UnreadByte(); err != nil { panic(err) // should never happen } return sb.String(), nil } sb.WriteByte(b) } } } } func (p *parser) consumePeeked() { if p.peekedToken == "" { panic("consumePeeked requires a peeked token to be present") } p.peekedToken = "" } func (p *parser) peekToken() (string, error) { if tok := p.peekedToken; tok != "" { return tok, nil } tok, err := p.consumeToken() if err != nil { if err == io.EOF { // peekToken is often used when a token is optional. // If we hit io.EOF, that's not an error. // TODO: consider making peekToken just not return an error? return "", nil } return "", err } p.peekedToken = tok return tok, nil } func (p *parser) consumeName() (string, error) { tok, err := p.consumeToken() if err != nil { return "", err } switch tok { case "\"", "{", "}", "[", "]", "(", ")", ":": return "", p.errf("expected a name, got %q", tok) } if tok[0] == '"' { return "", p.errf("expected a name, got string %s", tok) } return tok, nil } func (p *parser) consumeString() (string, error) { tok, err := p.consumeToken() if err != nil { return "", err } if tok[0] != '"' { return "", p.errf("expected a string, got %q", tok) } // Unquote, too. return tok[1 : len(tok)-1], nil } func (p *parser) consumeStringMap() (map[string]string, error) { result := map[string]string{} loop: for { tok, err := p.peekToken() if err != nil { return result, err } switch tok { case "{": p.consumePeeked() case "}": p.consumePeeked() break loop default: key, err := p.consumeName() if err != nil { return result, err } value, err := p.consumeString() if err != nil { return result, err } result[key] = value } } return result, nil } func (p *parser) consumeRequired(tok string) error { got, err := p.consumeToken() if err != nil { return err } if got != tok { return p.errf("expected %q, got %q", tok, got) } return nil } func (p *parser) typeDefn() (dmt.TypeDefn, error) { var defn dmt.TypeDefn kind, err := p.consumeToken() if err != nil { return defn, err } switch kind { case "struct": if err := p.consumeRequired("{"); err != nil { return defn, err } defn.TypeDefnStruct, err = p.typeStruct() case "union": if err := p.consumeRequired("{"); err != nil { return defn, err } defn.TypeDefnUnion, err = p.typeUnion() case "enum": if err := p.consumeRequired("{"); err != nil { return defn, err } defn.TypeDefnEnum, err = p.typeEnum() case "bool": defn.TypeDefnBool = &dmt.TypeDefnBool{} case "bytes": defn.TypeDefnBytes = &dmt.TypeDefnBytes{} case "float": defn.TypeDefnFloat = &dmt.TypeDefnFloat{} case "int": defn.TypeDefnInt = &dmt.TypeDefnInt{} case "link": defn.TypeDefnLink = &dmt.TypeDefnLink{} case "any": defn.TypeDefnAny = &dmt.TypeDefnAny{} case "&": target, err := p.consumeName() if err != nil { return defn, err } defn.TypeDefnLink = &dmt.TypeDefnLink{ExpectedType: &target} case "string": defn.TypeDefnString = &dmt.TypeDefnString{} case "{": defn.TypeDefnMap, err = p.typeMap() case "[": defn.TypeDefnList, err = p.typeList() case "=": from, err := p.consumeName() if err != nil { return defn, err } defn.TypeDefnCopy = &dmt.TypeDefnCopy{FromType: from} default: err = p.errf("unknown type keyword: %q", kind) } return defn, err } func (p *parser) typeStruct() (*dmt.TypeDefnStruct, error) { repr := &dmt.StructRepresentation_Map{} repr.Fields = &dmt.Map__FieldName__StructRepresentation_Map_FieldDetails{} defn := &dmt.TypeDefnStruct{} for { tok, err := p.consumeToken() if err != nil { return nil, err } if tok == "}" { break } name := tok var field dmt.StructField loop: for { tok, err := p.peekToken() if err != nil { return nil, err } switch tok { case "optional": if field.Optional != nil { return nil, p.errf("multiple optional keywords") } field.Optional = &globalTrue p.consumePeeked() case "nullable": if field.Nullable != nil { return nil, p.errf("multiple nullable keywords") } field.Nullable = &globalTrue p.consumePeeked() default: var err error field.Type, err = p.typeNameOrInlineDefn() if err != nil { return nil, err } break loop } } tok, err = p.peekToken() if err != nil { return nil, err } if tok == "(" { details := dmt.StructRepresentation_Map_FieldDetails{} p.consumePeeked() parenLoop: for { tok, err = p.consumeToken() if err != nil { return nil, err } switch tok { case ")": break parenLoop case "rename": str, err := p.consumeString() if err != nil { return nil, err } details.Rename = &str case "implicit": scalar, err := p.consumeToken() if err != nil { return nil, err } var anyScalar dmt.AnyScalar switch { case scalar[0] == '"': // string s, err := strconv.Unquote(scalar) if err != nil { return nil, p.forwardError(err) } anyScalar.String = &s case scalar == "true", scalar == "false": // bool t := scalar == "true" anyScalar.Bool = &t case scalar[0] >= '0' && scalar[0] <= '0': n, err := strconv.Atoi(scalar) if err != nil { return nil, p.forwardError(err) } anyScalar.Int = &n default: return nil, p.errf("unsupported implicit scalar: %s", scalar) } details.Implicit = &anyScalar } } mapAppend(repr.Fields, name, details) } mapAppend(&defn.Fields, name, field) } reprName := "map" // default repr if tok, err := p.peekToken(); err == nil && tok == "representation" { p.consumePeeked() name, err := p.consumeName() if err != nil { return nil, err } reprName = name } if reprName != "map" && len(repr.Fields.Keys) > 0 { return nil, p.errf("rename and implicit are only supported for struct map representations") } switch reprName { case "map": if len(repr.Fields.Keys) == 0 { // Fields is optional; omit it if empty. repr.Fields = nil } defn.Representation.StructRepresentation_Map = repr return defn, nil case "tuple": defn.Representation.StructRepresentation_Tuple = &dmt.StructRepresentation_Tuple{} return defn, nil // TODO: support custom fieldorder case "stringjoin": optMap, err := p.consumeStringMap() if err != nil { return nil, err } join, hasJoin := optMap["join"] if !hasJoin { return nil, p.errf("no join value provided for stringjoin repr") } defn.Representation.StructRepresentation_Stringjoin = &dmt.StructRepresentation_Stringjoin{ Join: join, } return defn, nil case "listpairs": defn.Representation.StructRepresentation_Listpairs = &dmt.StructRepresentation_Listpairs{} return defn, nil default: return nil, p.errf("unknown struct repr: %q", reprName) } } func (p *parser) typeNameOrInlineDefn() (dmt.TypeNameOrInlineDefn, error) { var typ dmt.TypeNameOrInlineDefn tok, err := p.consumeToken() if err != nil { return typ, err } switch tok { case "&": expectedName, err := p.consumeName() if err != nil { return typ, err } typ.InlineDefn = &dmt.InlineDefn{TypeDefnLink: &dmt.TypeDefnLink{ExpectedType: &expectedName}} case "[": tlist, err := p.typeList() if err != nil { return typ, err } typ.InlineDefn = &dmt.InlineDefn{TypeDefnList: tlist} case "{": tmap, err := p.typeMap() if err != nil { return typ, err } typ.InlineDefn = &dmt.InlineDefn{TypeDefnMap: tmap} default: typ.TypeName = &tok } return typ, nil } func (p *parser) typeList() (*dmt.TypeDefnList, error) { defn := &dmt.TypeDefnList{} tok, err := p.peekToken() if err != nil { return nil, err } if tok == "nullable" { defn.ValueNullable = &globalTrue p.consumePeeked() } defn.ValueType, err = p.typeNameOrInlineDefn() if err != nil { return nil, err } if err := p.consumeRequired("]"); err != nil { return defn, err } // TODO: repr return defn, nil } func (p *parser) typeMap() (*dmt.TypeDefnMap, error) { defn := &dmt.TypeDefnMap{} var err error defn.KeyType, err = p.consumeName() if err != nil { return nil, err } if err := p.consumeRequired(":"); err != nil { return defn, err } tok, err := p.peekToken() if err != nil { return nil, err } if tok == "nullable" { defn.ValueNullable = &globalTrue p.consumePeeked() } defn.ValueType, err = p.typeNameOrInlineDefn() if err != nil { return nil, err } if err := p.consumeRequired("}"); err != nil { return defn, err } return defn, nil } func (p *parser) typeUnion() (*dmt.TypeDefnUnion, error) { defn := &dmt.TypeDefnUnion{} var reprKeys []string for { tok, err := p.consumeToken() if err != nil { return nil, err } if tok == "}" { break } if tok != "|" { return nil, p.errf("expected %q or %q, got %q", "}", "|", tok) } var member dmt.UnionMember nameOrInline, err := p.typeNameOrInlineDefn() if err != nil { return nil, err } if nameOrInline.TypeName != nil { member.TypeName = nameOrInline.TypeName } else { if nameOrInline.InlineDefn.TypeDefnLink != nil { member.UnionMemberInlineDefn = &dmt.UnionMemberInlineDefn{TypeDefnLink: nameOrInline.InlineDefn.TypeDefnLink} } else { return nil, p.errf("expected a name or inline link, got neither") } } defn.Members = append(defn.Members, member) key, err := p.consumeToken() if err != nil { return nil, err } reprKeys = append(reprKeys, key) } if err := p.consumeRequired("representation"); err != nil { return nil, err } reprName, err := p.consumeName() if err != nil { return nil, err } switch reprName { case "keyed": repr := &dmt.UnionRepresentation_Keyed{} for i, keyStr := range reprKeys { key, err := strconv.Unquote(keyStr) if err != nil { return nil, p.forwardError(err) } mapAppend(repr, key, defn.Members[i]) } defn.Representation.UnionRepresentation_Keyed = repr case "kinded": repr := &dmt.UnionRepresentation_Kinded{} // TODO: verify keys are valid kinds? enum should do it for us? for i, key := range reprKeys { mapAppend(repr, key, defn.Members[i]) } defn.Representation.UnionRepresentation_Kinded = repr case "stringprefix": repr := &dmt.UnionRepresentation_StringPrefix{ Prefixes: dmt.Map__String__TypeName{ Values: map[string]string{}, }, } for i, key := range reprKeys { // unquote prefix string if len(key) < 2 || key[0] != '"' || key[len(key)-1] != '"' { return nil, p.errf("invalid stringprefix %q", key) } key = key[1 : len(key)-1] // add prefix to prefixes map repr.Prefixes.Keys = append(repr.Prefixes.Keys, key) repr.Prefixes.Values[key] = *defn.Members[i].TypeName } defn.Representation.UnionRepresentation_StringPrefix = repr case "inline": optMap, err := p.consumeStringMap() if err != nil { return nil, err } discriminantKey, hasDiscriminantKey := optMap["discriminantKey"] if !hasDiscriminantKey { return nil, p.errf("no discriminantKey value provided for inline repr") } repr := &dmt.UnionRepresentation_Inline{ DiscriminantKey: discriminantKey, DiscriminantTable: dmt.Map__String__TypeName{ Values: map[string]string{}, }, } // TODO: verify member types all have map representation for i, qkey := range reprKeys { key, err := strconv.Unquote(qkey) if err != nil { return nil, fmt.Errorf("invalid discriminant key %q: %w", key, err) } repr.DiscriminantTable.Keys = append(repr.DiscriminantTable.Keys, key) repr.DiscriminantTable.Values[key] = *defn.Members[i].TypeName } defn.Representation.UnionRepresentation_Inline = repr default: return nil, p.errf("TODO: union repr %q", reprName) } return defn, nil } func (p *parser) typeEnum() (*dmt.TypeDefnEnum, error) { defn := &dmt.TypeDefnEnum{} var reprKeys []string for { tok, err := p.consumeToken() if err != nil { return nil, err } if tok == "}" { break } if tok != "|" { return nil, p.errf("expected %q or %q, got %q", "}", "|", tok) } name, err := p.consumeToken() if err != nil { return nil, err } defn.Members = append(defn.Members, name) if tok, err := p.peekToken(); err == nil && tok == "(" { p.consumePeeked() key, err := p.consumeToken() if err != nil { return nil, err } reprKeys = append(reprKeys, key) if err := p.consumeRequired(")"); err != nil { return defn, err } } else { reprKeys = append(reprKeys, "") } } reprName := "string" // default repr if tok, err := p.peekToken(); err == nil && tok == "representation" { p.consumePeeked() name, err := p.consumeName() if err != nil { return nil, err } reprName = name } switch reprName { case "string": repr := &dmt.EnumRepresentation_String{} for i, key := range reprKeys { if key == "" { continue // no key; defaults to the name } if key[0] != '"' { return nil, p.errf("enum string representation used with non-string key: %s", key) } unquoted, err := strconv.Unquote(key) if err != nil { return nil, p.forwardError(err) } mapAppend(repr, defn.Members[i], unquoted) } defn.Representation.EnumRepresentation_String = repr case "int": repr := &dmt.EnumRepresentation_Int{} for i, key := range reprKeys { if key[0] != '"' { return nil, p.errf("enum int representation used with non-string key: %s", key) } unquoted, err := strconv.Unquote(key) if err != nil { return nil, p.forwardError(err) } parsed, err := strconv.Atoi(unquoted) if err != nil { return nil, p.forwardError(err) } mapAppend(repr, defn.Members[i], parsed) } defn.Representation.EnumRepresentation_Int = repr default: return nil, p.errf("unknown enum repr: %q", reprName) } return defn, nil } ================================================ FILE: schema/dsl/parse_test.go ================================================ package schemadsl_test import ( "bytes" "flag" "os" "path/filepath" "regexp" "strings" "testing" ipldjson "github.com/ipld/go-ipld-prime/codec/json" "github.com/ipld/go-ipld-prime/node/bindnode" "github.com/ipld/go-ipld-prime/schema" schemadmt "github.com/ipld/go-ipld-prime/schema/dmt" schemadsl "github.com/ipld/go-ipld-prime/schema/dsl" "gopkg.in/yaml.v2" qt "github.com/frankban/quicktest" ) var update = flag.Bool("update", false, "update testdata files in-place") func TestParseSchemaSchema(t *testing.T) { t.Parallel() inputSchema := "../../.ipld/specs/schemas/schema-schema.ipldsch" inputJSON := "../../.ipld/specs/schemas/schema-schema.ipldsch.json" src, err := os.ReadFile(inputSchema) qt.Assert(t, err, qt.IsNil) srcJSON, err := os.ReadFile(inputJSON) qt.Assert(t, err, qt.IsNil) testParse(t, string(src), string(srcJSON), func(updated string) { err := os.WriteFile(inputJSON, []byte(updated), 0o777) qt.Assert(t, err, qt.IsNil) }) } type yamlFixture struct { Schema string Canonical string `yaml:",omitempty"` Expected string ExpectedParsed interface{} `yaml:",omitempty"` Blocks []yamlFixtureBlock `yaml:",omitempty"` BadBlocks []string `yaml:"badBlocks,omitempty"` BadBlocksParsed []interface{} `yaml:",omitempty"` } type yamlFixtureBlock struct { Actual string `yaml:",omitempty"` ActualParsed interface{} `yaml:",omitempty"` Expected string `yaml:",omitempty"` ExpectedParsed interface{} `yaml:",omitempty"` } func TestParse(t *testing.T) { t.Parallel() matches, err := filepath.Glob("../../.ipld/specs/schemas/tests/*.yml") qt.Assert(t, err, qt.IsNil) qt.Assert(t, matches, qt.Not(qt.HasLen), 0) for _, ymlPath := range matches { ymlPath := ymlPath // do not reuse range var name := filepath.Base(ymlPath) t.Run(name, func(t *testing.T) { t.Parallel() data, err := os.ReadFile(ymlPath) qt.Assert(t, err, qt.IsNil) var fixt yamlFixture err = yaml.Unmarshal(data, &fixt) qt.Assert(t, err, qt.IsNil) testParse(t, fixt.Schema, fixt.Expected, func(updated string) { updated = strings.Replace(updated, "\t", " ", -1) fixt.Expected = updated data, err = yaml.Marshal(&fixt) qt.Assert(t, err, qt.IsNil) // Note that this will strip comments. // Probably don't commit its changes straight away. err = os.WriteFile(ymlPath, data, 0o777) qt.Assert(t, err, qt.IsNil) }) }) } } func testParse(t *testing.T, inSchema, inJSON string, updateFn func(string)) { t.Helper() inJSON = strings.Replace(inJSON, " ", "\t", -1) // fix non-tab indenting crre := regexp.MustCompile(`\r?\n`) inJSON = crre.ReplaceAllString(inJSON, "\n") // fix windows carriage-return sch, err := schemadsl.ParseBytes([]byte(inSchema)) qt.Assert(t, err, qt.IsNil) // Ensure the parsed schema compiles as expected. { var ts schema.TypeSystem ts.Init() err := schemadmt.Compile(&ts, sch) qt.Assert(t, err, qt.IsNil) qt.Assert(t, ts.Names(), qt.Not(qt.HasLen), 0) } // Ensure we can encode the schema as the json codec, // and that it results in the same bytes as the ipldsch.json file. { node := bindnode.Wrap(sch, schemadmt.Prototypes.Schema.Type()) var buf bytes.Buffer err := ipldjson.Encode(node.Representation(), &buf) qt.Assert(t, err, qt.IsNil) // If we're updating, write to the file. // Otherwise, expect the files to be equal. got := buf.String() if *update { updateFn(got) return } qt.Assert(t, got, qt.Equals, inJSON, qt.Commentf("run 'go test -update' to write to the iplsch.json file")) } // TODO: ensure that doing a json codec decode results in the same Schema Go // value that we got by parsing the DSL. } ================================================ FILE: schema/errors.go ================================================ package schema import ( "fmt" "strings" "github.com/ipld/go-ipld-prime/datamodel" ) // TODO: errors in this package remain somewhat slapdash. // // - datamodel.ErrUnmatchable is used as a catch-all in some places, and contains who-knows-what values wrapped in the Reason field. // - sometimes this wraps things like strconv errors... and on the one hand, i'm kinda okay with that; on the other, maybe saying a bit more with types before getting to that kind of shrug would be nice. // - we probably want to use `Type` values, right? // - or do we: because then we probably need a `Repr bool` next to it, or lots of messages would be nonsensical. // - this is *currently* problematic because we don't actually generate type info consts yet. Hopefully soon; but the pain, meanwhile, is... substantial. // - "substantial" is an understatement. it makes incremental development almost impossible because stringifying error reports turn into nil pointer crashes! // - other ipld-wide errors like `datamodel.ErrWrongKind` *sometimes* refer to a TypeName... but don't *have* to, because they also arise at the merely-datamodel level; what would we do with these? // - it's undesirable (not to mention intensely forbidden for import cycle reasons) for those error types to refer to schema.Type. // - if we must have TypeName treated stringily in some cases, is it really useful to use full type info in other cases -- inconsistently? // - regardless of where we end up with this, some sort of an embed for helping deal with munging and printing this would probably be wise. // - generally, whether you should expect an "datamodel.Err*" or a "schema.Err*" from various methods is quite unclear. // - it's possible that we should wrap *all* schema-level errors in a single "datamodel.ErrSchemaNoMatch" error of some kind, to fix the above. (and maybe that's what ErrUnmatchable really is.) as yet undecided. // ErrUnmatchable is the error raised when processing data with IPLD Schemas and // finding data which cannot be matched into the schema. // It will be returned by NodeAssemblers and NodeBuilders when they are fed unmatchable data. // As a result, it will also often be seen returned from unmarshalling // when unmarshalling into schema-constrained NodeAssemblers. // // ErrUnmatchable provides the name of the type in the schema that data couldn't be matched to, // and wraps another error as the more detailed reason. type ErrUnmatchable struct { // TypeName will indicate the named type of a node the function was called on. TypeName string // Reason must always be present. ErrUnmatchable doesn't say much otherwise. Reason error } func (e ErrUnmatchable) Error() string { return fmt.Sprintf("matching data to schema of %s rejected: %s", e.TypeName, e.Reason) } // Reasonf returns a new ErrUnmatchable with a Reason field set to the Errorf of the arguments. // It's a helper function for creating untyped error reasons without importing the fmt package. func (e ErrUnmatchable) Reasonf(format string, a ...interface{}) ErrUnmatchable { return ErrUnmatchable{e.TypeName, fmt.Errorf(format, a...)} } // Is provides support for Go's standard errors.Is function so that // errors.Is(yourError, ErrUnmatchable) may be used to match the type of error. func (e ErrUnmatchable) Is(err error) bool { _, ok := err.(ErrUnmatchable) return ok } // ErrMissingRequiredField is returned when calling 'Finish' on a NodeAssembler // for a Struct that has not has all required fields set. type ErrMissingRequiredField struct { Missing []string } func (e ErrMissingRequiredField) Error() string { return "missing required fields: " + strings.Join(e.Missing, ",") } // Is provides support for Go's standard errors.Is function so that // errors.Is(yourError, ErrMissingRequiredField) may be used to match the type of error. func (e ErrMissingRequiredField) Is(err error) bool { _, ok := err.(ErrMissingRequiredField) return ok } // ErrInvalidKey indicates a key is invalid for some reason. // // This is only possible for typed nodes; specifically, it may show up when // handling struct types, or maps with interesting key types. // (Other kinds of key invalidity that happen for untyped maps // fall under ErrRepeatedMapKey or ErrWrongKind.) // (Union types use ErrInvalidUnionDiscriminant instead of ErrInvalidKey, // even when their representation strategy is maplike.) type ErrInvalidKey struct { // TypeName will indicate the named type of a node the function was called on. TypeName string // Key is the key that was rejected. Key datamodel.Node // Reason, if set, may provide details (for example, the reason a key couldn't be converted to a type). // If absent, it'll be presumed "no such field". // ErrUnmatchable may show up as a reason for typed maps with complex keys. Reason error } func (e ErrInvalidKey) Error() string { if e.Reason == nil { return fmt.Sprintf("invalid key for map %s: %q: no such field", e.TypeName, e.Key) } else { return fmt.Sprintf("invalid key for map %s: %q: %s", e.TypeName, e.Key, e.Reason) } } // Is provides support for Go's standard errors.Is function so that // errors.Is(yourError, ErrInvalidKey) may be used to match the type of error. func (e ErrInvalidKey) Is(err error) bool { _, ok := err.(ErrInvalidKey) return ok } // ErrNoSuchField may be returned from lookup functions on the Node // interface when a field is requested which doesn't exist, // or from assigning data into on a MapAssembler for a struct // when the key doesn't match a field name in the structure // (or, when assigning data into a ListAssembler and the list size has // reached out of bounds, in case of a struct with list-like representations!). type ErrNoSuchField struct { Type Type Field datamodel.PathSegment } func (e ErrNoSuchField) Error() string { if e.Type == nil { return fmt.Sprintf("no such field: {typeinfomissing}.%s", e.Field) } return fmt.Sprintf("no such field: %s.%s", e.Type.Name(), e.Field) } // Is provides support for Go's standard errors.Is function so that // errors.Is(yourError, ErrNoSuchField) may be used to match the type of error. func (e ErrNoSuchField) Is(err error) bool { _, ok := err.(ErrNoSuchField) return ok } // ErrNotUnionStructure means data was fed into a union assembler that can't match the union. // // This could have one of several reasons, which are explained in the detail text: // // - there are too many entries in the map; // - the keys of critical entries aren't found; // - keys are found that aren't any of the expected critical keys; // - etc. // // TypeName is currently a string... see comments at the top of this file for // remarks on the issues we need to address about these identifiers in errors in general. type ErrNotUnionStructure struct { TypeName string Detail string } func (e ErrNotUnionStructure) Error() string { return fmt.Sprintf("cannot match schema: union structure constraints for %s caused rejection: %s", e.TypeName, e.Detail) } // Is provides support for Go's standard errors.Is function so that // errors.Is(yourError, ErrNotUnionStructure) may be used to match the type of error. func (e ErrNotUnionStructure) Is(err error) bool { _, ok := err.(ErrNotUnionStructure) return ok } ================================================ FILE: schema/gen/go/HACKME.md ================================================ hacking gengo ============= What the heck? -------------- We're doing code generation. The name of the game is "keep it simple". Most of this is implemented as string templating. No, we didn't use the Go AST system. We could have; we didn't. Implementing this as string templating seemed easier to mentally model, and the additional value provided by use of AST libraries seems minimal since we feed the outputs into a compiler for verification immediately anyway. Some things seem significantly redundant. That's probably because they are. In general, if there's a choice between apparent redundancy in the generator itself versus almost any other tradeoff which affects the outputs, we prioritize the outputs. (This may be especially noticeable when it comes to error messages: we emit a lot of them... while making sure they contain very specific references. This leads to some seemingly redundant code, but good error messages are worth it.) See [README_behaviors](README_behaviors.md) for notes about the behaviors of the code output by the generator; this document is about the generator code itself and the design thereof. Entrypoints ----------- The most important intefaces are all in [`generators.go`](generators.go). The function you're most likely looking for that "does the thing" is the `Generate(outputPath string, pkgName string, schema.TypeSystem, *AdjunctCfg)` method, which can be found in the [`generate.go`](generate.go) file. You can take any of the functions inside of that and use them as well, if you want more granular control over what content ends up in which files. The eventual plan is be able to drive this whole apparatus around via a CLI which consumes IPLD Schema files. Implementing this can come after more of the core is done. (Seealso the `schema/tmpBuilders.go` file a couple directories up for why this is currently filed as nontrivial/do-later.) Organization ------------ ### How many things are generated, anyway? There are roughly *seven* categories of API to generate per type: - 1: the readonly thing a native caller uses - 2: the builder thing a native caller uses - 3: the readonly typed node - 4: the builder/assembler for typed node - 5: the readonly representation node - 6: the builder/assembler via representation - 7: and a maybe wrapper (And these are just the ones nominally visible in the exported API surface! There are several more concrete types than this implied by some parts of that list, such as iterators for the nodes, internal parts of builders, and so forth.) These numbers will be used to describe some further organization. ### How are the generator components grouped? There are three noteworthy types of generator internals: - `TypeGenerator` - `NodeGenerator` - `NodebuilderGenerator` The first one is where you start; the latter two do double duty for each type. Exported types for purpose 1, 2, 3, and 7 are emitted from `TypeGenerator` (3 from the embedded `NodeGenerator`). The exported type for purpose 5 is emitted from another `NodeGenerator` instance. The exported types for purposes 4 and 6 are emitted from two distinct `NodebuilderGenerator` instances. For every variation in type kind and representation strategy for that type kind, one type implementing `TypeGenerator` is composed, and it has functions which yield all the other interfaces for addressing the various purposes. ### How are files and their contents grouped? Most of the files in this package are following a pattern: - for each kind: - `gen{Kind}.go` -- has emitters for the native type parts (1, 2, 7) and type-level node behaviors (3, 4). - for each representation that kind can have: - `gen{Kind}Repr{ReprStrat}.go` -- has emitters for (5, 6). A `mixins` sub-package contains some code which is used and embedded in the generators in this package. These features are mostly per-kind -- representation kind, not type-level kind. For example, you'll see "map" behaviors from the mixins package added to "struct" generators. ### What are all these abbreviations? See [HACKME_abbrevs.md](HACKME_abbrevs.md). ### Code architecture See [HACKME_tradeoffs.md](HACKME_tradeoffs.md) for an overview of tradeoffs, and which priorities we selected in this package. (There are *many* tradeoffs.) See [HACKME_memorylayout.md](HACKME_memorylayout.md) for a (large) amount of exposition on how this code is designed in order to be allocation-avoidant and fast in general. See [HACKME_templates.md](HACKME_templates.md) for some overview on how we've used templates, and what forms of reuse and abstraction there are. See [HACKME_scalars.md](HACKME_scalars.md) for some discussion of scalars and (why we generate more of them than you might expect). See [HACKME_maybe.md](HACKME_maybe.md) for notes how how the 'maybe' feature (how we describe `nullable` and `optional` schema features in generated golang code) has evolved. Testing ------- See [HACKME_testing.md](HACKME_testing.md) for some details about how this works. In general, try to copy some of the existing tests and get things to suit. Be advised that we use the golang plugin feature, and that has some additional requirements of your development environment than is usual in golang. (Namely, you have to be on linux and you have to have a c compiler!) ================================================ FILE: schema/gen/go/HACKME_abbrevs.md ================================================ abbreviations ============= A lot of abbreviations are used the generated code in the interest of minimizing the size of the output. This is a short index of the most common ones: - `n` -- **n**ode, of course -- the accessor functions on node implementations usually refer to their 'this' as 'n'. - `na` -- **n**ode **a**ssembler - `la` or `ma` may also be seen for **l**ist and **m**ap in some contexts (but may refer to same type being called `na` in other contexts). - `w` -- **w**ork-in-progress node -- you'll see this in nearly every assembler (e.g. `na.w` is a very common string). inside nodes: - `x` -- a placeholder for "the thing" for types that contain only one element of data (e.g., the string inside a codegen'd node of string kind). - `t` -- **t**able -- the slice inside most map nodes that is used for alloc amortizations and maintaining order. - `m` -- **m**ap -- the actual map inside most map nodes (seealso `t`, which is usually a sibling). inside assemblers: - `va` -- **v**alue **a**ssembler -- an assembler for values in lists or maps (often embedded in the node assembler, e.g. `ma.va` and `la.va` are common strings). - `ka` -- **k**ey **a**ssembler -- an assembler for keys in maps (often embedded in the node assembler, e.g. `ma.ka` is a common string). - `ca_*` -- **c**hild **a**ssembler -- the same concept as `ka` and `va`, but appearing in structs and other types that have differentiated children. - `cm` -- **c**hild **m**aybe -- a piece of memory sometimes found in a node assembler for statekeeping for child assemblers. - `m` -- **m**aybe pointer -- a pointer to where an assembler should put a mark when it's finished. (this is often a pointer to a parent structure's 'cm'!) ================================================ FILE: schema/gen/go/HACKME_dry.md ================================================ HACKME: "don't repeat yourself": how-to (and, limits of that goal) ================================================================== Which kind of extraction applies? --------------------------------- Things vary in how identical they are. - **Textually identical**: Some code is textually identical between different types, varying only in the most simple and obvious details, like the actual type name it's attached to. - These cases can often be extracted on the generation side... - We tend to put them in `genparts{*}.go` files. - But the output is still pretty duplicacious. - **Textually identical minus simple variations**: Some code is textually *nearly* identical, but varies in relatively minor ways (such as whether or not the "Repr" is part of munges, and "Representation()" calls are made, etc). - These cases can often be extracted on the generation side... - We tend to put them in `genparts{*}.go` files. - There's just a bit more `{{ various templating }}` injected in them, compared to other textually identical templates. - But the output is still pretty duplicacious. - **Linkologically identical**: When code is not _only_ textually identical, but also refers to identical types. - These cases can be extracted on the generation side... - but it may be questionable to do so: if its terse enough in the output, there's that much less incentive to make a template-side shorthand for it. - The output in this case can actually be deduplicated! - It's possible we haven't bothered yet. **That doesn't mean it's not worth it**; we probably just haven't had time yet. PRs welcome. - How? - functions? This is the most likely to apply. - embedded types? We haven't seen many cases where this can help, yet (unfortunately?). - shared constants? - It's not always easy to do this. - We usually put something in the "minima" file. - We don't currently have a way to toggle whether whole features or shared constants are emitted in the minima file. Todo? - This requires keeping state that records what's necessary as we go, so that we can do them all together at the end. - May also require varying the imports at the top of the minima file. (But: by doing it only here, we can avoid that complexity in every other file.) - **This is actually pretty rare**. _Things that are textually identical are not necessarily linkologically identical_. - One can generally turn things that are textually identical into linkologically identical by injecting an interface into the types... - ... but this isn't always a *good* idea: - if this would cause more allocations? Yeah, very no. - even if this can be done without a heap allocation, it probably means inlining and other optimizations will become impossible for the compiler to perform, and often, we're not okay with the performance implications of that either. - **Identical if it wasn't for debugability**: In some cases, code varies only by some small constants... and really, those constants could be removed entirely. If... we didn't care about debugging. Which we do. - This is really the same as "textually identical minus simple variations", but worth talking about briefly just because of the user story around it. - A bunch of the error-thunking methods on Node and NodeAssemblers exemplify this. - It's really annoying that we can't remove this boilerplate entirely from the generated code. - It's also basically impossible, because we *want* information that varies per type in those error messages. What mechanism of extraction should be used? -------------------------------------------- - (for gen-side dry) gen-side functions - this is most of what we've done so far - (for gen-side dry) sub-templates - we currently don't really use this at all - (for gen-side dry) template concatenation - some of this: kinded union representations do this - (for output-side dry) output side functions - some of this: see "minima" file. - (for output-side dry) output side embeds - we currently don't really use this at all (it hasn't really turned out applicable in any cases yet). Don't overdo it --------------- I'd rather have longer templates than harder-to-read and harder-to-maintain templates. There's a balance to this and it's tricky to pick out. A good heuristic to consider might be: are we extracting this thing because we can? Or because if we made changes to this thing in the future, we'd expect to need to make that change in every single place we've extracted it from, which therefore makes the extraction a net win for maintainability? If it's the latter: then yes, extract it. If it's not clear: maybe let it be. (It may be the case that the preferable balance for DRYing changes over time as we keep maintaining things. We'll see; but it's certainly the case that the first draft of this package has favored length heavily. There was a lot of "it's not clear" when the maintainability heuristic was applied during the first writing of this; that may change! If so, that's great.) ================================================ FILE: schema/gen/go/HACKME_maybe.md ================================================ How do maybe/nullable/optional work? ==================================== (No, this document is not about things that we should "maybe" hack on. It's about the feature we use to describe `nullable` and `optional` fields in generated golang code.) background ---------- You'll need to understand what the `nullable` and `optional` modifiers in IPLD schemas mean. The https://specs.datamodel.io/ site has more content about that. ### how this works outside of schemas There are concepts of null and of absent present in the core `Node` and `NodeAssembler` interfaces. `Node` specifies `IsNull() bool` and `IsAbsent() bool` predicates; and `NodeAssembler` specifies an `AssignNull` function. There are also singleton values available called `datamodel.Null` and `datamodel.Absent` which report true for `IsNull` and `IsAbsent`, respectively. These singletons can be used by an function that need to return a null or absence indicator. There's really no reason for any package full of `Node` implementations need to make their own types for these values, since the singletons are always fine to use. However, there's also nothing stopping a `Node` implementation from doing interesting custom internal memory layouts to describe whether they contain nulls, etc -- and there's nothing particularly blessed about the `datamodel.Null` singleton. Any value reporting `IsNull` to be `true` must be treated indistinguishably from `datamodel.Null`. This indistinguishability is bidirectional. For example, if you have some `myFancyNodeType`, and it answers `IsNull` as `true`, and you insert this into a `basicnode.Map`, then ask for that value back from the map later... you're very likely to get `datamodel.Null`, and not your concrete value of `myFancyNodeType` back again. (This contract is important because some node implementations may compress the concept of null into a bitmask, or otherwise similarly optimize things internally.) #### null The concept of "null" has a Kind in the IPLD Data Model. It's implemented by the `datamodel.nullNode` type (which has no fields -- it's a "unit" type), and is exposed as the `datamodel.Null` singleton value. (More generally, `datamodel.Node` can be null by having its `Kind()` method return `datamodel.Kind_Null`, and having the `IsNull()` method return `true`. However, most code prefers to return the `datamodel.Null` singleton value whenever it can.) Null values can be easily produced: the `AssignNull()` method on `datamodel.NodeAssembler` produces nulls; and many codecs have some concept of null, meaning deserialization can produce them. Null values work essentially the same way in both the plain Data Model and when working with Schemas. #### absent There's also a concept of "absent". "Absent" is separate and distinct from the concept of "null" -- null is still a _value_; absent just means _nothing there_. (Those familiar with javascript might note that javascript also has concepts of "null" versus "undefined". It's the same idea -- we just call it "absent" instead of "undefined".) Absent is implemented by the `datamodel.absentNode` type (which has no fields -- it's a "unit" type), and is exposed as the `datamodel.Absent` singleton value. (More generally, an `datamodel.Node` can describe itself as containing "absent" by having the `IsAbsent()` method return `true`. (The `Kind()` method still returns `datamodel.Kind_Null`, for lack of better option.) However, most code prefers to return the `datamodel.Absent` singleton value whenever it can.) Absent values aren't really used at the Data Model level. If you ask for a map key that isn't present in the map, the lookup method will return `nil` and `ErrNotExists`. Absent values *do* show up at the Schema level, however. Specifically, in structs: a struct can have a field which is `optional`, one of the values such an optional field may report itself as having is `datamodel.Absent`. This represents when a value *wasn't present* in the serialized form of the struct, even though the schema lets us know that it could be, and that it's part of the struct's type. (Accordingly, no `ErrNotExists` is returned for a lookup of that field -- the field is always considered to _exist_... the value is just _absent_.) Iterators will also return the field name key, together with `datamodel.Absent` as the value. However, absent values can't really be *created*. There's no such thing as an `AssignAbsent` or `AssignAbsent` method on the `datamodel.NodeAssembler` interface. Codecs similarly can't produce absent as a value (obviously -- codecs work over `datamodel.NodeAssembler`, so how could they?). Absent values are just produced by implication, when a field is defined, but its value isn't set. Despite absent values not being used or produced at the Data Model, we still have methods like `IsAbsent` specified as part of the `datamodel.Node` interface so that it's possible to write code which is generic over either plain Data Model or Schema data while using just that interface. ### the above is all regarding generic interfaces As long as we're talking about the `datamodel.Node` _interface_, we talk about the `datamodel.Null` and `datamodel.Absent` singletons, and their contracts in terms of the interface. (Part of the reason this works is because an interface, in golang, comes in two parts: a pointer to the typeinfo of the inhabitant value, and a pointer to the value itself. This means anywhere we have an `datamodel.Node` return type, we can toss `datamodel.Null` or `datamodel.Absent` into it with no additional overhead!) When we talk about concrete types, rather than the `datamodel.Node` _interface_ -- as we're going to, in codegen -- it's a different scenario. We can't just return `datamodel.Null` pointers for a `genresult.Foo` value; if `genresult.Foo` is a concrete type, that's just flat out a compile error. So what shall we do? We introduce the "maybe" types. the maybe types --------------- The general rule of "return `datamodel.Null` whenever you have a null value" holds up only as long as our API is returning monomorphized `datamodel.Node` interfaces -- in that situation, `datamodel.Null` fits within `datamodel.Node`, and there's no trouble. This doesn't hold up when we get to codegen. Or rather, more specifically, it even holds up for codegen... as long as we're still returning monomorphized `datamodel.Node` interfaces (and a decent amount of the API surface still does so). At the moment we want to return a concrete native type, it breaks. We call methods created by codegen that use specific types (e.g., methods that you _couldn't have_ without codegen) "speciated" methods. And we do want them! So we have to decide how to handle null and absent for these speciated methods. ### goals of the maybe types There are a couple of things we want to accomplish with the maybe types: - Be able to have speciated methods that return a specific type (for doc, editor autocomplete, etc purposes). - Be able to have speciated methods that return specific *concrete* type (i.e. not only do we want to be more specific than `datamodel.Node`, we don't want an interface _at all_ -- so that the compiler can do inlining and optimization and so forth). - Make reading and writing code that uses speciated methods and handles nullable or optional fields be reasonably ergonomic (and as always, this may vary by "taste"). And we'll consider one more fourth, bonus goal: - It would be nice if the maybe types can clearly discuss whether the type is `(null|value)` vs `(absent|value)` vs `(absent|null|value)`, because this would let the golang compiler help check more of our logical correctness in code written using optionals and nullables. ### there is only one type generated for each maybe For every type generated, there is one maybe type also generated. (At least this much is clearly necessary to satisfy the goals about "specific types".) This means *we dropped the bonus goal* above. Making `(null|value)` vs `(absent|value)` vs `(absent|null|value)` distinguishable to the golang compiler would require three *additional* generated types (for obvious reasons) for each type specified by the Schema. We decided that's simply too onerous. (A different codegen project could certainly make a different choice here, though.) ### the symbol for maybe types For some type named `T` generated into a package named `gen`... - the main type symbol is `gen.T`; - the maybe for that type is `gen.MaybeT`; Beware that this may spell trouble if your schema contains any types with names starting in "Maybe". (You can use adjunct config to change symbols for those types, if necessary.) (There are also internal symbols for the same things, but these are prefixed in such a way as to make collision not a concern.) ### maybe types don't implement the full Node interface The "maybe" types don't implement the full `datamodel.Node` interface. They could have! They don't. Arguments that went in favor of implementing `Node`: - generally "seem fine" - certainly makes sense to be able to 'IsNull' on it like any other Node. - if in practice the maybe is embedded, we can return an internal pointer to it just fine, so there's no obvious runtime perf reason not to. Arguments against: - it's another type with a ton of methods. or two, or four. - may increase the binary size. possibly by a significant constant multiplier. - definitely increases the gsloc size, significantly. - would it have a 'Type' accessor on it? - if so, what does it say? - simply not sure how useful this is! - istm one will often either be passing the MaybeT to other speciated functions, or, fairly immediately de-maybing it. - if this is true, the number of times anyone wants to treat it as a Node in practice are near zero. - does this imply the existence of a _MaybeT__Assembler type, as well? - binary and gsloc size still drifting up; this needs to justify itself and provide value to be worth it. - what would be the expected behavior of handing a _MaybeT__Assembler to something like unmarshalling? - if you have a null in the root, you can describe this with a kinded union, and probably would be better off for it. - if you have can absent value in the root of a document you're unmarshalling... what? That's called "EOF". - does a _MaybeT__Assembler show up usefully in the middle of a tree? - it does not! there's always a _P_ValueAssembler type involved there anyway (this is needed for parent state machine purposes), and it largely delegates to the _T__Assembler, but is already a perfect position to add on the "maybe" semantics if the P type has them for its children. The arguments against carried the day. ### the maybe type is emebbedable It's important that the "maybe" types be embeddable, for all the same reasons that [we normally want embeddable types](./HACKME_memorylayout.md#embed-by-default). It's interesting to consider the alternatives, though: We could've bitpacked the isAbsent or isNull flags for a field into one word at the top of a struct, for example. But, there are numerous drawbacks to this: - the complexity of this is high. - it would be exposed to anyone who writes addntl code in-package, which is asking for errors. - the only thing this buys us is *slightly* less resident memory size. - and long story short: if you look at how many other programming language do this, pareto-wise, no one in the world at large appears to care. - it only applies to structs! maps or lists would require yet more custom bitpacking of a different arrangement. If someone wants to do another codegen project someday, or make PRs to this one, which does choose bitpacking, it would probably be neat. It's just a lot of effort for a payout that doesn't seem to often be worth it. (We also ended up using pointers to a field with a `schema.Maybe` type _heavily_ in the internals of our codegen outputs, in order to let child and parent assemblers coordinate. Rebuilding this to work with a bitpacking alignment and yet still be composable enough to do its job... uufdah. Tricky. It might be possible to use the current system in the assembler state, but flip it bitpack in the resulting immutable nodes, and thereby get the best of both worlds. If you who reads this is enthusiastic, feel free to explore it.) ### ...but the user is only exposed to the pointer form This is the same story as for the main types: it's covered in [unexported implementations, exported aliases](./HACKME_memorylayout.md#unexported-implementations-exported-aliases). Genenerally, this "shielded" type means you can only have a MaybeT with valid contents, because no one can ever produce the uninitialized "zero" value of the type. This means there's no "invalid" state which can kick you in the shins at runtime, and we generally regard that as a good thing. It also just keeps things syntactically simple. One always refers to "MaybeT"; never with a star. ### whether or not the maybe's inhabitant type is embedded is based on adjunct config Although the maybe type itself is embeddable, its _inhabitant_ may be either embedded in the maybe type or be a pointer, at your option. This is clearest to explain in code: you can have either: ```go type MaybeFoo struct { m schema.Maybe // enum bit for present|absent|null v Foo // the inhabitant (here, embedded) } ``` or: ```go type MaybeFoo struct { m schema.Maybe // enum bit for present|absent|null v *Foo // the inhabitant (here, a pointer!) } ``` (Yes, we're talking about a one-character difference in the code.) Which of these two forms is generated can be selected by adjunct config. ("Adjunct" config just means: it's not part of the schema; it's part of the config for this codegen tool.) There are advantages to each: - the embedded form is ([as usual](./HACKME_memorylayout.md#embed-by-default)), faster for workloads where the value is usually present (it provokes fewer allocations). - the pointer form may use less memory when the value is absent; it works for cyclic structures; and if assigning a whole subtree at once, it allows faster assignment. Also, for cyclic structures, such as `type Foo {String:nullable Foo}`, or `type Bar struct{ recurse optional Bar }`, the pointer form is *required*. (Otherwise... how big of a slab of memory would we be allocating? Infinite? Nope; compile error.) By default, we generate the pointer form. However, your application may experience significant performance improvements by selectively using the embed form. Check it out and tune for what's right for your application. (FUTURE: we should make more clever defaults: it's reasonable to default to embed form for any type that is of scalar kind.) implementation detail notes --------------------------- ### how state machines and maybes work Assemblers for recursive stuff have state machines that are used to insure orderly transitions between each key and value assembly, and that a complete entry has been assembled before the next entry or the finish. (For example, you can't go key-then-key in a map, nor start a value and then start another value before finishing the first one in a list, nor finish a map when you've just inserted a key and no value, and so forth.) One part of this is straightforward: we simply implement state machines, using bog-standard patterns around a typed uint and logical transition guards in all the relevant functions. Done and done. Except... How do child assemblers signal to their parent that they've become finished? Theoretically, easy; in practice, to work efficiently... This poses a bit of an implementation challenge. One obvious solution is to put a callback field in assemblers, and have the parent assembler supply the child assembler with a callback that can update the parent's state machine when the child becomes finished. This is logically correct, but practically, problematic and Not Fast: it requires generating a closure of some kind which composes the function pointer with the pointer to that parent assembler: and since this is two words of memory, it implies an allocation and (unfortunately) a heap escape. An allocation per child key and value in a recursive structure is unacceptable; we want to set a _much_ higher bar for performance here. So, we move on to less obvious solutions: we're all in the same package here, so we can twiddle the bits of our neighboring structures quite directly, yes? What if we just have assemblers contain pointers to a state machine uint, and they do a fixed-value compare-and-swap when they're done? This is terrifyingly direct and has no abstractions, yes indeed: but we do generally assume control all the code in this package for any of our correctness constraints, so this is in-bounds (if admittedly uncomfortable). Now let's combine that with one more concern: nullables. When an assembler is not at the root of a document, it may need to accept null values. We could do this by generating distinct assembler types for use in positions where nulls are allowed; but though such an approach would work, it is bulky. We'd much rather be able to reuse assembler types in either scenario. So, let's have assemblers contain two pointers: the already-familiar 'w' pointer, and also an 'm' pointer. The 'm' pointer effectively communicates up whether the child has become finished when it becomes either 'Maybe_Null' or 'Maybe_Value'. We add a few new states to the 'm' value, and use it to hint in both directions: assemblers will assume nulls are not an acceptable transition *unless* the 'm' value comes initialized with a hint that we are in a situation where they work. The costs here are "some": it's another pointer indirection and memory set. However, compared to the alternatives, it's pretty good: versus an allocation (in the callback approach), this is a huge win; and we're even pretty safe to bet that that pointer indirection is going to land in a cache line already hot. You can find the additional magic consts crammed into `schema.Maybe` fields for this statekeeping during assembly defined in the "minima" file in codegen output. They are named `midvalue` and `allowNull`. this could have been different ------------------------------ There are many ways this design could've been different: ### we could have every maybe type implement Node As already discussed above, it would cause a lot of extra boilerplate methods, increasing both the generated code source size and binary size; but on the plus side, it would've been in some ways arguably more consistent. We didn't. ### we could've generated three maybes per type Already discussed above. We didn't. ### we could've designed schemas differently A lot of the twists of the design originate from the fact that both `optional` and `nullable` are both rather special as well as very contextual in IPLD Schemas (e.g., `optional` is only permitted in a very few special places in a schema). If we had built a very different type system, maybe things would come out differently. Some of this has some exploration in some gists: - https://gist.github.com/warpfork/9dd8b68deff2b90f96167c900ea31eec#dubious-soln-drop-nullable-completely-make-inline-anonymous-union-syntax-instead - https://gist.github.com/warpfork/9dd8b68deff2b90f96167c900ea31eec#soln-change-how-schemas-regard-nullable-and-optional - https://gist.github.com/warpfork/9dd8b68deff2b90f96167c900ea31eec#soln-support-absent-as-a-discriminator-in-kinded-unions But suffice to say, that's a very big topic. Optionals and nullables are the way they are because they seemed like useful concepts for describing the structure of data which has serial forms; how they map onto any particular programming language (such as Go) was a secondary concern. This design for a golang library is trying to do its best within that. ### we could've done X with technique Y Probably, yes :) This is just one implementation of codegen for Golang for IPLD Schemas. Competing implementations that make different choices are absolutely welcome :) ================================================ FILE: schema/gen/go/HACKME_memorylayout.md ================================================ about memory layout =================== Memory layout is important when designing a system for going fast. It also shows up in exported types (whether or not they're pointers, etc). For the most part, we try to hide these details; or, failing that, at least make them appear consistent. There's some deeper logic required to *pick* which way we do things, though. This document was written to describe codegen and all of the tradeoffs here, but much of it (particularly the details about embedding and internal pointers) also strongly informed the design of the core NodeAssembler semantics, and thus also may be useful reading to understand some of the forces that shaped even the various un-typed node implementations. Prerequisite understandings --------------------------- The following headings contain brief summaries of information that's important to know in order to understand how we designed the IPLD data structure memory layouts (and how to tune them). Most of these concepts are common to many programming languages, so you can likely skim those sections if you know them. Others are fairly golang-specific. ### heap vs stack The concept of heap vs stack in Golang is pretty similar to the concept in most other languages with garbage collection, so we won't cover it in great detail here. The key concept to know: the *count* of allocations which are made on the heap significantly affects performance. Allocations on the heap consume CPU time both when made, and later, as part of GC. The *size* of the allocations affects the total memory needed, but does *not* significantly affect the speed of execution. Allocations which are made on the stack are (familiarly) effectively free. ### escape analysis "Escape Analysis" refers to the efforts the compiler makes to figure out if some piece of memory can be kept on the stack or if it must "escape" to the heap. If escape analysis finds that some memory can be kept on the stack, it will prefer to do so (and this is faster/preferable because it both means allocation is simple and that no 'garbage' is generated to collect later). Since whether things are allocated on the stack or the heap affects performance, the concept of escape analysis is important. The details (fortunately) are not: For the purposes of what we need to do in in our IPLD data structures, our goal with our code is to A) flunk out and escape to heap as soon as possible, but B) do that in one big chunk of memory at once (because we'll be able to use [internal pointers](#internal-pointers) thereafter). One implication of escape analysis that's both useful and easy to note is that whether or not you use a struct literal (`Foo{}`) or a pointer (`&Foo{}`) *does not determine* whether that memory gets allocated on the heap or stack. If you use a pointer, the escape analysis can still prove that the pointer never escapes, it will still end up allocated on the stack. Another way to thing about this is: use pointers freely! By using pointers, you're in effect giving the compiler *more* freedom to decide where memory resides; in contrast, avoiding the use of pointers in method signitures, etc, will give the compiler *less* choice about where the memory should reside, and typically forces copying. Giving the compiler more freedom generally has better results. **pro-tip**: you can compile a program with the arguments `-gcflags "-m -m"` to get lots of information about the escape analysis the compiler performs. ### embed vs pointer Structs can be embedded -- e.g. `type Foo struct { field Otherstruct }` -- or referenced by a pointer -- e.g. `type Foo struct { field *Otherstruct }`. The difference is substantial. When structs are embedded, the layout in memory of the larger struct is simply a concatenation of the embedded structs. This means the amount of memory that structure takes is the sum of the size of all of the embedded things; and by the other side of the same coint, the *count* of allocations needed (remember! the *count* affects performance more than the *size*, as we briefly discussed in the [heap-vs-stack](#heap-vs-stack) section) is exactly *one*. When pointers are used instead of embedding, the parent struct is typically smaller (pointers are one word of memory, whereas the embedded thing can often be larger), and null values can be used... but if fields are assigned to some other value than null, there's a very high likelihood that heap allocations will start cropping up in the process of creating values to take pointers to before then assigning the pointer field! (This can be subverted by either [escape analysis](#escape-analysis) (though it's fairly uncommon), or by [internal pointers](#internal-pointers) (which are going to turn out very important, and will be discussed later)... but it's wise to default to worrying about it until you can prove that one of the two will save you.) When setting fields, another difference appears: a pointer field takes one instruction (assuming the value already exists, and we're not invoking heap allocation to get the pointer!) to assign, whereas an embedded field generally signifies a memcopy, which may take several instructions if the embedded value is large. You can see how the choice between use of pointers and embeds results in significantly different memory usage and performance characteristics! (Quick mention in passing: "cache lines", etc, are also potential concerns that can be addressed by embedding choices. However, it's probably wise to attend to GC first. While cache alignment *can* be important, it's almost always going to be a winning bet that GC will be a much higher impact concern.) It is an unfortunate truth that whether or not a field can be null in Golang and whether or not it's a pointer are two properties that are conflated -- you can't choose one independently of the other. (The reasoning for this is based on intuitions around mechanical sympathy -- but it's worth mentioning that a sufficiently smart compiler *could* address both the logical separation and simultaneously have the compiler solve for the mechanical sympathy concerns in order to reach good performance in many cases; Golang just doesn't do so.) ### interfaces are two words and may cause implicit allocation Interfaces in Golang are always two words in size. The first word is a pointer to the type information for what the interface contains. The second word is a pointer to the data itself. This means if some data is assigned into an interface value, it *must* become a pointer -- the compiler will do this implicitly; and this is the case even if the type info in the first word retains a claim that the data is not a pointer. In practice, this also almost guarantees in practice that the data in question will escape to the heap. (This applies even to primitives that are one word in size! At least, as of golang version 1.13 -- keep an eye on on the `runtime.convT32` functions if you want to look into this further; the `mallocgc` call is clear to see. There's a special case inside `malloc` which causes zero values to get a free pass (!), but in all other cases, allocation will occur.) Knowing this, you probably can conclude a general rule of thumb: if your application is going to put a value in an interface, and *especially* if it's going to do that more than once, you're probably best off explicitly handling it as a pointer rather than a value. Any other approach will be very likely to provoke unnecessary copy behavior and/or multiple unnecessary heap allocations as the value moves in and out of pointer form. (Fun note: if attempting to probe this by microbenchmarking experiments, be careful to avoid using zero values! Zero values get special treatment and avoid allocations in ways that aren't general.) ### internal pointers "Internal pointers" refer to any pointer taken to some position in a piece of memory that was already allocated somewhere. For example, given some `type Foo struct { a, b, c Otherstruct }`, the value of `f := &Foo{}` and `b := &f.b` will be very related: they will differ by the size of `Otherstruct`! The main consequence of this is: using internal pointers can allow you to construct large structure containing many pointers... *without* using a correspondingly large *count of allocations*. This unlocks a lot of potential choices for how to build data structures in memory while minimizing allocs! Internal pointers are not without their tradeoffs, however: in particular, internal pointers have an interesting relationship with garbage collection. When there's an internal pointer to some field in a large struct, that pointer will cause the *entire* containing struct to be still considered to be referenced for garbage collection purposes -- that is, *it won't be collected*. So, in our example above, keeping a reference to `&f.b` will in fact cause memory of the size of *three* `Otherstruct`s to be uncollectable, not one. You can find more information about internal pointers in this talk: https://blog.golang.org/ismmkeynote ### inlining functions Function inlining is an important compiler optimization. Inlining optimizes in two regards: one, can remove some of the overhead of function calls; and two, it can enable *other* optimizations by getting the relevant instruction blocks to be located together and thus rearrangeable. (Inlining does increase the compiled binary size, so it's not all upside.) Calling a function has some fixed overhead -- shuffling arguments from registers into calling convention order on the stack; potentially growing the stack; etc. While these overheads are small in practice... if the function is called many (many) times, this overhead can still add up. Inlining can remove these costs! More interestingly, function inlining can also enable *other* optimizations. For example, a function that *would* have caused escape analysis to flunk something out to the heap *if* that function as called was alone... can potentially be inlined in such a way that in its contextual usage, the escape analysis flunking can actually disappear entirely. Many other kinds of optimizations can similarly be enabled by inlining. This makes designing library code to be inline-friendly a potentially high-impact concern -- sometimes even more so than can be easily seen. The exact mechanisms used by the compiler to determine what can (and should) be inlined may vary significantly from version to version of the Go compiler, which means one should be cautious of spending too much time in the details. However, we *can* make useful choices around things that will predictably obstruct inlining -- such as [virtual function calls](#virtual-function-calls). Occasionally there are positive stories in teasing the inliner to do well, such as https://blog.filippo.io/efficient-go-apis-with-the-inliner/ (but these seem to generally require a lot of thought and probably aren't the first stop on most optimization quests). ### virtual function calls Function calls which are intermediated by interfaces are called "virtual" function calls. (You may also encounter the term "v-table" in compiler and runtime design literature -- this 'v' stands for "virtual".) Virtual function calls generally can't be inlined. This can have significant effects, as described in the [inlining functions](#inlining-functions) section -- it both means function call overhead can't be removed, and it can have cascading consequences by making other potential optimizations unreachable. Resultant Design Features ------------------------- ### concrete implementations We generate a concrete type for each type in the schema. Using a concrete type means methods on it are possible to inline. This is important to us because most of the methods are "accessors" -- that is, a style of function that has a small body and does little work -- and these are precisely the sort of function where inlining can add up. ### natively-typed methods in addition to the general interface We generate two sets of methods: **both** the general interface methods to comply with Node and NodeBuilder interfaces, **and** also natively-typed variants of the same methods (e.g. a `Lookup` method for maps that takes the concrete type key and returns the concrete type value, rather than taking and returning `Node` interfaces). While both sets of methods can accomplish the same end goals, both are needed. There are two distinct advantages to natively-typed methods; and at the same time, the need for the general methods is system critical. Firstly, to programmers writing code that can use the concrete types, the natively-typed methods provide more value in the form of compile-time type checking, autocompletion and other tooling assist opportunities, and less verbosity. Secondly, natively-typed functions on concrete types can be higher performance: since they're not [virtual function calls](#virtual-function-calls), we can expect [inlining](#inlining-functions) to work. We might expect this to be particularly consequential in builders and in accessor methods, since these involve numerous calls to methods with small bodies -- precisely the sort of situation that often substantially benefits from inlining. At the same time, it goes without saying that we need the general Node and NodeBuilder interfaces to be satisfied, so that we can write generic library code such as reusable traversals, etc. It is not possible to satisfy both needs with a single set of methods with the Golang typesystem; therefore, we generate both. ### embed by default Embedded structs amortizes the count of memory allocations. This addresses what is typically our biggest concern. The increase in size is generally not consequential. We expect most fields end up filled anyway, so reserving that memory up front is reasonable. (Indeed, unfilled fields are only possible for nullable or optional fields which are implemented as embedded.) If assigning whole sub-trees at once, assignment into embedded fields incurs the cost of a memcopy (whereas by contrast, if fields were pointers, assigning them would be cheap... it's just that we would've had to pay a (possibly _extra_) allocation cost elsewhere earlier.) However, this is usually a worthy trade. Linear memcpy in practice can be significantly cheaper than extra allocations (especially if it's one long memcpy vs many allocations); and if we assume a balance of use cases such as "unmarshal happens more often than sub-tree-assignment", then it's pretty clear we should prioritize getting allocation minimization for unmarshal rather than fret sub-tree assignment. ### nodebuilders point to the concrete type We generate NodeBuilder types which contain a pointer to the type they build. This means we can hold onto the Node pointer when its building is completed, and discard the NodeBuilder. (Or, reset and reuse the NodeBuilder.) Garbage collection can apply on the NodeBuilder independently of the lifespan of the Node it built. This means a single NodeBuilder and its produced Node will require **two** allocations -- one for the NodeBuilder, and a separate one for the Node. (An alternative would be to embed the concrete Node value in the NodeBuilder, and return a pointer to when finalizing the creation of the Node; however, because due to the garbage collection semantics around [internal pointers](#internal-pointers), such a design would cause the entirety of the memory needed in the NodeBuilder to remain uncollectable as long as completed Node is reachable! This would be an unfortunate trade.) While we pay two allocations for the Node and its Builder, we earn that back in spades via our approach to recursion with [NodeAssemblers](#nodeassemblers-accumulate-mutations), and specifically, how [NodeAssemblers embed more NodeAssemblers](#nodeassemblers-embed-nodeassemblers). Long story short: we pay two allocations, yes. But it's *fixed* at two, no matter how large and complex the structure is. ### nodeassemblers accumulate mutations The NodeBuilder type is only used at the root of construction of a value. After that, recursion works with an interface called NodeAssembler instead. A NodeAssembler is essentially the same thing as a NodeBuilder, except _it doesn't return a Node_. This means we can use the NodeAssembler interface to describe constructing the data in the middle of some complex value, and we're not burdened by the need to be able to return the finished product. (Sufficient state-keeping and defensive checks to ensure we don't leak mutable references would not come for free; reducing the number of points we might need to do this makes it possible to create a more efficient system overall.) The documentation on the datamodel.NodeAssembler type gives some general description of this. NodeBuilder types end up being just a NodeAssembler embed, plus a few methods for exposing the final results and optionally resetting the whole system. ### nodeassemblers embed nodeassemblers In addition to each NodeAssembler containing a pointer to the value they modify (the same as [NodeBuilders](#nodebuilders-point-to-the-concrete-type))... for assemblers that work with recursive structures, they also embed another NodeAssembler for each of their child values. This lets us amortize the allocations for all the *assemblers* in the same way as embedding in the actual value structs let us amortized allocations there. The code for this gets a little complex, and the result also carries several additional limitations to the usage, but it does keep the allocations finite, and thus makes the overall performance fast. (To be more specific, for recursive types that are infinite (namely, maps and lists; whereas structs and unions are finite), the NodeAssembler embeds *one* NodeAssembler for all values. (Obviously, we can't embed an infinite number of them, right?) This leads to a restriction: you can't assemble multiple children of an infinite recursive value simultaneously.) ### nullable and optional struct fields embed too TODO intro There is some chance of over-allocation in the event of nullable or optional fields. We support tuning that via adjunct configuration to the code generator which allows you to opt in to using pointers for fields; choosing to do this will of course cause you to lose out on alloc amortization features in exchange. TODO also resolve the loops note, at bottom ### unexported implementations, exported aliases Our concrete types are unexported. For those that need to be exported, we export an alias to the pointer type. This has an interesting set of effects: - copy-by-value from outside the package becomes impossible; - creating zero values from outside the package becomes impossible; - and yet referring to the type for type assertions remains possible. This addresses one downside to using [concrete implementations](#concrete-implementations): if the concrete implementation is an exported symbol, it means any code external to the package can produce Golang's natural "zero" for the type. This is problematic because it's true even if the Golang "zero" value for the type doesn't correspond to a valid value. While keeping an unexported implementation and an exported interface makes external fabrication of zero values impossible, it breaks inlining. Exporting an alias of the pointer type, however, strikes both goals at once: external fabrication of zero values is blocked, and yet inlining works. Amusing Details and Edge Cases ------------------------------ ### looped references // who's job is it to detect this? // the schema validator should check it... // but something that breaks the cycle *there* doesn't necessarily do so for the emitted code! aggh! // ... unless we go back to optional and nullable both making ptrs unconditionally. Learning more (the hard way) ---------------------------- If this document doesn't provide enough information for you, you've probably graduated to the point where doing experiments is next. :) Prototypes and research examples can be found in the `go-ipld-prime/_rsrch/` directories. In particular, the "multihoisting" and "nodeassembler" packages are relevant, containing research that lead to the drafting of this doc, as well as some partially-worked alternative interface drafts. (You may have to search back through git history to find these directories; they're removed after some time, when the lessons have been applied.) Tests there include some benchmarks (self-explanitory); some tests based on runtime memory stats inspection; and some tests which are simply meant to be disassembled and read thusly. Compiler flags can provide useful insights: - `-gcflags '-S'` -- gives you assembler dump. - read this to see for sure what's inlined and not. - easy to quickly skim for calls like `runtime.newObject`, etc. - often critically useful to ensure a benchmark hasn't optimized out the question you meant to ask it! - generally gives a ground truth which puts an end to guessing. - `-gcflags '-m -m'` -- reports escape analysis and other decisions. - note the two m's -- not a typo: this gives you info in stack form, which is radically more informative than the single-m output. - `-gcflags '-l'` -- disables inlining! - useful on benchmarks to quickly detect whether inlining is a major part of performance. These flags can apply to any command like `go install`... as well as `go test`. Profiling information collected from live systems in use is of course always intensely useful... if you have any on hand. When handling this, be aware of how data-dependent performance can be when handling serialization systems: different workload content can very much lead to different hot spots. Happy hunting. ================================================ FILE: schema/gen/go/HACKME_scalars.md ================================================ What's the deal with scalars, anyway? ===================================== Two sorts of scalars -------------------- There are two sorts of scalars that show up in codegen: - 1: scalars that are just the plain kind (e.g. "string", not even named); - 2: scalars that have named types. Plain scalars can't have any special rules or semantics attached to them. Named types with scalar kinds (aka a "typedef") **can** have additional rules and semantics attached to them. Let's talk about named scalars first, because it's clearer that there's fun there. ### named scalars Named scalars cause a type to be generated. That type information is part of their identity (practically speaking: affects their definition of equality). #### named scalars are never equal even if their contents are It stands to reason that named scalars can't be freely interchanged. If you have a schema: ```ipldsch type Foo string type Bar string ``` ... then you'll get codegen output code with an exported type for each: ```go type Foo struct{ x string } /*...*/ type Bar struct{ x string } /*...*/ ``` ... and clearly, `(Foo{"asdf"} == Bar{"asdf"}) == false`. #### named scalars appear in specialized method argument types and return types Just like any other named type, named scalars will appear in specialized methods which are exported on codegen'd types. For example, if you have a schema: ```ipldsch type Foo string type Bar string type Foomp map {Foo:Bar} ``` ... then you'll get codegen output code which includes a method on Foomp: ```go func (x *Foomp) LookupByNode(k *Foo) (*Bar) { /*...*/ } ``` Such specialized methods are often much shorter, much more efficient to execute, and involve much less error handling to use than their more generalized counterparts on the `datamodel.Node` interface. Note that when named scalars appear in the signitures of specialized methods, they always appear as pointers. They will never be `nil`, but there is still a reason that pointers are used here, and it's based on performance. (The details don't matter as a user, but: it means if those values need to be regarded as the `datamodel.Node` interface again in the future, that boxing is inexpensive since we already have a (heap-escaped long ago) pointer. By contrast, copying by value in more places is likely to result in more heap escapes and thus additional undesirable new allocation costs in the (entirely common!) case that the values end up handled as `datamodel.Node` later.) #### named scalars have a specialized method which unboxes them to a native primitive unconditionally Every named scalar type as a specialized unbox method corresponding to its kind. For example, for a `type Foo string`, there will be a `func (f Foo) String() string` method (in addition to the `func (f Foo) AsString() (string, error)` method, which does the same thing but is stuck presenting an error due to interface conformance even though we know that it's statically impossible). #### named scalars can have additional methods attached to them It's possible for users of codegen to attach additional methods to the types generated for a named scalar. This can be either done for purely aesthetic/ergonomic purposes particular to the user's exact product, or, as part of some extended library features. For example, we plan support extended features like "validation" methods via detecting when a user adds a `Valdiate() error` method to a generated type. ### plain scalars Plain scalars also cause a type to be generated; one type for each kind in the Data Model is sufficient. Plain scalars show up in codegen output packages almost exactly as if there was a short preamble in every schema: ```ipldsch type Int int type Bool bool type Float float type String string type Bytes bytes ``` #### note about schema syntax There's an issue about capitalization that's somewhat unresolved in schemas: namely, is `type Fwee struct { someField string }` allowed, or a parse error? This syntax is questionable because it means some of the scalar kind identifier keywords are allowed in the same place as type names, and it's potentially confusing because when we come to interacting with the generated output code in golang, we still have `String`-with-a-capital-S as a type identifier. At any rate, it seems clear that you can mentally capitalize the 's' at any time you see this debatable syntax. (We should resolve this issue in the specs, which are in the `datamodel.specs` repo.) #### plain scalars appear in specialized method argument types and return types This is the same story as for named scalars. For example, if you have a schema: ```ipldsch type Foomp map {String:String} ``` ... then you'll get codegen output code which includes a method on Foomp: ```go func (x *Foomp) LookupByNode(k String) (String) { /*...*/ } ``` (The exact symbols involved and whether or not they're pointers may vary.) The type might carry less semantic information than it does when a named scalar shows up in the same position, but we still use a generated type (and a pointer) here for two reasons: first of all, and more simply, consistency; but secondly, for the same performance reasons as applied to named scalars (if we need to treat this value as an `datamodel.Node` again in the future, it's much better if we already have a heap pointer rather than a bare primitive value (`runtime.convT*` functions are often not your favorite thing to see in a pprof flamegraph)). (FUTURE: this is still worth review. We might actually want to use bare primitives in a lot of these cases, because surely, if you're about to want to treat something as an `datamodel.Node` again, then you can use the generalized methods conforming to `datamodel.Node` which already yield that...? We'll get more information and impressions about this after trying to use codegen in bulk (especially the specialized methods).) #### plain scalars do not allow additional method attachedments While we can't *stop* developers from modifying the source code emitted by codegen, adding a method to any of the plain scalars is intensely discouraged. Nothing sensible or good can come of trying to attach a "Validate" method to something like the `String` type. Don't do it. Code reuse for plain scalars ---------------------------- We *always* need some type that can contain a plain scalar while also implementing all the `datamodel.Node` methods. Even if we didn't export it or show it in any method signitures anywhere at all, we'd *still* need it for internal implementation of other types, because it's important those types be able to return a pointer to their fields in their implements of the `datamodel.Node` contract (otherwise, they'd be terribly slow and alloc-heavy). ### can we reuse another package's plain scalars? Since there's no functional difference between the plain scalars in a schema and the scalars implementation from another package that's untyped in the first place, can we reuse some code from an untyped package in codegen output packages? No. (Or: "maybe, conditionally, and it would have a lot of caveats and make the untyped package we try to hitch a ride on become significantly weirder, so... it's probably not worth it".) The reason to desire this so there's less (admittedly quite duplicative) code in the package emitted by using codegen. However, there are *many* "cons" which outweigh that single "pro": - This would require the untyped package to export their concrete implementation types. - This is the *only* reason those implementation types would need to be exported, which is a concerning smell all by itself. - In the case of we consider using the 'basicnode' package in particular: - Exporting those types allows creation by casting, which exposes an API surface that's not conventional (nor necessarily even possible) for other packages, and will thus be likely to create confusion as well as create multiple ways of doing things which will make refactors harder. - We don't like allowing casting for creating values in general for reasons explored well in the go-cid refactors to use wrapper structs: if casting is possible, it's far too easy for an end-user to write shoddy code which dodges all constructors and validation logic. - Exporting those types allows unboxing by casting, which again exposes an API surface that's not conventional (nor necessarily even possible) for other packages, and will thus be likely to create confusion as well as create multiple ways of doing things which will make refactors harder. - Since we're talking about scalars and they're essentially copy-by-value (except for bytes -- but we give up and rely on "lawful" code for those anyway, since defensive copies are completely nonviable in performance terms), this doesn't create incorrectness issues... but it's still not *good*. - Note that while casting to concrete types exported by the output package of codegen is considered acceptable, this is a different beast: you still can't get the raw content out without using at least one more unboxing method; and, if you're casting or doing a type switch with type in a codegen package, it should already instantly be clear that your code is no longer general-purpose, and this will surprise no one. - ...And while the above two are true only because the implementation is by typedefs and they could be fixed by using a wrapper struct... that fix would have exactly the effect of making reuse impossible anyway, since the field in that wrapper struct would need to be unexported (otherwise, immutability would then in turn trivially shatter). - The implementation of the scalar for link kinds can't be reused anyway (it *does* use a wrapper struct already, and needs to; type aliases on an interface don't permit adding methods), adding yet more inconsistency and jagged edges to the picture. - The "more unnecessarily(-for-end-user-perspectives) exported symbols" code smell counts about 10x as hard for this package in particular, since it's often one of the first ones a newcomer to this library will see: there shouldn't be weird designs with elaborate and far away justifications poking up here. - Reusing concrete types between packages makes it more likely uncautious users could write code that uses native equality on scalars and get away with it *sometimes*. Since this is still incorrect and would sometimes fail in fully general code, it's better if code like this flunks out as early as possible, which results in a better ecosystem overall. - We like it when error messages can include a type name. It's marginally better for that to be something like "gendemo.String" ('gendemo' being consistent with whatever the rest of the package also says) than just bare "string". There are also a few bits that aren't entirely known (at least, at the time of this writing): namely, how 'any' types are going to be handled in codegen. Probably, though, the answer is: it's just treated as 'datamodel.Node', and the codegen package doesn't export *any* more types which regard this situation because that's already sufficient. Long story short? It's better to have plain scalar types in codegen output, even if they look somewhat duplicative, because trying to do anything fancier either fails outright or spawns ridiculously detailed epicycles of complexity. Emitting the plain scalar types in codegen output is *more consistent* in almost every way, will generate less cognitive load for users, and just plain *works unconditionally*. ================================================ FILE: schema/gen/go/HACKME_templates.md ================================================ Notes about how Templates are used in this package ================================================== This package makes heavy use of go's `text/template`. Some of it is not pretty. But beware of trying to trade elegance for legibility. An overview of the choices that got us here: ### string templating is fine String templating for code is fine, actually. An alternative would be to use the golang AST packages. While those are nice... for our purposes, they're a bit verbose. It's not necessarily helpful to have the visual distance between our generation code and the actual result _increase_ (at least, without a good reason). ### don't make indirections around uncomplicated munges Don't make indirections that aren't needed for simple string operations. (This goes for both the template funcmap, or in the form of reusable templates.) One very common situation is that for some type `T`, there's a related thing to be generated called `T__Foo`, where "Foo" is some kind of modifier that is completely predictable and not user-selected nor configurable. In this situation: simply conjoining `{{ .T }}__Foo` as string in the template is fine! Don't turn it into a shell game with functions that make the reader jump around more to see what's happening. (Even when refactoring, it's easy enough to use simple string replace on these patterns; extracting a function to enable "changing it in one place" doesn't really add much value.) If there's something more complicated going on, or it's user-configurable? Fine, get more functions involved. But be judicious about it. (An earlier draft of the code introduced a new method for each modifier form in the example above. The result was... just repeating the modifiers in the function name in the template! It produced more code, yet no additional flexibility. It is advisable to resist making this mistake again ;)) ### maintain distinction between **symbol** versus **name** - **symbol** is what is used in type and function names in code. - **name** is the string that comes from the schema; it is never modified nor overridable. Symbols can change based on adjunct config. (In theory, they could also be munged to avoid collisions with language constructs or favor certain library conventions... however, this is currently not automatic.) Names can't change. They're _exactly_ what was written in in the Schema. Types and functions and most of the _code_ part of codegen use Symbols. Error messages, reflectable type information, etc, use Names. Symbols may also need to be _generated_ for types that don't have names. For example, `type Foo struct { field {String:Int} }` might result in codegen creating *two* symbols: `Foo`, and `Map__String__Int`. One way to check that the goal is being satisfied is to consider that someone just experiencing error messages from a program should not end up exposed to any information about: - what language the program is written in, - or which codegen tool was used (or even *if* a codegen tool was used), - or what adjunct config was present when codegen was performed. (n.b. this consideration does not include errors with stack traces -- then of course you will unavoidably see symbols appear.) ### anything that is configurable goes through adjunctCfg; adjunctCfg is accessed via the template funcmap For example, all Symbol processing all pipes through an adjunct configuration object. We make this available in the templates via a funcmap so it's available context-free as a nice tidy pipe syntax. ### there are several kinds of reuse (Oh, boy.) It may be wise to read the ["values we can balance"](./HACKME_tradeoffs.md#values-we-can-balance) document before continuing. There's also a _seventh_ tradeoff to consider, in addition to those from that document: how much reuse there is in our _template_ code, and (_eighth!_) how _readable_ and maintainable the template code is. Also, (_ninth!!_) how _fast_ the template code is. We've generally favored *all* of the priorities for the output code (speed, size, allocation count, etc) over the niceness of the codegen. We've also _completely_ disregarded speed of the template code (it's always going to be "fast enough"; you don't run this in a hot loop!). When there's a tension between readability and reuse, we've often favored readability. That means sometimes text is outright duplicated, if it seemed like extracting it would make it harder to read the template. Here's what we've ended up with: - There are mixins from the `node/mixins` package, which save some amount of code and standardize some behaviors. - These end up in the final result code. - It doesn't save *much* code, and they generally don't save *any* binary size (because it all gets inlined). - The consistency is the main virtue of these. - These are used mainly for error handling (specifically, returning of errors for methods called on nodes that have the wrong kind for that method). - There are mixins from the `schema/gen/go/mixins` package. - These are in the template code only -- they don't show up in the final result code. - These attempt to make it easier to create new 'generator' types. (There's a 'generator' type for each kind-plus-representation.) - They only attempt to take away some boilerplate, and you don't _have_ to use them. - There are functions in the template funcmap. - ... not many of them, though. - There's the idea of using associated templates (templates that are invoked by other templates). - There's currently none of this in use. Might it be helpful? Maybe. - There are functions which apply well-known templates to a generator. - These compose at the golang level, so it's easy to have the compiler check that they're all in order without running them (unlike templates, which have to be exercised in order to detect even basic problems like "does this named template exist"). - Many of these assume some pattern of methods on the generator objects. (Not of all these are super well documented.) - Generators usually call out to one or more of these from within the methods that their interface requires them to have. - The generator types themselves are usually split into two parts: the mechanisms for type-plus-repr, and just mechanisms for the type. - The mechanisms for the type alone aren't actually a full generator. The type-plus-repr thing just embeds the type-level semantics in itself. *Mostly*, it's general aim has been to keep relatively close to the structure of the code being generated. When reading a generator, one generally has to do *zero* or *one* jump-to-definition in order to see the fulltext of a template -- no more than that. (And so far, all those jump-to-definition lookups are on _go code_, not inside the template -- so an IDE can help you.) By example: if there are things which turn out common between _representation kinds_, those will probably end up in a function containing a well-known template, and that will end up being called from the generator type in one of the functions its required to have per its interface contract. This all probably has plenty of room for improvement! But know you know the reasoning that got things to be in the shape they are. ================================================ FILE: schema/gen/go/HACKME_testing.md ================================================ Testing Generated Code ====================== Getting nice testing scenarios for generated code is tricky. There are three major phases of testing we can do: 1. unit tests on the codegen tooling itself 2. tests that the emitted generated code can compile 3. behavioral tests on the emitted generated code Groups 1 and 2 are pretty easy (and we don't have a lot of group 1, because group 2 covers it pretty nicely). Group 3, however, is tricky. The rest of the document will talk more about this kind of testing. Behavioral Tests ---------------- ### Behavioral tests are run via plugins This package does some fancy footwork with the golang `plugin` system and `go build -buildmode=plugin` in order to compile and load the generated code into the same memory as the test process, thereby letting us do behavioral tests on the gen'd code quite seamlessly. This does have some downsides -- namely, the `plugin` system requires the use of `cgo`. This means you'll need more things installed on your host system than just the go toolchain -- you might need `gcc` and friends too. The `plugin` system also (at time of writing) doesn't support windows. You can skip the behavioral tests if this is a problem: see the next section. ### Skipping the behavioral tests You can skip behavioral tests by adding a build tag of `"skipgenbehavtests"`. They'll also be automatically skipped if you're running in `"!cgo"` mode -- however, the `go` tools don't automatically set `"!cgo"` just because it doesn't have enough tools, so you'll still need to be explicit about this. The ability of the generated code to be compiled will still be checked, even if the behavioral tests are skipped. You can grep through your test output for "bhvtest=skip" to see at-a-glance if the behavioral tests are being skipped. ### Plugins don't jive with race detection Long story short: if running tests with the race detector, skip the gen tests. Any `go test -race` is going to need `go test -race -tags 'skipgenbehavtests'`. The go plugin system requires the plugin and host process have the same "runtime". The way '-race' works makes for an effectively distinct runtime. ### Alternatives to plugins Are there other ways this could be done? Well, surely. There always are. Invoking 'go test' as a subprocess is usually central to alternative ideas, but this has several downsides I haven't yet figured out how to counter: - it tends to result in very difficult-to-wrangle output; - it ends up imposing a lot of constraints on file organization, which in turn makes writing tests into a very high-friction endeavour; - passing down flags to the test process (e.g., '-v' and friends) begins to require additional infrastructure; - some flags such as '-run' are even yet more difficult to pass down usefully; - every single behavioral test has to have an exported top-level function, making some things that are trivial with closures now difficult... - You get the idea. You can see some attempts around commit 79de0e26469f0d2899c813a2c70d921fe5946f23 and its halfdozen or so parents; remarks can be found in the git commit messages. There are probably yet more variations in ways files and functions could be reorganized, particularly to minimize the downsides of the file and package splitting requirements, but if you're looking at this scenario and wanting to propose one... Do read those commits to avoid getting into a Chesterton's Fence situation, and kindly try it before proposing it. Not all of the hurdles are immediately obvious. ### Plugins are only used in testing This might not need a clarification statement, but just in case it does: **plugins** (and by extension, cgo) **are not necessary** for **doing** codegen nor for **using** the resulting generated code. They are _only_ used for our testing of the codegen tooling (and specifically, at that, for the behavioral tests). We would not foist a dependency like cgo on codegen users. ================================================ FILE: schema/gen/go/HACKME_tradeoffs.md ================================================ tradeoffs and design decisions in codegen ========================================= In creating codegen for IPLD, as with any piece of software, there are design decisions to be made, and tradeoffs to be considered. Some of these decisions and tradeoffs are particularly interesting in IPLD codegen because they're: - significantly different resolutions and answers than the same decisions for non-codegen Node implementations - able to make significantly different choices, expanding the decision space dimentionality, since they have more information before compile-time - can reach higher upper bounds of performance, due to that pre-compile-time foothold - have correspondingly less flexibility in many ways because of the same. values we can balance --------------------- Let's enumerate the things we can balance (and give them some short reference codes): - AS: assembly/binary/final-shipping size, in bytes - BM: builder memory, in bytes, used as long as a NodeBuilder is in use - SP: execution speed, in seconds, especially of NodeBuilder in deserialization use - AC: allocations, in count, needed for operations (though in truth, this is just a proxy for SP due to its outsized impact there) - ERG: ergonomics, as an ineffable, ellusive-to-measurement sort of vibe of the thing, and how well it self-explains use and deters erroneous application - GLOC: generated lines of code, as a line count or in bytes, of interest because it may be of noticeable cost in version control weight This list is in particular regarding concerns that come to light in considering performant deserialization operations... however, it's fairly representative of general use as well: traversals and serialization are generally easier situations to handle (they essentially get to skip the "BM" term); and while different operations might encounter different scalars for how much these different values affect them, as we'll see in the prioritization coming up in the next section... that turns out not to matter for our priorities. We can also other code which knows it's addressing generated code can use special methods, which means we can in a way disregard its effect on this ordering (mostly). Side note: though "AC" is *mostly* just a proxy for SP, AC can also count on its own in *addition* to SP because it increases the *variance* in SP. (But we don't often regard this; it's a pretty fine detail, and the goal is "minimize" either way.) prioritization of those values ------------------------------ The designs here emerge from `SP > BM > AS`. More fully: `SP|AC > BM > ERG > AS > GLOC`. In other words: speed is the overwhelming priority; thereafter, we'd like to conserve memory (but will readily sell it for speed); ergonomics takes a side seat to both of these (the intention being that we can add 'porcelain' layers separately later); assembly size is a concern but fourth fiddle (if this is your dominant concern, you may not want to use codegen, or may want a different library implementation that aims at the same specs); and generated code size is a concern but we'll trade it away for any of the other priorities (because it's a cost that doesn't end up affecting final users of products built with this system!). (Some caveats: it's still possible to consider it a red flag if ratios on these get wild. For example if BM gets > 2x, it's questionable; and at some point we could imagine saying that AS has really gotten out of hand.) (BM also has some special conditions such that if it increases on recursive kinds, but not on scalars, we regard that as roughly half price, because generally most of a tree is leaves. (As it happens, though, this has turned out not to change any results much.)) "Ergonomics" remains a tricky to account for. It's true that when push comes to shove, speed and memory economy win. But it's not at all single-dimentional; and with codegen, there are many options which set a higher bar for all three concerns at the same time. (In contrast, there's a stark upper limit to the ergonomic possbilities for non-codegen no-schema handling of data -- code handling the data model has the limits that its monomorphized approach imposes on it, and there's little that can be done to avoid or improve upon that.) ================================================ FILE: schema/gen/go/HACKME_wip.md ================================================ ### absent values The handling of absent values is still not consistent. Currently: - reading (via accessors or iterators) yields `datamodel.Absent` values for absent fields - putting those datamodel.Absent values via NodeAssembler.AssignNode will result in `ErrWrongKind`. - *the recursive copies embedded in AssignNode methods don't handle absents either*. The first two are defensible and consistent (if not necessarily ergonomic). The third is downright a bug, and needs to be fixed. How we fix it is not entirely settled. - Option 1: keep the hostility to absent assignment - Option 2: *require* explicit absent assignment - Option 3: become indifferent to absent assignment when it's valid - Option 4: don't yield values that are absent during iteration at all Option 3 seems the most preferable (and least user-hostile). (Options 1 and 2 create work for end users; Option 4 has questionable logical consistency.) Updating the codegen to do Option 3 needs some work, though. It's likely that the way to go about this would involve adding two more valid bit states to the extended schema.Maybe values: one for allowAbsent (similar to the existing allowNull), and another for both (for "nullable optional" fields). Every NodeAssembler would then have to support that, just as they each support allowNull now. I think the above design is valid, but it's not implemented nor tested yet. ### AssignNode optimality The AssignNode methods we generate currently do pretty blithe things with large structures: they iterate over the given node, and hurl entries into the assembler's AssignKey and AssignValue methods. This isn't always optimal. For any structure that is more efficient when fed info in an ideal order, we might want to take account of that. For example, unions with representation mode "inline" are a stellar example of this: if the discriminant key comes first, they can work *much, much* more efficiently. By contrast, if the discriminant key shows up late in the object, it is necessary to have buffered *all the other* data, then backtrack to handle it once the discriminant is found and parsed. At best, this probably means iterating once, plucking out the discriminant entry, and then *getting a new iterator* that starts from the beginning (which shifts the buffer problem to the Node we're consuming data from). Even more irritatingly: since NodeAssembler has to accept entries in any order if it is to accept information streamingly from codecs, the NodeAssembler *also* has to be ready to do the buffering work... TODO ouch what are the ValueAssembler going to yield for dealing with children? TODO we have to hand out dummy ValueAssembler types that buffer... a crazy amount of stuff. (Reinvent refmt.Tok?? argh.) cannot avoid??? TODO this means where errors arise from will be nuts: you can't say if anything is wrong until you figure out the discriminant. then we replay everything? your errors for deeper stuff will appear... uh... midway, from a random AssembleValue finishing that happens to be for the discriminant. that is not pleasant. ... let's leave that thought aside: suffice to say, some assemblers are *really* not happy or performant if they have to accept things in unpleasant orderings. So. We should flip all this on its head. The AssignNode methods should lean in on the knowledge they have about the structure they're building, and assume that the Node we're copying content from supports random access: pluck the fields that we care most about out first with direct lookups, and only use iteration to cover the remaining data that the new structure doesn't care about the ordering of. Perhaps this only matters for certain styles of unions. ### sidenote about codec interfaces Perhaps we should get used to the idea of codec packages offering two styles of methods: - `UnmarshalIntoAssembler(io.Reader, datamodel.NodeAssembler) error` - this is for when you have opinions about what kind of in-memory format should be used - `Unmarshal(io.Reader) (datamodel.Node, error)` - this is for when you want to let the codec pick. We might actually end up preferring the latter in a fair number of cases. Looking at this inline union ordering situation described above: the best path through that (other than saying "don't fking use inline unions, and if you do, put the discriminant in the first fking entry or gtfo") would probably be to do a cbor (or whatever) unmarshal that produces the half-deserialized skip-list nodes (which are specialized to the cbor format rather than general purpose, but we want that in this story)... and those can then claim to do random access, thereby letting them take on the "buffering". This approach would let the serialization-specialized nodes take on the work, rather than forcing the union's NodeAssembler to do buffer at a higher level... which is good because doing that buffering in a structured way at a higher level is actually more work and causes more memory fragmentation and allocations. Whew. I have not worked out what this implies for multicodecs or other muxes that do compositions of codecs. ### enums of union keys It's extremely common to have an enum that is the discrimant values of a union. We should make a schema syntax for that. We tend to generate such an enum in codegen anyway, for various purposes. Might as well let people name it outright too, if they have the slightest desire to do so. (Doesn't apply to kinded unions.) ### can reset methods be replaced with duff's device? Yes. Well, sort of. Okay, no. It's close! Assemblers were all written such that their zero values are ready to go. However, there's a couple of situations where you *wouldn't* want to blithely zero everything: for example, if an assembler has to do some allocations, but they're reusable, you wouldn't want to turn those other objects into garbage by zeroing the pointer to them. See the following section about new-alloc child assemblers for an example of this. ### what's up with new-alloc child assemblers? Mostly, child assemblers are embedded in the assembler for the type that contains them; this is part of our allocation amortization strategy and important to performance. However, it doesn't always apply: Sometimes we *need* independently allocated assemblers, even when they're child assemblers: recursive structures need this (otherwise, how big would the slab be? infinite? no; halt). Sometimes we also just *want* them, somewhat more mildly: if a union is one of several things, and some of them are uncommonly used but huuuuge, then maybe we'd rather allocate the child assemblers individually on demand rather than pay a large resident memory cost to embed all the possibilities. There's a couple things to think about with these: - resetting assemblers with a duff's device strategy wouldn't recursively reset these; it would just orphan them. While possibly leaving them pointed into some parts of memory in the parent slab ('cm' in particular comes to mind). This could be a significant correctness issue. - But who's responsibility is it to "safe" this? Nilling 'w' proactively should also make this pretty innocuous, as one option (but we don't currently do this). - if the parent assembler is being used in some highly reusable situation (e.g. it's a list value or map value), is the parent able to hold onto and re-use the child assembler? We probably usually still want to do this, even if it's in a separate piece of heap. - For unions, there's a question of if we should hold onto each child assembler, or just the most recent; that's a choice we could make and tune. If the answer is "most recent only", we could even crank down the resident size by use of more interfaces instead of concrete types (at the cost of some other runtime performance debufs, most likely). We've chosen to discard the possibility of duff's device as an assembler resetting implementation. As a result, we don't have to do proactive 'w'-nil'ing in places we might otherwise have to. And union assemblers hold on to all child assembler types they've ever needed. ================================================ FILE: schema/gen/go/README.md ================================================ gengo ===== This package contains a codegenerator for emitting Golang source code for datastructures based on IPLD Schemas. It is reasonably complete. See the [feature table](#completeness) below for details. There is a CLI tool which can be used to run this generator. See https://github.com/ipld/go-ipldtool ! Some features may still requiring writing code to fully configure them. (PRs, here or in the go-ipldtool repo, welcome.) See [README_behaviors](README_behaviors.md) for notes about the behaviors of the code output by the generator. Check out the [HACKME](HACKME.md) document for more info about the internals, how they're organized, and how to hack on this package. aims ---- `gengo` aims to: - generate native Golang code - that faithfully represents the data structuring specified by an IPLD Schema, - operating efficiently, both in speed (both creating and inspecting) and memory compactness; - producing a better type system for Golang (we've got unions and enums!) - that is both powerful and generic (when you need it) - and minimalist (when you don't), - with immutable data structures, - good validation primitives and type-supported safety systems, - and is friendly to embellishments of other hand-written Golang code. Some of these aims should be satisfied. Some are still a stretch ;) (we definitely don't have "minimalist" outputs, yet. Making this reachable by tuning is a goal, however!) completeness ------------ Legend: - `✔` - supported! - `✘` - not currently supported. - `⚠` - not currently supported -- and might not be obvious; be careful. - `-` - is not applicable - `?` - feature definition needed! (applies to many of the "native extras" rows -- often there's partial features, but also room for more.) - ` ` - table is not finished, please refer to the code and help fix the table :) | feature | accessors | builders | |:---------------------------------|:---------:|:--------:| | structs | ... | ... | | ... type level | ✔ | ✔ | | ... native extras | ? | ? | | ... map representation | ✔ | ✔ | | ... ... including optional | ✔ | ✔ | | ... ... including renames | ✔ | ✔ | | ... ... including implicits | ⚠ | ⚠ | | ... tuple representation | ✔ | ✔ | | ... ... including optional | ✔ | ✔ | | ... ... including renames | - | - | | ... ... including implicits | ⚠ | ⚠ | | ... stringjoin representation | ✔ | ✔ | | ... ... including optional | - | - | | ... ... including renames | - | - | | ... ... including implicits | - | - | | ... stringpairs representation | ✘ | ✘ | | ... ... including optional | | | | ... ... including renames | | | | ... ... including implicits | | | | ... listpairs representation | ✘ | ✘ | | ... ... including optional | | | | ... ... including renames | | | | ... ... including implicits | | | | feature | accessors | builders | |:---------------------------------|:---------:|:--------:| | lists | ... | ... | | ... type level | ✔ | ✔ | | ... native extras | ? | ? | | ... list representation | ✔ | ✔ | | feature | accessors | builders | |:---------------------------------|:---------:|:--------:| | maps | ... | ... | | ... type level | ✔ | ✔ | | ... native extras | ? | ? | | ... map representation | ✔ | ✔ | | ... stringpairs representation | ✘ | ✘ | | ... listpairs representation | ✘ | ✘ | | feature | accessors | builders | |:---------------------------------|:---------:|:--------:| | unions | ... | ... | | ... type level | ✔ | ✔ | | ... keyed representation | ✔ | ✔ | | ... envelope representation | ✘ | ✘ | | ... kinded representation | ✔ | ✔ | | ... inline representation | ✘ | ✘ | | ... stringprefix representation | ✔ | ✔ | | ... byteprefix representation | ✘ | ✘ | | feature | accessors | builders | |:---------------------------------|:---------:|:--------:| | strings | ✔ | ✔ | | bytes | ✔ | ✔ | | ints | ✔ | ✔ | | floats | ✔ | ✔ | | bools | ✔ | ✔ | | links | ✔ | ✔ | | feature | accessors | builders | |:---------------------------------|:---------:|:--------:| | enums | ... | ... | | ... type level | ✘ | ✘ | | ... string representation | ✘ | ✘ | | ... int representation | ✘ | ✘ | ================================================ FILE: schema/gen/go/README_behaviors.md ================================================ Behaviors of Generated Types ============================ Types generated by `gogen` obey the rules that already exist for the IPLD Data Model, and the rules that exist for Schema typed nodes. There are some further details of behavior that might be worth noting, though. Some of these details aren't necessarily programmer-friendly(!), and arise from prioritizing performance over other concerns; so especially watch out for these as you develop against the code output by this tool. ### retaining assemblers beyond their intended lifespan is not guaranteed to be safe There is no promise of nice-to-read errors if you over-hold child assemblers beyond their valid lifespan. `NodeAssembler` values should not be retained for any longer than they're actively in use. - We **do** care about making things fail hard and fast rather than potentially leak inappropriate mutability. - We do **not** care about making these errors pretty (it's high cost to do so, and code that hits this path is almost certainly statically (and hopefully fairly obviously) wrong). In some cases it may also be the case that a `NodeAssembler` that populates the internals of some large structure may become invalid (because of state transitions that block inappropriate mutability), and yet become possible to use again later (because of coincidences of how we reuse memory internally for efficiency reasons). We don't reliably raise errors in some of these situations, for efficiency reasons, but wish we could. Users of the generated code should not rely on these behaviors: it results in difficult-to-read code in any case, and such internal details should not be considered part of the intended public API (e.g., such details may be subject to change without notice). ### absent values Iterating a type-level node with optional fields will yield the field key and the `datamodel.Absent` constant as a value. Getting a such a field will also yield the `datamodel.Absent` constant as a value, and will not return a "not found" error. Attempting to *assign* an `datamodel.Absent` value, however -- via the `NodeAssembler.AssignNode` function (none of the other function signatures permit expressing this) -- will result in an `datamodel.ErrWrongKind` error. // Seealso some unresolved todos in the [HACKME_wip](HACKME_wip.md) document regarding how absent values are handled. ================================================ FILE: schema/gen/go/README_wishes.md ================================================ wishes and dreams of other things that could be =============================================== ### minimal gen for regular code (In other short words: gen without the monomorphization.) It'd be neat to have a mode that generates all public fields (shrugging on immutability), and can also toggle off generating all the data model Node interface satisfaction methods. This would let us use IPLD Schemas to define types (including enums and unions) and get Golang code out, and easily use it in programs even where we don't particularly care about the Node interface. (E.g., suppose we want an enum or a union type -- which Go doesn't naturally have -- but aren't going to use it for serialization or anything else. We've already got a lot of codegen here; why not make it help there too?) It's unclear if we can load that much into this gen project, or if it'd be easier to make another one for this. The output code would definitely be substantially structurally different; and it would also tend to be useless/silly to generate different parts of a type system in each of the different modes, so it's pretty likely that any practical user story would involve different process invocations to use them anyway. ================================================ FILE: schema/gen/go/_test/.gitignore ================================================ /* ================================================ FILE: schema/gen/go/adjunctCfg.go ================================================ package gengo import ( "fmt" "reflect" "strings" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/schema" ) // This entire file is placeholder-quality implementations. // // The AdjunctCfg struct should be replaced with an IPLD Schema-specified thing! // The values in the unionMemlayout field should be an enum; // etcetera! type FieldTuple struct { TypeName schema.TypeName FieldName string } type AdjunctCfg struct { typeSymbolOverrides map[schema.TypeName]string FieldSymbolLowerOverrides map[FieldTuple]string fieldSymbolUpperOverrides map[FieldTuple]string maybeUsesPtr map[schema.TypeName]bool // absent uses a heuristic CfgUnionMemlayout map[schema.TypeName]string // "embedAll"|"interface"; maybe more options later, unclear for now. // ... some of these fields have sprouted messy name prefixes so they don't collide with their matching method names. // this structure has reached the critical threshold where it due to be cleaned up and taken seriously. // note: PkgName doesn't appear in here, because it's... // not adjunct data. it's a generation invocation parameter. // ... this might not hold up in the future though. // There are unanswered questions about how (also, tbf, *if*) we'll handle generation of multiple packages which use each other's types. } // TypeSymbol returns the symbol for a type; // by default, it's the same string as its name in the schema, // but it can be overridden. // // This is the base, unembellished symbol. // It's frequently augmented: // prefixing an underscore to make it unexported; // suffixing "__Something" to make the name of a supporting type; // etc. // (Most such augmentations are not configurable.) func (cfg *AdjunctCfg) TypeSymbol(t schema.Type) string { if x, ok := cfg.typeSymbolOverrides[t.Name()]; ok { return x } return string(t.Name()) // presumed already upper } func (cfg *AdjunctCfg) FieldSymbolLower(f schema.StructField) string { if x, ok := cfg.FieldSymbolLowerOverrides[FieldTuple{f.Parent().Name(), f.Name()}]; ok { return x } return f.Name() // presumed already lower } func (cfg *AdjunctCfg) FieldSymbolUpper(f schema.StructField) string { if x, ok := cfg.fieldSymbolUpperOverrides[FieldTuple{f.Type().Name(), f.Name()}]; ok { return x } return strings.Title(f.Name()) //lint:ignore SA1019 cases.Title doesn't work for this } // Comments returns a bool for whether comments should be included in gen output or not. func (cfg *AdjunctCfg) Comments() bool { return true // FUTURE: okay, maybe this should be configurable :) } func (cfg *AdjunctCfg) MaybeUsesPtr(t schema.Type) bool { if x, ok := cfg.maybeUsesPtr[t.Name()]; ok { return x } // As a simple heuristic, // check how large the Go representation of this type will be. // If it weighs little, we estimate that a pointer is not worthwhile, // as storing the data directly will barely take more memory. // Plus, the resulting code will be shorter and have fewer branches. return sizeOfSchemaType(t) > sizeSmallEnoughForInlining } var ( // The cutoff for "weighs little" is any size up to this number. // It's hasn't been measured with any benchmarks or stats just yet. // It's possible that, with those, it might increase in the future. // Intuitively, any type 4x the size of a pointer is fine to inline. // Adding a pointer will already add 1x overhead, anyway. sizeSmallEnoughForInlining = 4 * reflect.TypeOf(new(int)).Size() sizeOfTypeKind [128]uintptr ) func init() { // Uncomment for debugging. // fmt.Fprintf(os.Stderr, "sizeOf(small): %d (4x pointer size)\n", sizeSmallEnoughForInlining) // Get the basic node sizes via basicnode. for _, tk := range []struct { typeKind schema.TypeKind prototype datamodel.NodePrototype }{ {schema.TypeKind_Bool, basicnode.Prototype.Bool}, {schema.TypeKind_Int, basicnode.Prototype.Int}, {schema.TypeKind_Float, basicnode.Prototype.Float}, {schema.TypeKind_String, basicnode.Prototype.String}, {schema.TypeKind_Bytes, basicnode.Prototype.Bytes}, {schema.TypeKind_List, basicnode.Prototype.List}, {schema.TypeKind_Map, basicnode.Prototype.Map}, {schema.TypeKind_Link, basicnode.Prototype.Link}, } { nb := tk.prototype.NewBuilder() switch tk.typeKind { case schema.TypeKind_List: am, err := nb.BeginList(0) if err != nil { panic(err) } if err := am.Finish(); err != nil { panic(err) } case schema.TypeKind_Map: am, err := nb.BeginMap(0) if err != nil { panic(err) } if err := am.Finish(); err != nil { panic(err) } } // Note that the Node interface has a pointer underneath, // so we use Elem to reach the underlying type. size := reflect.TypeOf(nb.Build()).Elem().Size() sizeOfTypeKind[tk.typeKind] = size // Uncomment for debugging. // fmt.Fprintf(os.Stderr, "sizeOf(%s): %d\n", tk.typeKind, size) } } // sizeOfSchemaType returns the size of a schema type, // relative to the size of a pointer in native Go. // // For example, TypeInt and TypeMap returns 1, but TypeList returns 3, as a // slice in Go has a pointer and two integers for length and capacity. // Any basic type smaller than a pointer, such as TypeBool, returns 1. func sizeOfSchemaType(t schema.Type) uintptr { kind := t.TypeKind() // If this TypeKind is represented by the basicnode package, // we statically know its size and we can return here. if size := sizeOfTypeKind[kind]; size > 0 { return size } // TODO: handle typekinds like structs, unions, etc. // For now, return a large size to fall back to using a pointer. return 100 * sizeSmallEnoughForInlining } // UnionMemlayout returns a plain string at present; // there's a case-switch in the templates that processes it. // We validate that it's a known string when this method is called. // This should probably be improved in type-safety, // and validated more aggressively up front when adjcfg is loaded. func (cfg *AdjunctCfg) UnionMemlayout(t schema.Type) string { if t.TypeKind() != schema.TypeKind_Union { panic(fmt.Errorf("%s is not a union", t.Name())) } v, ok := cfg.CfgUnionMemlayout[t.Name()] if !ok { return "embedAll" } switch v { case "embedAll", "interface": return v default: panic(fmt.Errorf("invalid config: unionMemlayout values must be either \"embedAll\" or \"interface\", not %q", v)) } } ================================================ FILE: schema/gen/go/externUtil.go ================================================ package gengo import ( "go/ast" "go/parser" "go/token" path "path/filepath" "strings" ) // getExternTypes provides a mapping of all types defined in the destination package. // It is used by generate to not duplicate defined types to allow overriding of types. func getExternTypes(pth string) (map[string]struct{}, error) { set := token.NewFileSet() packs, err := parser.ParseDir(set, pth, nil, 0) if err != nil { return nil, err } types := make(map[string]struct{}) for _, pack := range packs { for fname, f := range pack.Files { if strings.HasPrefix(path.Base(fname), "ipldsch_") { continue } for _, d := range f.Decls { if t, isType := d.(*ast.GenDecl); isType { if t.Tok == token.TYPE { for _, s := range t.Specs { ts := s.(*ast.TypeSpec) types[ts.Name.Name] = struct{}{} } } } } } } return types, nil } ================================================ FILE: schema/gen/go/genBool.go ================================================ package gengo import ( "io" "github.com/ipld/go-ipld-prime/schema" "github.com/ipld/go-ipld-prime/schema/gen/go/mixins" ) type boolGenerator struct { AdjCfg *AdjunctCfg mixins.BoolTraits PkgName string Type *schema.TypeBool } func (boolGenerator) IsRepr() bool { return false } // hint used in some generalized templates. // --- native content and specializations ---> func (g boolGenerator) EmitNativeType(w io.Writer) { emitNativeType_scalar(w, g.AdjCfg, g) } func (g boolGenerator) EmitNativeAccessors(w io.Writer) { emitNativeAccessors_scalar(w, g.AdjCfg, g) } func (g boolGenerator) EmitNativeBuilder(w io.Writer) { emitNativeBuilder_scalar(w, g.AdjCfg, g) } func (g boolGenerator) EmitNativeMaybe(w io.Writer) { emitNativeMaybe(w, g.AdjCfg, g) } // --- type info ---> func (g boolGenerator) EmitTypeConst(w io.Writer) { doTemplate(` // TODO EmitTypeConst `, w, g.AdjCfg, g) } // --- TypedNode interface satisfaction ---> func (g boolGenerator) EmitTypedNodeMethodType(w io.Writer) { doTemplate(` func ({{ .Type | TypeSymbol }}) Type() schema.Type { return nil /*TODO:typelit*/ } `, w, g.AdjCfg, g) } func (g boolGenerator) EmitTypedNodeMethodRepresentation(w io.Writer) { emitTypicalTypedNodeMethodRepresentation(w, g.AdjCfg, g) } // --- Node interface satisfaction ---> func (g boolGenerator) EmitNodeType(w io.Writer) { // No additional types needed. Methods all attach to the native type. } func (g boolGenerator) EmitNodeTypeAssertions(w io.Writer) { emitNodeTypeAssertions_typical(w, g.AdjCfg, g) } func (g boolGenerator) EmitNodeMethodAsBool(w io.Writer) { emitNodeMethodAsKind_scalar(w, g.AdjCfg, g) } func (g boolGenerator) EmitNodeMethodPrototype(w io.Writer) { emitNodeMethodPrototype_typical(w, g.AdjCfg, g) } func (g boolGenerator) EmitNodePrototypeType(w io.Writer) { emitNodePrototypeType_typical(w, g.AdjCfg, g) } // --- NodeBuilder and NodeAssembler ---> func (g boolGenerator) GetNodeBuilderGenerator() NodeBuilderGenerator { return boolBuilderGenerator{ g.AdjCfg, mixins.BoolAssemblerTraits{ PkgName: g.PkgName, TypeName: g.TypeName, AppliedPrefix: "_" + g.AdjCfg.TypeSymbol(g.Type) + "__", }, g.PkgName, g.Type, } } type boolBuilderGenerator struct { AdjCfg *AdjunctCfg mixins.BoolAssemblerTraits PkgName string Type *schema.TypeBool } func (boolBuilderGenerator) IsRepr() bool { return false } // hint used in some generalized templates. func (g boolBuilderGenerator) EmitNodeBuilderType(w io.Writer) { emitEmitNodeBuilderType_typical(w, g.AdjCfg, g) } func (g boolBuilderGenerator) EmitNodeBuilderMethods(w io.Writer) { emitNodeBuilderMethods_typical(w, g.AdjCfg, g) } func (g boolBuilderGenerator) EmitNodeAssemblerType(w io.Writer) { emitNodeAssemblerType_scalar(w, g.AdjCfg, g) } func (g boolBuilderGenerator) EmitNodeAssemblerMethodAssignNull(w io.Writer) { emitNodeAssemblerMethodAssignNull_scalar(w, g.AdjCfg, g) } func (g boolBuilderGenerator) EmitNodeAssemblerMethodAssignBool(w io.Writer) { emitNodeAssemblerMethodAssignKind_scalar(w, g.AdjCfg, g) } func (g boolBuilderGenerator) EmitNodeAssemblerMethodAssignNode(w io.Writer) { emitNodeAssemblerMethodAssignNode_scalar(w, g.AdjCfg, g) } func (g boolBuilderGenerator) EmitNodeAssemblerOtherBits(w io.Writer) { // Nothing needed here for bool kinds. } ================================================ FILE: schema/gen/go/genBoolReprBool.go ================================================ package gengo import ( "io" "github.com/ipld/go-ipld-prime/schema" "github.com/ipld/go-ipld-prime/schema/gen/go/mixins" ) var _ TypeGenerator = &boolReprBoolGenerator{} func NewBoolReprBoolGenerator(pkgName string, typ *schema.TypeBool, adjCfg *AdjunctCfg) TypeGenerator { return boolReprBoolGenerator{ boolGenerator{ adjCfg, mixins.BoolTraits{ PkgName: pkgName, TypeName: string(typ.Name()), TypeSymbol: adjCfg.TypeSymbol(typ), }, pkgName, typ, }, } } type boolReprBoolGenerator struct { boolGenerator } func (g boolReprBoolGenerator) GetRepresentationNodeGen() NodeGenerator { return boolReprBoolReprGenerator{ g.AdjCfg, g.Type, } } type boolReprBoolReprGenerator struct { AdjCfg *AdjunctCfg Type *schema.TypeBool } func (g boolReprBoolReprGenerator) EmitNodeType(w io.Writer) { // Since this is a "natural" representation... there's just a type alias here. // No new functions are necessary. doTemplate(` type _{{ .Type | TypeSymbol }}__Repr = _{{ .Type | TypeSymbol }} `, w, g.AdjCfg, g) } func (g boolReprBoolReprGenerator) EmitNodeTypeAssertions(w io.Writer) { doTemplate(` var _ datamodel.Node = &_{{ .Type | TypeSymbol }}__Repr{} `, w, g.AdjCfg, g) } func (boolReprBoolReprGenerator) EmitNodeMethodKind(io.Writer) {} func (boolReprBoolReprGenerator) EmitNodeMethodLookupByString(io.Writer) {} func (boolReprBoolReprGenerator) EmitNodeMethodLookupByNode(io.Writer) {} func (boolReprBoolReprGenerator) EmitNodeMethodLookupByIndex(io.Writer) {} func (boolReprBoolReprGenerator) EmitNodeMethodLookupBySegment(io.Writer) {} func (boolReprBoolReprGenerator) EmitNodeMethodMapIterator(io.Writer) {} func (boolReprBoolReprGenerator) EmitNodeMethodListIterator(io.Writer) {} func (boolReprBoolReprGenerator) EmitNodeMethodLength(io.Writer) {} func (boolReprBoolReprGenerator) EmitNodeMethodIsAbsent(io.Writer) {} func (boolReprBoolReprGenerator) EmitNodeMethodIsNull(io.Writer) {} func (boolReprBoolReprGenerator) EmitNodeMethodAsBool(io.Writer) {} func (boolReprBoolReprGenerator) EmitNodeMethodAsInt(io.Writer) {} func (boolReprBoolReprGenerator) EmitNodeMethodAsFloat(io.Writer) {} func (boolReprBoolReprGenerator) EmitNodeMethodAsString(io.Writer) {} func (boolReprBoolReprGenerator) EmitNodeMethodAsBytes(io.Writer) {} func (boolReprBoolReprGenerator) EmitNodeMethodAsLink(io.Writer) {} func (boolReprBoolReprGenerator) EmitNodeMethodPrototype(io.Writer) {} func (g boolReprBoolReprGenerator) EmitNodePrototypeType(w io.Writer) { // Since this is a "natural" representation... there's just a type alias here. // No new functions are necessary. doTemplate(` type _{{ .Type | TypeSymbol }}__ReprPrototype = _{{ .Type | TypeSymbol }}__Prototype `, w, g.AdjCfg, g) } func (g boolReprBoolReprGenerator) GetNodeBuilderGenerator() NodeBuilderGenerator { return boolReprBoolReprBuilderGenerator(g) } type boolReprBoolReprBuilderGenerator struct { AdjCfg *AdjunctCfg Type *schema.TypeBool } func (boolReprBoolReprBuilderGenerator) EmitNodeBuilderType(io.Writer) {} func (boolReprBoolReprBuilderGenerator) EmitNodeBuilderMethods(io.Writer) {} func (g boolReprBoolReprBuilderGenerator) EmitNodeAssemblerType(w io.Writer) { // Since this is a "natural" representation... there's just a type alias here. // No new functions are necessary. doTemplate(` type _{{ .Type | TypeSymbol }}__ReprAssembler = _{{ .Type | TypeSymbol }}__Assembler `, w, g.AdjCfg, g) } func (boolReprBoolReprBuilderGenerator) EmitNodeAssemblerMethodBeginMap(io.Writer) {} func (boolReprBoolReprBuilderGenerator) EmitNodeAssemblerMethodBeginList(io.Writer) {} func (boolReprBoolReprBuilderGenerator) EmitNodeAssemblerMethodAssignNull(io.Writer) {} func (boolReprBoolReprBuilderGenerator) EmitNodeAssemblerMethodAssignBool(io.Writer) {} func (boolReprBoolReprBuilderGenerator) EmitNodeAssemblerMethodAssignInt(io.Writer) {} func (boolReprBoolReprBuilderGenerator) EmitNodeAssemblerMethodAssignFloat(io.Writer) {} func (boolReprBoolReprBuilderGenerator) EmitNodeAssemblerMethodAssignString(io.Writer) {} func (boolReprBoolReprBuilderGenerator) EmitNodeAssemblerMethodAssignBytes(io.Writer) {} func (boolReprBoolReprBuilderGenerator) EmitNodeAssemblerMethodAssignLink(io.Writer) {} func (boolReprBoolReprBuilderGenerator) EmitNodeAssemblerMethodAssignNode(io.Writer) {} func (boolReprBoolReprBuilderGenerator) EmitNodeAssemblerMethodPrototype(io.Writer) {} func (boolReprBoolReprBuilderGenerator) EmitNodeAssemblerOtherBits(io.Writer) {} ================================================ FILE: schema/gen/go/genBytes.go ================================================ package gengo import ( "io" "github.com/ipld/go-ipld-prime/schema" "github.com/ipld/go-ipld-prime/schema/gen/go/mixins" ) type bytesGenerator struct { AdjCfg *AdjunctCfg mixins.BytesTraits PkgName string Type *schema.TypeBytes } func (bytesGenerator) IsRepr() bool { return false } // hint used in some generalized templates. // --- native content and specializations ---> func (g bytesGenerator) EmitNativeType(w io.Writer) { emitNativeType_scalar(w, g.AdjCfg, g) } func (g bytesGenerator) EmitNativeAccessors(w io.Writer) { emitNativeAccessors_scalar(w, g.AdjCfg, g) } func (g bytesGenerator) EmitNativeBuilder(w io.Writer) { emitNativeBuilder_scalar(w, g.AdjCfg, g) } func (g bytesGenerator) EmitNativeMaybe(w io.Writer) { emitNativeMaybe(w, g.AdjCfg, g) } // --- type info ---> func (g bytesGenerator) EmitTypeConst(w io.Writer) { doTemplate(` // TODO EmitTypeConst `, w, g.AdjCfg, g) } // --- TypedNode interface satisfaction ---> func (g bytesGenerator) EmitTypedNodeMethodType(w io.Writer) { doTemplate(` func ({{ .Type | TypeSymbol }}) Type() schema.Type { return nil /*TODO:typelit*/ } `, w, g.AdjCfg, g) } func (g bytesGenerator) EmitTypedNodeMethodRepresentation(w io.Writer) { emitTypicalTypedNodeMethodRepresentation(w, g.AdjCfg, g) } // --- Node interface satisfaction ---> func (g bytesGenerator) EmitNodeType(w io.Writer) { // No additional types needed. Methods all attach to the native type. } func (g bytesGenerator) EmitNodeTypeAssertions(w io.Writer) { emitNodeTypeAssertions_typical(w, g.AdjCfg, g) } func (g bytesGenerator) EmitNodeMethodAsBytes(w io.Writer) { emitNodeMethodAsKind_scalar(w, g.AdjCfg, g) } func (g bytesGenerator) EmitNodeMethodPrototype(w io.Writer) { emitNodeMethodPrototype_typical(w, g.AdjCfg, g) } func (g bytesGenerator) EmitNodePrototypeType(w io.Writer) { emitNodePrototypeType_typical(w, g.AdjCfg, g) } // --- NodeBuilder and NodeAssembler ---> func (g bytesGenerator) GetNodeBuilderGenerator() NodeBuilderGenerator { return bytesBuilderGenerator{ g.AdjCfg, mixins.BytesAssemblerTraits{ PkgName: g.PkgName, TypeName: g.TypeName, AppliedPrefix: "_" + g.AdjCfg.TypeSymbol(g.Type) + "__", }, g.PkgName, g.Type, } } type bytesBuilderGenerator struct { AdjCfg *AdjunctCfg mixins.BytesAssemblerTraits PkgName string Type *schema.TypeBytes } func (bytesBuilderGenerator) IsRepr() bool { return false } // hint used in some generalized templates. func (g bytesBuilderGenerator) EmitNodeBuilderType(w io.Writer) { emitEmitNodeBuilderType_typical(w, g.AdjCfg, g) } func (g bytesBuilderGenerator) EmitNodeBuilderMethods(w io.Writer) { emitNodeBuilderMethods_typical(w, g.AdjCfg, g) } func (g bytesBuilderGenerator) EmitNodeAssemblerType(w io.Writer) { emitNodeAssemblerType_scalar(w, g.AdjCfg, g) } func (g bytesBuilderGenerator) EmitNodeAssemblerMethodAssignNull(w io.Writer) { emitNodeAssemblerMethodAssignNull_scalar(w, g.AdjCfg, g) } func (g bytesBuilderGenerator) EmitNodeAssemblerMethodAssignBytes(w io.Writer) { emitNodeAssemblerMethodAssignKind_scalar(w, g.AdjCfg, g) } func (g bytesBuilderGenerator) EmitNodeAssemblerMethodAssignNode(w io.Writer) { emitNodeAssemblerMethodAssignNode_scalar(w, g.AdjCfg, g) } func (g bytesBuilderGenerator) EmitNodeAssemblerOtherBits(w io.Writer) { // Nothing needed here for bytes kinds. } ================================================ FILE: schema/gen/go/genBytesReprBytes.go ================================================ package gengo import ( "io" "github.com/ipld/go-ipld-prime/schema" "github.com/ipld/go-ipld-prime/schema/gen/go/mixins" ) var _ TypeGenerator = &bytesReprBytesGenerator{} func NewBytesReprBytesGenerator(pkgName string, typ *schema.TypeBytes, adjCfg *AdjunctCfg) TypeGenerator { return bytesReprBytesGenerator{ bytesGenerator{ adjCfg, mixins.BytesTraits{ PkgName: pkgName, TypeName: string(typ.Name()), TypeSymbol: adjCfg.TypeSymbol(typ), }, pkgName, typ, }, } } type bytesReprBytesGenerator struct { bytesGenerator } func (g bytesReprBytesGenerator) GetRepresentationNodeGen() NodeGenerator { return bytesReprBytesReprGenerator{ g.AdjCfg, g.Type, } } type bytesReprBytesReprGenerator struct { AdjCfg *AdjunctCfg Type *schema.TypeBytes } func (g bytesReprBytesReprGenerator) EmitNodeType(w io.Writer) { // Since this is a "natural" representation... there's just a type alias here. // No new functions are necessary. doTemplate(` type _{{ .Type | TypeSymbol }}__Repr = _{{ .Type | TypeSymbol }} `, w, g.AdjCfg, g) } func (g bytesReprBytesReprGenerator) EmitNodeTypeAssertions(w io.Writer) { doTemplate(` var _ datamodel.Node = &_{{ .Type | TypeSymbol }}__Repr{} `, w, g.AdjCfg, g) } func (bytesReprBytesReprGenerator) EmitNodeMethodKind(io.Writer) {} func (bytesReprBytesReprGenerator) EmitNodeMethodLookupByString(io.Writer) {} func (bytesReprBytesReprGenerator) EmitNodeMethodLookupByNode(io.Writer) {} func (bytesReprBytesReprGenerator) EmitNodeMethodLookupByIndex(io.Writer) {} func (bytesReprBytesReprGenerator) EmitNodeMethodLookupBySegment(io.Writer) {} func (bytesReprBytesReprGenerator) EmitNodeMethodMapIterator(io.Writer) {} func (bytesReprBytesReprGenerator) EmitNodeMethodListIterator(io.Writer) {} func (bytesReprBytesReprGenerator) EmitNodeMethodLength(io.Writer) {} func (bytesReprBytesReprGenerator) EmitNodeMethodIsAbsent(io.Writer) {} func (bytesReprBytesReprGenerator) EmitNodeMethodIsNull(io.Writer) {} func (bytesReprBytesReprGenerator) EmitNodeMethodAsBool(io.Writer) {} func (bytesReprBytesReprGenerator) EmitNodeMethodAsInt(io.Writer) {} func (bytesReprBytesReprGenerator) EmitNodeMethodAsFloat(io.Writer) {} func (bytesReprBytesReprGenerator) EmitNodeMethodAsString(io.Writer) {} func (bytesReprBytesReprGenerator) EmitNodeMethodAsBytes(io.Writer) {} func (bytesReprBytesReprGenerator) EmitNodeMethodAsLink(io.Writer) {} func (bytesReprBytesReprGenerator) EmitNodeMethodPrototype(io.Writer) {} func (g bytesReprBytesReprGenerator) EmitNodePrototypeType(w io.Writer) { // Since this is a "natural" representation... there's just a type alias here. // No new functions are necessary. doTemplate(` type _{{ .Type | TypeSymbol }}__ReprPrototype = _{{ .Type | TypeSymbol }}__Prototype `, w, g.AdjCfg, g) } func (g bytesReprBytesReprGenerator) GetNodeBuilderGenerator() NodeBuilderGenerator { return bytesReprBytesReprBuilderGenerator(g) } type bytesReprBytesReprBuilderGenerator struct { AdjCfg *AdjunctCfg Type *schema.TypeBytes } func (bytesReprBytesReprBuilderGenerator) EmitNodeBuilderType(io.Writer) {} func (bytesReprBytesReprBuilderGenerator) EmitNodeBuilderMethods(io.Writer) {} func (g bytesReprBytesReprBuilderGenerator) EmitNodeAssemblerType(w io.Writer) { // Since this is a "natural" representation... there's just a type alias here. // No new functions are necessary. doTemplate(` type _{{ .Type | TypeSymbol }}__ReprAssembler = _{{ .Type | TypeSymbol }}__Assembler `, w, g.AdjCfg, g) } func (bytesReprBytesReprBuilderGenerator) EmitNodeAssemblerMethodBeginMap(io.Writer) {} func (bytesReprBytesReprBuilderGenerator) EmitNodeAssemblerMethodBeginList(io.Writer) {} func (bytesReprBytesReprBuilderGenerator) EmitNodeAssemblerMethodAssignNull(io.Writer) {} func (bytesReprBytesReprBuilderGenerator) EmitNodeAssemblerMethodAssignBool(io.Writer) {} func (bytesReprBytesReprBuilderGenerator) EmitNodeAssemblerMethodAssignInt(io.Writer) {} func (bytesReprBytesReprBuilderGenerator) EmitNodeAssemblerMethodAssignFloat(io.Writer) {} func (bytesReprBytesReprBuilderGenerator) EmitNodeAssemblerMethodAssignString(io.Writer) {} func (bytesReprBytesReprBuilderGenerator) EmitNodeAssemblerMethodAssignBytes(io.Writer) {} func (bytesReprBytesReprBuilderGenerator) EmitNodeAssemblerMethodAssignLink(io.Writer) {} func (bytesReprBytesReprBuilderGenerator) EmitNodeAssemblerMethodAssignNode(io.Writer) {} func (bytesReprBytesReprBuilderGenerator) EmitNodeAssemblerMethodPrototype(io.Writer) {} func (bytesReprBytesReprBuilderGenerator) EmitNodeAssemblerOtherBits(io.Writer) {} ================================================ FILE: schema/gen/go/genFloat.go ================================================ package gengo import ( "io" "github.com/ipld/go-ipld-prime/schema" "github.com/ipld/go-ipld-prime/schema/gen/go/mixins" ) type float64Generator struct { AdjCfg *AdjunctCfg mixins.FloatTraits PkgName string Type *schema.TypeFloat } func (float64Generator) IsRepr() bool { return false } // hint used in some generalized templates. // --- native content and specializations ---> func (g float64Generator) EmitNativeType(w io.Writer) { emitNativeType_scalar(w, g.AdjCfg, g) } func (g float64Generator) EmitNativeAccessors(w io.Writer) { emitNativeAccessors_scalar(w, g.AdjCfg, g) } func (g float64Generator) EmitNativeBuilder(w io.Writer) { emitNativeBuilder_scalar(w, g.AdjCfg, g) } func (g float64Generator) EmitNativeMaybe(w io.Writer) { emitNativeMaybe(w, g.AdjCfg, g) } // --- type info ---> func (g float64Generator) EmitTypeConst(w io.Writer) { doTemplate(` // TODO EmitTypeConst `, w, g.AdjCfg, g) } // --- TypedNode interface satisfaction ---> func (g float64Generator) EmitTypedNodeMethodType(w io.Writer) { doTemplate(` func ({{ .Type | TypeSymbol }}) Type() schema.Type { return nil /*TODO:typelit*/ } `, w, g.AdjCfg, g) } func (g float64Generator) EmitTypedNodeMethodRepresentation(w io.Writer) { emitTypicalTypedNodeMethodRepresentation(w, g.AdjCfg, g) } // --- Node interface satisfaction ---> func (g float64Generator) EmitNodeType(w io.Writer) { // No additional types needed. Methods all attach to the native type. } func (g float64Generator) EmitNodeTypeAssertions(w io.Writer) { emitNodeTypeAssertions_typical(w, g.AdjCfg, g) } func (g float64Generator) EmitNodeMethodAsFloat(w io.Writer) { emitNodeMethodAsKind_scalar(w, g.AdjCfg, g) } func (g float64Generator) EmitNodeMethodPrototype(w io.Writer) { emitNodeMethodPrototype_typical(w, g.AdjCfg, g) } func (g float64Generator) EmitNodePrototypeType(w io.Writer) { emitNodePrototypeType_typical(w, g.AdjCfg, g) } // --- NodeBuilder and NodeAssembler ---> func (g float64Generator) GetNodeBuilderGenerator() NodeBuilderGenerator { return float64BuilderGenerator{ g.AdjCfg, mixins.FloatAssemblerTraits{ PkgName: g.PkgName, TypeName: g.TypeName, AppliedPrefix: "_" + g.AdjCfg.TypeSymbol(g.Type) + "__", }, g.PkgName, g.Type, } } type float64BuilderGenerator struct { AdjCfg *AdjunctCfg mixins.FloatAssemblerTraits PkgName string Type *schema.TypeFloat } func (float64BuilderGenerator) IsRepr() bool { return false } // hint used in some generalized templates. func (g float64BuilderGenerator) EmitNodeBuilderType(w io.Writer) { emitEmitNodeBuilderType_typical(w, g.AdjCfg, g) } func (g float64BuilderGenerator) EmitNodeBuilderMethods(w io.Writer) { emitNodeBuilderMethods_typical(w, g.AdjCfg, g) } func (g float64BuilderGenerator) EmitNodeAssemblerType(w io.Writer) { emitNodeAssemblerType_scalar(w, g.AdjCfg, g) } func (g float64BuilderGenerator) EmitNodeAssemblerMethodAssignNull(w io.Writer) { emitNodeAssemblerMethodAssignNull_scalar(w, g.AdjCfg, g) } func (g float64BuilderGenerator) EmitNodeAssemblerMethodAssignFloat(w io.Writer) { emitNodeAssemblerMethodAssignKind_scalar(w, g.AdjCfg, g) } func (g float64BuilderGenerator) EmitNodeAssemblerMethodAssignNode(w io.Writer) { emitNodeAssemblerMethodAssignNode_scalar(w, g.AdjCfg, g) } func (g float64BuilderGenerator) EmitNodeAssemblerOtherBits(w io.Writer) { // Nothing needed here for float64 kinds. } ================================================ FILE: schema/gen/go/genFloatReprFloat.go ================================================ package gengo import ( "io" "github.com/ipld/go-ipld-prime/schema" "github.com/ipld/go-ipld-prime/schema/gen/go/mixins" ) var _ TypeGenerator = &float64ReprFloatGenerator{} func NewFloatReprFloatGenerator(pkgName string, typ *schema.TypeFloat, adjCfg *AdjunctCfg) TypeGenerator { return float64ReprFloatGenerator{ float64Generator{ adjCfg, mixins.FloatTraits{ PkgName: pkgName, TypeName: string(typ.Name()), TypeSymbol: adjCfg.TypeSymbol(typ), }, pkgName, typ, }, } } type float64ReprFloatGenerator struct { float64Generator } func (g float64ReprFloatGenerator) GetRepresentationNodeGen() NodeGenerator { return float64ReprFloatReprGenerator{ g.AdjCfg, g.Type, } } type float64ReprFloatReprGenerator struct { AdjCfg *AdjunctCfg Type *schema.TypeFloat } func (g float64ReprFloatReprGenerator) EmitNodeType(w io.Writer) { // Since this is a "natural" representation... there's just a type alias here. // No new functions are necessary. doTemplate(` type _{{ .Type | TypeSymbol }}__Repr = _{{ .Type | TypeSymbol }} `, w, g.AdjCfg, g) } func (g float64ReprFloatReprGenerator) EmitNodeTypeAssertions(w io.Writer) { doTemplate(` var _ datamodel.Node = &_{{ .Type | TypeSymbol }}__Repr{} `, w, g.AdjCfg, g) } func (float64ReprFloatReprGenerator) EmitNodeMethodKind(io.Writer) {} func (float64ReprFloatReprGenerator) EmitNodeMethodLookupByString(io.Writer) {} func (float64ReprFloatReprGenerator) EmitNodeMethodLookupByNode(io.Writer) {} func (float64ReprFloatReprGenerator) EmitNodeMethodLookupByIndex(io.Writer) {} func (float64ReprFloatReprGenerator) EmitNodeMethodLookupBySegment(io.Writer) {} func (float64ReprFloatReprGenerator) EmitNodeMethodMapIterator(io.Writer) {} func (float64ReprFloatReprGenerator) EmitNodeMethodListIterator(io.Writer) {} func (float64ReprFloatReprGenerator) EmitNodeMethodLength(io.Writer) {} func (float64ReprFloatReprGenerator) EmitNodeMethodIsAbsent(io.Writer) {} func (float64ReprFloatReprGenerator) EmitNodeMethodIsNull(io.Writer) {} func (float64ReprFloatReprGenerator) EmitNodeMethodAsBool(io.Writer) {} func (float64ReprFloatReprGenerator) EmitNodeMethodAsInt(io.Writer) {} func (float64ReprFloatReprGenerator) EmitNodeMethodAsFloat(io.Writer) {} func (float64ReprFloatReprGenerator) EmitNodeMethodAsString(io.Writer) {} func (float64ReprFloatReprGenerator) EmitNodeMethodAsBytes(io.Writer) {} func (float64ReprFloatReprGenerator) EmitNodeMethodAsLink(io.Writer) {} func (float64ReprFloatReprGenerator) EmitNodeMethodPrototype(io.Writer) {} func (g float64ReprFloatReprGenerator) EmitNodePrototypeType(w io.Writer) { // Since this is a "natural" representation... there's just a type alias here. // No new functions are necessary. doTemplate(` type _{{ .Type | TypeSymbol }}__ReprPrototype = _{{ .Type | TypeSymbol }}__Prototype `, w, g.AdjCfg, g) } func (g float64ReprFloatReprGenerator) GetNodeBuilderGenerator() NodeBuilderGenerator { return float64ReprFloatReprBuilderGenerator(g) } type float64ReprFloatReprBuilderGenerator struct { AdjCfg *AdjunctCfg Type *schema.TypeFloat } func (float64ReprFloatReprBuilderGenerator) EmitNodeBuilderType(io.Writer) {} func (float64ReprFloatReprBuilderGenerator) EmitNodeBuilderMethods(io.Writer) {} func (g float64ReprFloatReprBuilderGenerator) EmitNodeAssemblerType(w io.Writer) { // Since this is a "natural" representation... there's just a type alias here. // No new functions are necessary. doTemplate(` type _{{ .Type | TypeSymbol }}__ReprAssembler = _{{ .Type | TypeSymbol }}__Assembler `, w, g.AdjCfg, g) } func (float64ReprFloatReprBuilderGenerator) EmitNodeAssemblerMethodBeginMap(io.Writer) {} func (float64ReprFloatReprBuilderGenerator) EmitNodeAssemblerMethodBeginList(io.Writer) {} func (float64ReprFloatReprBuilderGenerator) EmitNodeAssemblerMethodAssignNull(io.Writer) {} func (float64ReprFloatReprBuilderGenerator) EmitNodeAssemblerMethodAssignBool(io.Writer) {} func (float64ReprFloatReprBuilderGenerator) EmitNodeAssemblerMethodAssignInt(io.Writer) {} func (float64ReprFloatReprBuilderGenerator) EmitNodeAssemblerMethodAssignFloat(io.Writer) {} func (float64ReprFloatReprBuilderGenerator) EmitNodeAssemblerMethodAssignString(io.Writer) {} func (float64ReprFloatReprBuilderGenerator) EmitNodeAssemblerMethodAssignBytes(io.Writer) {} func (float64ReprFloatReprBuilderGenerator) EmitNodeAssemblerMethodAssignLink(io.Writer) {} func (float64ReprFloatReprBuilderGenerator) EmitNodeAssemblerMethodAssignNode(io.Writer) {} func (float64ReprFloatReprBuilderGenerator) EmitNodeAssemblerMethodPrototype(io.Writer) {} func (float64ReprFloatReprBuilderGenerator) EmitNodeAssemblerOtherBits(io.Writer) {} ================================================ FILE: schema/gen/go/genInt.go ================================================ package gengo import ( "io" "github.com/ipld/go-ipld-prime/schema" "github.com/ipld/go-ipld-prime/schema/gen/go/mixins" ) type intGenerator struct { AdjCfg *AdjunctCfg mixins.IntTraits PkgName string Type *schema.TypeInt } func (intGenerator) IsRepr() bool { return false } // hint used in some generalized templates. // --- native content and specializations ---> func (g intGenerator) EmitNativeType(w io.Writer) { emitNativeType_scalar(w, g.AdjCfg, g) } func (g intGenerator) EmitNativeAccessors(w io.Writer) { emitNativeAccessors_scalar(w, g.AdjCfg, g) } func (g intGenerator) EmitNativeBuilder(w io.Writer) { emitNativeBuilder_scalar(w, g.AdjCfg, g) } func (g intGenerator) EmitNativeMaybe(w io.Writer) { emitNativeMaybe(w, g.AdjCfg, g) } // --- type info ---> func (g intGenerator) EmitTypeConst(w io.Writer) { doTemplate(` // TODO EmitTypeConst `, w, g.AdjCfg, g) } // --- TypedNode interface satisfaction ---> func (g intGenerator) EmitTypedNodeMethodType(w io.Writer) { doTemplate(` func ({{ .Type | TypeSymbol }}) Type() schema.Type { return nil /*TODO:typelit*/ } `, w, g.AdjCfg, g) } func (g intGenerator) EmitTypedNodeMethodRepresentation(w io.Writer) { emitTypicalTypedNodeMethodRepresentation(w, g.AdjCfg, g) } // --- Node interface satisfaction ---> func (g intGenerator) EmitNodeType(w io.Writer) { // No additional types needed. Methods all attach to the native type. } func (g intGenerator) EmitNodeTypeAssertions(w io.Writer) { emitNodeTypeAssertions_typical(w, g.AdjCfg, g) } func (g intGenerator) EmitNodeMethodAsInt(w io.Writer) { emitNodeMethodAsKind_scalar(w, g.AdjCfg, g) } func (g intGenerator) EmitNodeMethodPrototype(w io.Writer) { emitNodeMethodPrototype_typical(w, g.AdjCfg, g) } func (g intGenerator) EmitNodePrototypeType(w io.Writer) { emitNodePrototypeType_typical(w, g.AdjCfg, g) } // --- NodeBuilder and NodeAssembler ---> func (g intGenerator) GetNodeBuilderGenerator() NodeBuilderGenerator { return intBuilderGenerator{ g.AdjCfg, mixins.IntAssemblerTraits{ PkgName: g.PkgName, TypeName: g.TypeName, AppliedPrefix: "_" + g.AdjCfg.TypeSymbol(g.Type) + "__", }, g.PkgName, g.Type, } } type intBuilderGenerator struct { AdjCfg *AdjunctCfg mixins.IntAssemblerTraits PkgName string Type *schema.TypeInt } func (intBuilderGenerator) IsRepr() bool { return false } // hint used in some generalized templates. func (g intBuilderGenerator) EmitNodeBuilderType(w io.Writer) { emitEmitNodeBuilderType_typical(w, g.AdjCfg, g) } func (g intBuilderGenerator) EmitNodeBuilderMethods(w io.Writer) { emitNodeBuilderMethods_typical(w, g.AdjCfg, g) } func (g intBuilderGenerator) EmitNodeAssemblerType(w io.Writer) { emitNodeAssemblerType_scalar(w, g.AdjCfg, g) } func (g intBuilderGenerator) EmitNodeAssemblerMethodAssignNull(w io.Writer) { emitNodeAssemblerMethodAssignNull_scalar(w, g.AdjCfg, g) } func (g intBuilderGenerator) EmitNodeAssemblerMethodAssignInt(w io.Writer) { emitNodeAssemblerMethodAssignKind_scalar(w, g.AdjCfg, g) } func (g intBuilderGenerator) EmitNodeAssemblerMethodAssignNode(w io.Writer) { emitNodeAssemblerMethodAssignNode_scalar(w, g.AdjCfg, g) } func (g intBuilderGenerator) EmitNodeAssemblerOtherBits(w io.Writer) { // Nothing needed here for int kinds. } ================================================ FILE: schema/gen/go/genIntReprInt.go ================================================ package gengo import ( "io" "github.com/ipld/go-ipld-prime/schema" "github.com/ipld/go-ipld-prime/schema/gen/go/mixins" ) var _ TypeGenerator = &intReprIntGenerator{} func NewIntReprIntGenerator(pkgName string, typ *schema.TypeInt, adjCfg *AdjunctCfg) TypeGenerator { return intReprIntGenerator{ intGenerator{ adjCfg, mixins.IntTraits{ PkgName: pkgName, TypeName: string(typ.Name()), TypeSymbol: adjCfg.TypeSymbol(typ), }, pkgName, typ, }, } } type intReprIntGenerator struct { intGenerator } func (g intReprIntGenerator) GetRepresentationNodeGen() NodeGenerator { return intReprIntReprGenerator{ g.AdjCfg, g.Type, } } type intReprIntReprGenerator struct { AdjCfg *AdjunctCfg Type *schema.TypeInt } func (g intReprIntReprGenerator) EmitNodeType(w io.Writer) { // Since this is a "natural" representation... there's just a type alias here. // No new functions are necessary. doTemplate(` type _{{ .Type | TypeSymbol }}__Repr = _{{ .Type | TypeSymbol }} `, w, g.AdjCfg, g) } func (g intReprIntReprGenerator) EmitNodeTypeAssertions(w io.Writer) { doTemplate(` var _ datamodel.Node = &_{{ .Type | TypeSymbol }}__Repr{} `, w, g.AdjCfg, g) } func (intReprIntReprGenerator) EmitNodeMethodKind(io.Writer) {} func (intReprIntReprGenerator) EmitNodeMethodLookupByString(io.Writer) {} func (intReprIntReprGenerator) EmitNodeMethodLookupByNode(io.Writer) {} func (intReprIntReprGenerator) EmitNodeMethodLookupByIndex(io.Writer) {} func (intReprIntReprGenerator) EmitNodeMethodLookupBySegment(io.Writer) {} func (intReprIntReprGenerator) EmitNodeMethodMapIterator(io.Writer) {} func (intReprIntReprGenerator) EmitNodeMethodListIterator(io.Writer) {} func (intReprIntReprGenerator) EmitNodeMethodLength(io.Writer) {} func (intReprIntReprGenerator) EmitNodeMethodIsAbsent(io.Writer) {} func (intReprIntReprGenerator) EmitNodeMethodIsNull(io.Writer) {} func (intReprIntReprGenerator) EmitNodeMethodAsBool(io.Writer) {} func (intReprIntReprGenerator) EmitNodeMethodAsInt(io.Writer) {} func (intReprIntReprGenerator) EmitNodeMethodAsFloat(io.Writer) {} func (intReprIntReprGenerator) EmitNodeMethodAsString(io.Writer) {} func (intReprIntReprGenerator) EmitNodeMethodAsBytes(io.Writer) {} func (intReprIntReprGenerator) EmitNodeMethodAsLink(io.Writer) {} func (intReprIntReprGenerator) EmitNodeMethodPrototype(io.Writer) {} func (g intReprIntReprGenerator) EmitNodePrototypeType(w io.Writer) { // Since this is a "natural" representation... there's just a type alias here. // No new functions are necessary. doTemplate(` type _{{ .Type | TypeSymbol }}__ReprPrototype = _{{ .Type | TypeSymbol }}__Prototype `, w, g.AdjCfg, g) } func (g intReprIntReprGenerator) GetNodeBuilderGenerator() NodeBuilderGenerator { return intReprIntReprBuilderGenerator(g) } type intReprIntReprBuilderGenerator struct { AdjCfg *AdjunctCfg Type *schema.TypeInt } func (intReprIntReprBuilderGenerator) EmitNodeBuilderType(io.Writer) {} func (intReprIntReprBuilderGenerator) EmitNodeBuilderMethods(io.Writer) {} func (g intReprIntReprBuilderGenerator) EmitNodeAssemblerType(w io.Writer) { // Since this is a "natural" representation... there's just a type alias here. // No new functions are necessary. doTemplate(` type _{{ .Type | TypeSymbol }}__ReprAssembler = _{{ .Type | TypeSymbol }}__Assembler `, w, g.AdjCfg, g) } func (intReprIntReprBuilderGenerator) EmitNodeAssemblerMethodBeginMap(io.Writer) {} func (intReprIntReprBuilderGenerator) EmitNodeAssemblerMethodBeginList(io.Writer) {} func (intReprIntReprBuilderGenerator) EmitNodeAssemblerMethodAssignNull(io.Writer) {} func (intReprIntReprBuilderGenerator) EmitNodeAssemblerMethodAssignBool(io.Writer) {} func (intReprIntReprBuilderGenerator) EmitNodeAssemblerMethodAssignInt(io.Writer) {} func (intReprIntReprBuilderGenerator) EmitNodeAssemblerMethodAssignFloat(io.Writer) {} func (intReprIntReprBuilderGenerator) EmitNodeAssemblerMethodAssignString(io.Writer) {} func (intReprIntReprBuilderGenerator) EmitNodeAssemblerMethodAssignBytes(io.Writer) {} func (intReprIntReprBuilderGenerator) EmitNodeAssemblerMethodAssignLink(io.Writer) {} func (intReprIntReprBuilderGenerator) EmitNodeAssemblerMethodAssignNode(io.Writer) {} func (intReprIntReprBuilderGenerator) EmitNodeAssemblerMethodPrototype(io.Writer) {} func (intReprIntReprBuilderGenerator) EmitNodeAssemblerOtherBits(io.Writer) {} ================================================ FILE: schema/gen/go/genLink.go ================================================ package gengo import ( "io" "github.com/ipld/go-ipld-prime/schema" "github.com/ipld/go-ipld-prime/schema/gen/go/mixins" ) type linkGenerator struct { AdjCfg *AdjunctCfg mixins.LinkTraits PkgName string Type *schema.TypeLink } func (linkGenerator) IsRepr() bool { return false } // hint used in some generalized templates. // --- native content and specializations ---> func (g linkGenerator) EmitNativeType(w io.Writer) { emitNativeType_scalar(w, g.AdjCfg, g) } func (g linkGenerator) EmitNativeAccessors(w io.Writer) { emitNativeAccessors_scalar(w, g.AdjCfg, g) } func (g linkGenerator) EmitNativeBuilder(w io.Writer) { emitNativeBuilder_scalar(w, g.AdjCfg, g) } func (g linkGenerator) EmitNativeMaybe(w io.Writer) { emitNativeMaybe(w, g.AdjCfg, g) } // --- type info ---> func (g linkGenerator) EmitTypeConst(w io.Writer) { doTemplate(` // TODO EmitTypeConst `, w, g.AdjCfg, g) } // --- TypedNode interface satisfaction ---> func (g linkGenerator) EmitTypedNodeMethodType(w io.Writer) { doTemplate(` func ({{ .Type | TypeSymbol }}) Type() schema.Type { return nil /*TODO:typelit*/ } `, w, g.AdjCfg, g) // Bonus feature for some links (conforms to the schema.TypedLinkNode interface): if g.Type.HasReferencedType() { doTemplate(` func ({{ .Type | TypeSymbol }}) LinkTargetNodePrototype() datamodel.NodePrototype { return Type.{{ .Type.ReferencedType | TypeSymbol }}__Repr } `, w, g.AdjCfg, g) } } func (g linkGenerator) EmitTypedNodeMethodRepresentation(w io.Writer) { emitTypicalTypedNodeMethodRepresentation(w, g.AdjCfg, g) } // --- Node interface satisfaction ---> func (g linkGenerator) EmitNodeType(w io.Writer) { // No additional types needed. Methods all attach to the native type. } func (g linkGenerator) EmitNodeTypeAssertions(w io.Writer) { emitNodeTypeAssertions_typical(w, g.AdjCfg, g) } func (g linkGenerator) EmitNodeMethodAsLink(w io.Writer) { emitNodeMethodAsKind_scalar(w, g.AdjCfg, g) } func (g linkGenerator) EmitNodeMethodPrototype(w io.Writer) { emitNodeMethodPrototype_typical(w, g.AdjCfg, g) } func (g linkGenerator) EmitNodePrototypeType(w io.Writer) { emitNodePrototypeType_typical(w, g.AdjCfg, g) } // --- NodeBuilder and NodeAssembler ---> func (g linkGenerator) GetNodeBuilderGenerator() NodeBuilderGenerator { return linkBuilderGenerator{ g.AdjCfg, mixins.LinkAssemblerTraits{ PkgName: g.PkgName, TypeName: g.TypeName, AppliedPrefix: "_" + g.AdjCfg.TypeSymbol(g.Type) + "__", }, g.PkgName, g.Type, } } type linkBuilderGenerator struct { AdjCfg *AdjunctCfg mixins.LinkAssemblerTraits PkgName string Type *schema.TypeLink } func (linkBuilderGenerator) IsRepr() bool { return false } // hint used in some generalized templates. func (g linkBuilderGenerator) EmitNodeBuilderType(w io.Writer) { emitEmitNodeBuilderType_typical(w, g.AdjCfg, g) } func (g linkBuilderGenerator) EmitNodeBuilderMethods(w io.Writer) { emitNodeBuilderMethods_typical(w, g.AdjCfg, g) } func (g linkBuilderGenerator) EmitNodeAssemblerType(w io.Writer) { emitNodeAssemblerType_scalar(w, g.AdjCfg, g) } func (g linkBuilderGenerator) EmitNodeAssemblerMethodAssignNull(w io.Writer) { emitNodeAssemblerMethodAssignNull_scalar(w, g.AdjCfg, g) } func (g linkBuilderGenerator) EmitNodeAssemblerMethodAssignLink(w io.Writer) { emitNodeAssemblerMethodAssignKind_scalar(w, g.AdjCfg, g) } func (g linkBuilderGenerator) EmitNodeAssemblerMethodAssignNode(w io.Writer) { emitNodeAssemblerMethodAssignNode_scalar(w, g.AdjCfg, g) } func (g linkBuilderGenerator) EmitNodeAssemblerOtherBits(w io.Writer) { // Nothing needed here for link kinds. } ================================================ FILE: schema/gen/go/genLinkReprLink.go ================================================ package gengo import ( "io" "github.com/ipld/go-ipld-prime/schema" "github.com/ipld/go-ipld-prime/schema/gen/go/mixins" ) var _ TypeGenerator = &linkReprLinkGenerator{} func NewLinkReprLinkGenerator(pkgName string, typ *schema.TypeLink, adjCfg *AdjunctCfg) TypeGenerator { return linkReprLinkGenerator{ linkGenerator{ adjCfg, mixins.LinkTraits{ PkgName: pkgName, TypeName: string(typ.Name()), TypeSymbol: adjCfg.TypeSymbol(typ), }, pkgName, typ, }, } } type linkReprLinkGenerator struct { linkGenerator } func (g linkReprLinkGenerator) GetRepresentationNodeGen() NodeGenerator { return linkReprLinkReprGenerator{ g.AdjCfg, g.Type, } } type linkReprLinkReprGenerator struct { AdjCfg *AdjunctCfg Type *schema.TypeLink } func (g linkReprLinkReprGenerator) EmitNodeType(w io.Writer) { // Since this is a "natural" representation... there's just a type alias here. // No new functions are necessary. doTemplate(` type _{{ .Type | TypeSymbol }}__Repr = _{{ .Type | TypeSymbol }} `, w, g.AdjCfg, g) } func (g linkReprLinkReprGenerator) EmitNodeTypeAssertions(w io.Writer) { doTemplate(` var _ datamodel.Node = &_{{ .Type | TypeSymbol }}__Repr{} `, w, g.AdjCfg, g) } func (linkReprLinkReprGenerator) EmitNodeMethodKind(io.Writer) {} func (linkReprLinkReprGenerator) EmitNodeMethodLookupByString(io.Writer) {} func (linkReprLinkReprGenerator) EmitNodeMethodLookupByNode(io.Writer) {} func (linkReprLinkReprGenerator) EmitNodeMethodLookupByIndex(io.Writer) {} func (linkReprLinkReprGenerator) EmitNodeMethodLookupBySegment(io.Writer) {} func (linkReprLinkReprGenerator) EmitNodeMethodMapIterator(io.Writer) {} func (linkReprLinkReprGenerator) EmitNodeMethodListIterator(io.Writer) {} func (linkReprLinkReprGenerator) EmitNodeMethodLength(io.Writer) {} func (linkReprLinkReprGenerator) EmitNodeMethodIsAbsent(io.Writer) {} func (linkReprLinkReprGenerator) EmitNodeMethodIsNull(io.Writer) {} func (linkReprLinkReprGenerator) EmitNodeMethodAsBool(io.Writer) {} func (linkReprLinkReprGenerator) EmitNodeMethodAsInt(io.Writer) {} func (linkReprLinkReprGenerator) EmitNodeMethodAsFloat(io.Writer) {} func (linkReprLinkReprGenerator) EmitNodeMethodAsString(io.Writer) {} func (linkReprLinkReprGenerator) EmitNodeMethodAsBytes(io.Writer) {} func (linkReprLinkReprGenerator) EmitNodeMethodAsLink(io.Writer) {} func (linkReprLinkReprGenerator) EmitNodeMethodPrototype(io.Writer) {} func (g linkReprLinkReprGenerator) EmitNodePrototypeType(w io.Writer) { // Since this is a "natural" representation... there's just a type alias here. // No new functions are necessary. doTemplate(` type _{{ .Type | TypeSymbol }}__ReprPrototype = _{{ .Type | TypeSymbol }}__Prototype `, w, g.AdjCfg, g) } func (g linkReprLinkReprGenerator) GetNodeBuilderGenerator() NodeBuilderGenerator { return linkReprLinkReprBuilderGenerator(g) } type linkReprLinkReprBuilderGenerator struct { AdjCfg *AdjunctCfg Type *schema.TypeLink } func (linkReprLinkReprBuilderGenerator) EmitNodeBuilderType(io.Writer) {} func (linkReprLinkReprBuilderGenerator) EmitNodeBuilderMethods(io.Writer) {} func (g linkReprLinkReprBuilderGenerator) EmitNodeAssemblerType(w io.Writer) { // Since this is a "natural" representation... there's just a type alias here. // No new functions are necessary. doTemplate(` type _{{ .Type | TypeSymbol }}__ReprAssembler = _{{ .Type | TypeSymbol }}__Assembler `, w, g.AdjCfg, g) } func (linkReprLinkReprBuilderGenerator) EmitNodeAssemblerMethodBeginMap(io.Writer) {} func (linkReprLinkReprBuilderGenerator) EmitNodeAssemblerMethodBeginList(io.Writer) {} func (linkReprLinkReprBuilderGenerator) EmitNodeAssemblerMethodAssignNull(io.Writer) {} func (linkReprLinkReprBuilderGenerator) EmitNodeAssemblerMethodAssignBool(io.Writer) {} func (linkReprLinkReprBuilderGenerator) EmitNodeAssemblerMethodAssignInt(io.Writer) {} func (linkReprLinkReprBuilderGenerator) EmitNodeAssemblerMethodAssignFloat(io.Writer) {} func (linkReprLinkReprBuilderGenerator) EmitNodeAssemblerMethodAssignString(io.Writer) {} func (linkReprLinkReprBuilderGenerator) EmitNodeAssemblerMethodAssignBytes(io.Writer) {} func (linkReprLinkReprBuilderGenerator) EmitNodeAssemblerMethodAssignLink(io.Writer) {} func (linkReprLinkReprBuilderGenerator) EmitNodeAssemblerMethodAssignNode(io.Writer) {} func (linkReprLinkReprBuilderGenerator) EmitNodeAssemblerMethodPrototype(io.Writer) {} func (linkReprLinkReprBuilderGenerator) EmitNodeAssemblerOtherBits(io.Writer) {} ================================================ FILE: schema/gen/go/genList.go ================================================ package gengo import ( "io" "github.com/ipld/go-ipld-prime/schema" "github.com/ipld/go-ipld-prime/schema/gen/go/mixins" ) type listGenerator struct { AdjCfg *AdjunctCfg mixins.ListTraits PkgName string Type *schema.TypeList } func (listGenerator) IsRepr() bool { return false } // hint used in some generalized templates. // --- native content and specializations ---> func (g listGenerator) EmitNativeType(w io.Writer) { // Lists are a pretty straightforward struct enclosing a slice. doTemplate(` {{- if Comments -}} // {{ .Type | TypeSymbol }} matches the IPLD Schema type "{{ .Type.Name }}". It has {{ .Kind }} kind. {{- end}} type {{ .Type | TypeSymbol }} = *_{{ .Type | TypeSymbol }} type _{{ .Type | TypeSymbol }} struct { x []_{{ .Type.ValueType | TypeSymbol }}{{if .Type.ValueIsNullable }}__Maybe{{end}} } `, w, g.AdjCfg, g) } func (g listGenerator) EmitNativeAccessors(w io.Writer) { // Generate a speciated Lookup as well as LookupMaybe method. // The Lookup method returns nil in case of *either* an out-of-range/absent value or a null value, // and so should only be used if the list type doesn't allow nullable keys or if the caller doesn't care about the difference. // The LookupMaybe method returns a MaybeT type for the list value, // and is needed if the list allows nullable values and the caller wishes to distinguish between null and out-of-range/absent. // (The Lookup method should be preferred for lists that have non-nullable keys, because LookupMaybe may incur additional costs; // boxing something into a maybe when it wasn't already stored that way costs an alloc(!), // and may additionally incur a memcpy if the maybe for the value type doesn't use pointers internally). doTemplate(` func (n *_{{ .Type | TypeSymbol }}) Lookup(idx int64) {{ .Type.ValueType | TypeSymbol }} { if n.Length() <= idx { return nil } v := &n.x[idx] {{- if .Type.ValueIsNullable }} if v.m == schema.Maybe_Null { return nil } return {{ if not (MaybeUsesPtr .Type.ValueType) }}&{{end}}v.v {{- else}} return v {{- end}} } func (n *_{{ .Type | TypeSymbol }}) LookupMaybe(idx int64) Maybe{{ .Type.ValueType | TypeSymbol }} { if n.Length() <= idx { return nil } v := &n.x[idx] {{- if .Type.ValueIsNullable }} return v {{- else}} return &_{{ .Type.ValueType | TypeSymbol }}__Maybe{ m: schema.Maybe_Value, v: {{ if not (MaybeUsesPtr .Type.ValueType) }}*{{end}}v, } {{- end}} } var _{{ .Type | TypeSymbol }}__valueAbsent = _{{ .Type.ValueType | TypeSymbol }}__Maybe{m:schema.Maybe_Absent} `, w, g.AdjCfg, g) // Generate a speciated iterator. // The main advantage of this over the general datamodel.ListIterator is of course keeping types visible (and concrete, to the compiler's eyes in optimizations, too). // It also elides the error return from the iterator's Next method. (Overreads will result in -1 as an index and nil values; this is both easily avoidable, and unambiguous if you do goof and hit it.) doTemplate(` func (n {{ .Type | TypeSymbol }}) Iterator() *{{ .Type | TypeSymbol }}__Itr { return &{{ .Type | TypeSymbol }}__Itr{n, 0} } type {{ .Type | TypeSymbol }}__Itr struct { n {{ .Type | TypeSymbol }} idx int } func (itr *{{ .Type | TypeSymbol }}__Itr) Next() (idx int64, v {{if .Type.ValueIsNullable }}Maybe{{end}}{{ .Type.ValueType | TypeSymbol }}) { if itr.idx >= len(itr.n.x) { return -1, nil } idx = int64(itr.idx) v = &itr.n.x[itr.idx] itr.idx++ return } func (itr *{{ .Type | TypeSymbol }}__Itr) Done() bool { return itr.idx >= len(itr.n.x) } `, w, g.AdjCfg, g) } func (g listGenerator) EmitNativeBuilder(w io.Writer) { // FUTURE: come back to this -- not yet clear what exactly might be most worth emitting here. } func (g listGenerator) EmitNativeMaybe(w io.Writer) { emitNativeMaybe(w, g.AdjCfg, g) } // --- type info ---> func (g listGenerator) EmitTypeConst(w io.Writer) { doTemplate(` // TODO EmitTypeConst `, w, g.AdjCfg, g) } // --- TypedNode interface satisfaction ---> func (g listGenerator) EmitTypedNodeMethodType(w io.Writer) { doTemplate(` func ({{ .Type | TypeSymbol }}) Type() schema.Type { return nil /*TODO:typelit*/ } `, w, g.AdjCfg, g) } func (g listGenerator) EmitTypedNodeMethodRepresentation(w io.Writer) { emitTypicalTypedNodeMethodRepresentation(w, g.AdjCfg, g) } // --- Node interface satisfaction ---> func (g listGenerator) EmitNodeType(w io.Writer) { // No additional types needed. Methods all attach to the native type. } func (g listGenerator) EmitNodeTypeAssertions(w io.Writer) { emitNodeTypeAssertions_typical(w, g.AdjCfg, g) } func (g listGenerator) EmitNodeMethodLookupByIndex(w io.Writer) { doTemplate(` func (n {{ .Type | TypeSymbol }}) LookupByIndex(idx int64) (datamodel.Node, error) { if n.Length() <= idx { return nil, datamodel.ErrNotExists{Segment: datamodel.PathSegmentOfInt(idx)} } v := &n.x[idx] {{- if .Type.ValueIsNullable }} if v.m == schema.Maybe_Null { return datamodel.Null, nil } return {{ if not (MaybeUsesPtr .Type.ValueType) }}&{{end}}v.v, nil {{- else}} return v, nil {{- end}} } `, w, g.AdjCfg, g) } func (g listGenerator) EmitNodeMethodLookupByNode(w io.Writer) { // LookupByNode will proceed by coercing to int64 if it can; or fail; those are really the only options. // REVIEW: how much coercion is done by other types varies quite wildly. so we should figure out if that inconsistency is acceptable, and at least document it if so. doTemplate(` func (n {{ .Type | TypeSymbol }}) LookupByNode(k datamodel.Node) (datamodel.Node, error) { idx, err := k.AsInt() if err != nil { return nil, err } return n.LookupByIndex(idx) } `, w, g.AdjCfg, g) } func (g listGenerator) EmitNodeMethodListIterator(w io.Writer) { doTemplate(` func (n {{ .Type | TypeSymbol }}) ListIterator() datamodel.ListIterator { return &_{{ .Type | TypeSymbol }}__ListItr{n, 0} } type _{{ .Type | TypeSymbol }}__ListItr struct { n {{ .Type | TypeSymbol }} idx int } func (itr *_{{ .Type | TypeSymbol }}__ListItr) Next() (idx int64, v datamodel.Node, _ error) { if itr.idx >= len(itr.n.x) { return -1, nil, datamodel.ErrIteratorOverread{} } idx = int64(itr.idx) x := &itr.n.x[itr.idx] {{- if .Type.ValueIsNullable }} switch x.m { case schema.Maybe_Null: v = datamodel.Null case schema.Maybe_Value: v = {{ if not (MaybeUsesPtr .Type.ValueType) }}&{{end}}x.v } {{- else}} v = x {{- end}} itr.idx++ return } func (itr *_{{ .Type | TypeSymbol }}__ListItr) Done() bool { return itr.idx >= len(itr.n.x) } `, w, g.AdjCfg, g) } func (g listGenerator) EmitNodeMethodLength(w io.Writer) { doTemplate(` func (n {{ .Type | TypeSymbol }}) Length() int64 { return int64(len(n.x)) } `, w, g.AdjCfg, g) } func (g listGenerator) EmitNodeMethodPrototype(w io.Writer) { emitNodeMethodPrototype_typical(w, g.AdjCfg, g) } func (g listGenerator) EmitNodePrototypeType(w io.Writer) { emitNodePrototypeType_typical(w, g.AdjCfg, g) } // --- NodeBuilder and NodeAssembler ---> func (g listGenerator) GetNodeBuilderGenerator() NodeBuilderGenerator { return listBuilderGenerator{ g.AdjCfg, mixins.ListAssemblerTraits{ PkgName: g.PkgName, TypeName: g.TypeName, AppliedPrefix: "_" + g.AdjCfg.TypeSymbol(g.Type) + "__", }, g.PkgName, g.Type, } } type listBuilderGenerator struct { AdjCfg *AdjunctCfg mixins.ListAssemblerTraits PkgName string Type *schema.TypeList } func (listBuilderGenerator) IsRepr() bool { return false } // hint used in some generalized templates. func (g listBuilderGenerator) EmitNodeBuilderType(w io.Writer) { emitEmitNodeBuilderType_typical(w, g.AdjCfg, g) } func (g listBuilderGenerator) EmitNodeBuilderMethods(w io.Writer) { emitNodeBuilderMethods_typical(w, g.AdjCfg, g) } func (g listBuilderGenerator) EmitNodeAssemblerType(w io.Writer) { // - 'w' is the "**w**ip" pointer. // - 'm' is the **m**aybe which communicates our completeness to the parent if we're a child assembler. // - 'state' is what it says on the tin. this is used for the list state (the broad transitions between null, start-list, and finish are handled by 'm' for consistency with other types). // // - 'cm' is **c**hild **m**aybe and is used for the completion message from children. // It's only present if list values *aren't* allowed to be nullable, since otherwise they have their own per-value maybe slot we can use. // - 'va' is the embedded child value assembler. doTemplate(` type _{{ .Type | TypeSymbol }}__Assembler struct { w *_{{ .Type | TypeSymbol }} m *schema.Maybe state laState {{ if not .Type.ValueIsNullable }}cm schema.Maybe{{end}} va _{{ .Type.ValueType | TypeSymbol }}__Assembler } func (na *_{{ .Type | TypeSymbol }}__Assembler) reset() { na.state = laState_initial na.va.reset() } `, w, g.AdjCfg, g) } func (g listBuilderGenerator) EmitNodeAssemblerMethodBeginList(w io.Writer) { emitNodeAssemblerMethodBeginList_listoid(w, g.AdjCfg, g) } func (g listBuilderGenerator) EmitNodeAssemblerMethodAssignNull(w io.Writer) { emitNodeAssemblerMethodAssignNull_recursive(w, g.AdjCfg, g) } func (g listBuilderGenerator) EmitNodeAssemblerMethodAssignNode(w io.Writer) { emitNodeAssemblerMethodAssignNode_listoid(w, g.AdjCfg, g) } func (g listBuilderGenerator) EmitNodeAssemblerOtherBits(w io.Writer) { emitNodeAssemblerHelper_listoid_tidyHelper(w, g.AdjCfg, g) emitNodeAssemblerHelper_listoid_listAssemblerMethods(w, g.AdjCfg, g) } ================================================ FILE: schema/gen/go/genListReprList.go ================================================ package gengo import ( "io" "github.com/ipld/go-ipld-prime/schema" "github.com/ipld/go-ipld-prime/schema/gen/go/mixins" ) var _ TypeGenerator = &listReprListGenerator{} func NewListReprListGenerator(pkgName string, typ *schema.TypeList, adjCfg *AdjunctCfg) TypeGenerator { return listReprListGenerator{ listGenerator{ adjCfg, mixins.ListTraits{ PkgName: pkgName, TypeName: string(typ.Name()), TypeSymbol: adjCfg.TypeSymbol(typ), }, pkgName, typ, }, } } type listReprListGenerator struct { listGenerator } func (g listReprListGenerator) GetRepresentationNodeGen() NodeGenerator { return listReprListReprGenerator{ g.AdjCfg, mixins.ListTraits{ PkgName: g.PkgName, TypeName: string(g.Type.Name()) + ".Repr", TypeSymbol: "_" + g.AdjCfg.TypeSymbol(g.Type) + "__Repr", }, g.PkgName, g.Type, } } type listReprListReprGenerator struct { AdjCfg *AdjunctCfg mixins.ListTraits PkgName string Type *schema.TypeList } func (listReprListReprGenerator) IsRepr() bool { return true } // hint used in some generalized templates. func (g listReprListReprGenerator) EmitNodeType(w io.Writer) { // Even though this is a "natural" representation... we need a new type here, // because lists are recursive, and so all our functions that access // children need to remember to return the representation node of those child values. // It's still structurally the same, though (and we'll be able to cast in the methodset pattern). // Error-thunking methods also have a different string in their error, so those are unique even if they don't seem particularly interesting. doTemplate(` type _{{ .Type | TypeSymbol }}__Repr _{{ .Type | TypeSymbol }} `, w, g.AdjCfg, g) } func (g listReprListReprGenerator) EmitNodeTypeAssertions(w io.Writer) { doTemplate(` var _ datamodel.Node = &_{{ .Type | TypeSymbol }}__Repr{} `, w, g.AdjCfg, g) } func (g listReprListReprGenerator) EmitNodeMethodLookupByNode(w io.Writer) { // Null is also already a branch in the method we're calling; hopefully the compiler inlines and sees this and DTRT. // REVIEW: these unchecked casts are definitely safe at compile time, but I'm not sure if the compiler considers that provable, // so we should investigate if there's any runtime checks injected here that waste time. If so: write this with more gsloc to avoid :( doTemplate(` func (nr *_{{ .Type | TypeSymbol }}__Repr) LookupByNode(k datamodel.Node) (datamodel.Node, error) { v, err := ({{ .Type | TypeSymbol }})(nr).LookupByNode(k) if err != nil || v == datamodel.Null { return v, err } return v.({{ .Type.ValueType | TypeSymbol}}).Representation(), nil } `, w, g.AdjCfg, g) } func (g listReprListReprGenerator) EmitNodeMethodLookupByIndex(w io.Writer) { doTemplate(` func (nr *_{{ .Type | TypeSymbol }}__Repr) LookupByIndex(idx int64) (datamodel.Node, error) { v, err := ({{ .Type | TypeSymbol }})(nr).LookupByIndex(idx) if err != nil || v == datamodel.Null { return v, err } return v.({{ .Type.ValueType | TypeSymbol}}).Representation(), nil } `, w, g.AdjCfg, g) } func (g listReprListReprGenerator) EmitNodeMethodListIterator(w io.Writer) { // FUTURE: trying to get this to share the preallocated memory if we get iterators wedged into their node slab will be ... fun. doTemplate(` func (nr *_{{ .Type | TypeSymbol }}__Repr) ListIterator() datamodel.ListIterator { return &_{{ .Type | TypeSymbol }}__ReprListItr{({{ .Type | TypeSymbol }})(nr), 0} } type _{{ .Type | TypeSymbol }}__ReprListItr _{{ .Type | TypeSymbol }}__ListItr func (itr *_{{ .Type | TypeSymbol }}__ReprListItr) Next() (idx int64, v datamodel.Node, err error) { idx, v, err = (*_{{ .Type | TypeSymbol }}__ListItr)(itr).Next() if err != nil || v == datamodel.Null { return } return idx, v.({{ .Type.ValueType | TypeSymbol}}).Representation(), nil } func (itr *_{{ .Type | TypeSymbol }}__ReprListItr) Done() bool { return (*_{{ .Type | TypeSymbol }}__ListItr)(itr).Done() } `, w, g.AdjCfg, g) } func (g listReprListReprGenerator) EmitNodeMethodLength(w io.Writer) { doTemplate(` func (rn *_{{ .Type | TypeSymbol }}__Repr) Length() int64 { return int64(len(rn.x)) } `, w, g.AdjCfg, g) } func (g listReprListReprGenerator) EmitNodeMethodPrototype(w io.Writer) { emitNodeMethodPrototype_typical(w, g.AdjCfg, g) } func (g listReprListReprGenerator) EmitNodePrototypeType(w io.Writer) { emitNodePrototypeType_typical(w, g.AdjCfg, g) } // --- NodeBuilder and NodeAssembler ---> func (g listReprListReprGenerator) GetNodeBuilderGenerator() NodeBuilderGenerator { return listReprListReprBuilderGenerator{ g.AdjCfg, mixins.ListAssemblerTraits{ PkgName: g.PkgName, TypeName: g.TypeName, AppliedPrefix: "_" + g.AdjCfg.TypeSymbol(g.Type) + "__Repr", }, g.PkgName, g.Type, } } type listReprListReprBuilderGenerator struct { AdjCfg *AdjunctCfg mixins.ListAssemblerTraits PkgName string Type *schema.TypeList } func (listReprListReprBuilderGenerator) IsRepr() bool { return true } // hint used in some generalized templates. func (g listReprListReprBuilderGenerator) EmitNodeBuilderType(w io.Writer) { emitEmitNodeBuilderType_typical(w, g.AdjCfg, g) } func (g listReprListReprBuilderGenerator) EmitNodeBuilderMethods(w io.Writer) { emitNodeBuilderMethods_typical(w, g.AdjCfg, g) } func (g listReprListReprBuilderGenerator) EmitNodeAssemblerType(w io.Writer) { // - 'w' is the "**w**ip" pointer. // - 'm' is the **m**aybe which communicates our completeness to the parent if we're a child assembler. // - 'state' is what it says on the tin. this is used for the list state (the broad transitions between null, start-list, and finish are handled by 'm' for consistency with other types). // // - 'cm' is **c**hild **m**aybe and is used for the completion message from children. // It's only present if list values *aren't* allowed to be nullable, since otherwise they have their own per-value maybe slot we can use. // - 'va' is the embedded child value assembler. // // Note that this textually similar to the type-level assembler, but because it embeds the repr assembler for the child types, // it might be *significantly* different in size and memory layout in that trailing part of the struct. doTemplate(` type _{{ .Type | TypeSymbol }}__ReprAssembler struct { w *_{{ .Type | TypeSymbol }} m *schema.Maybe state laState {{ if not .Type.ValueIsNullable }}cm schema.Maybe{{end}} va _{{ .Type.ValueType | TypeSymbol }}__ReprAssembler } func (na *_{{ .Type | TypeSymbol }}__ReprAssembler) reset() { na.state = laState_initial na.va.reset() } `, w, g.AdjCfg, g) } func (g listReprListReprBuilderGenerator) EmitNodeAssemblerMethodBeginList(w io.Writer) { emitNodeAssemblerMethodBeginList_listoid(w, g.AdjCfg, g) } func (g listReprListReprBuilderGenerator) EmitNodeAssemblerMethodAssignNull(w io.Writer) { emitNodeAssemblerMethodAssignNull_recursive(w, g.AdjCfg, g) } func (g listReprListReprBuilderGenerator) EmitNodeAssemblerMethodAssignNode(w io.Writer) { emitNodeAssemblerMethodAssignNode_listoid(w, g.AdjCfg, g) } func (g listReprListReprBuilderGenerator) EmitNodeAssemblerOtherBits(w io.Writer) { emitNodeAssemblerHelper_listoid_tidyHelper(w, g.AdjCfg, g) emitNodeAssemblerHelper_listoid_listAssemblerMethods(w, g.AdjCfg, g) } ================================================ FILE: schema/gen/go/genMap.go ================================================ package gengo import ( "io" "github.com/ipld/go-ipld-prime/schema" "github.com/ipld/go-ipld-prime/schema/gen/go/mixins" ) type mapGenerator struct { AdjCfg *AdjunctCfg mixins.MapTraits PkgName string Type *schema.TypeMap } func (mapGenerator) IsRepr() bool { return false } // hint used in some generalized templates. // --- native content and specializations ---> func (g mapGenerator) EmitNativeType(w io.Writer) { // Maps do double bookkeeping. // - 'm' is used for quick lookup. // - 't' is used for both for order maintenance, and for allocation amortization for both keys and values. // Note that the key in 'm' is *not* a pointer. // The value in 'm' is a pointer into 't' (except when it's a maybe; maybes are already pointers). doTemplate(` {{- if Comments -}} // {{ .Type | TypeSymbol }} matches the IPLD Schema type "{{ .Type.Name }}". It has {{ .Kind }} kind. {{- end}} type {{ .Type | TypeSymbol }} = *_{{ .Type | TypeSymbol }} type _{{ .Type | TypeSymbol }} struct { m map[_{{ .Type.KeyType | TypeSymbol }}]{{if .Type.ValueIsNullable }}Maybe{{else}}*_{{end}}{{ .Type.ValueType | TypeSymbol }} t []_{{ .Type | TypeSymbol }}__entry } `, w, g.AdjCfg, g) // - address of 'k' is used when we return keys as nodes, such as in iterators. // Having these in the 't' slice above amortizes moving all of them to heap at once, // which makes iterators that have to return them as an interface much (much) lower cost -- no 'runtime.conv*' pain. // - address of 'v' is used in map values, to return, and of course also in iterators. doTemplate(` type _{{ .Type | TypeSymbol }}__entry struct { k _{{ .Type.KeyType | TypeSymbol }} v _{{ .Type.ValueType | TypeSymbol }}{{if .Type.ValueIsNullable }}__Maybe{{end}} } `, w, g.AdjCfg, g) } func (g mapGenerator) EmitNativeAccessors(w io.Writer) { // Generate a speciated Lookup as well as LookupMaybe method. // The Lookup method returns nil in case of *either* an absent value or a null value, // and so should only be used if the map type doesn't allow nullable keys or if the caller doesn't care about the difference. // The LookupMaybe method returns a MaybeT type for the map value, // and is needed if the map allows nullable values and the caller wishes to distinguish between null and absent. // (The Lookup method should be preferred for maps that have non-nullable keys, because LookupMaybe may incur additional costs; // boxing something into a maybe when it wasn't already stored that way costs an alloc(!), // and may additionally incur a memcpy if the maybe for the value type doesn't use pointers internally). doTemplate(` func (n *_{{ .Type | TypeSymbol }}) Lookup(k {{ .Type.KeyType | TypeSymbol }}) {{ .Type.ValueType | TypeSymbol }} { v, exists := n.m[*k] if !exists { return nil } {{- if .Type.ValueIsNullable }} if v.m == schema.Maybe_Null { return nil } return {{ if not (MaybeUsesPtr .Type.ValueType) }}&{{end}}v.v {{- else}} return v {{- end}} } func (n *_{{ .Type | TypeSymbol }}) LookupMaybe(k {{ .Type.KeyType | TypeSymbol }}) Maybe{{ .Type.ValueType | TypeSymbol }} { v, exists := n.m[*k] if !exists { return &_{{ .Type | TypeSymbol }}__valueAbsent } {{- if .Type.ValueIsNullable }} return v {{- else}} return &_{{ .Type.ValueType | TypeSymbol }}__Maybe{ m: schema.Maybe_Value, v: {{ if not (MaybeUsesPtr .Type.ValueType) }}*{{end}}v, } {{- end}} } var _{{ .Type | TypeSymbol }}__valueAbsent = _{{ .Type.ValueType | TypeSymbol }}__Maybe{m:schema.Maybe_Absent} `, w, g.AdjCfg, g) // Generate a speciated iterator. // The main advantage of this over the general datamodel.MapIterator is of course keeping types visible (and concrete, to the compiler's eyes in optimizations, too). // It also elides the error return from the iterator's Next method. (Overreads will result in nil keys; this is both easily avoidable, and unambiguous if you do goof and hit it.) doTemplate(` func (n {{ .Type | TypeSymbol }}) Iterator() *{{ .Type | TypeSymbol }}__Itr { return &{{ .Type | TypeSymbol }}__Itr{n, 0} } type {{ .Type | TypeSymbol }}__Itr struct { n {{ .Type | TypeSymbol }} idx int } func (itr *{{ .Type | TypeSymbol }}__Itr) Next() (k {{ .Type.KeyType | TypeSymbol }}, v {{if .Type.ValueIsNullable }}Maybe{{end}}{{ .Type.ValueType | TypeSymbol }}) { if itr.idx >= len(itr.n.t) { return nil, nil } x := &itr.n.t[itr.idx] k = &x.k v = &x.v itr.idx++ return } func (itr *{{ .Type | TypeSymbol }}__Itr) Done() bool { return itr.idx >= len(itr.n.t) } `, w, g.AdjCfg, g) } func (g mapGenerator) EmitNativeBuilder(w io.Writer) { // Not yet clear what exactly might be most worth emitting here. } func (g mapGenerator) EmitNativeMaybe(w io.Writer) { emitNativeMaybe(w, g.AdjCfg, g) } // --- type info ---> func (g mapGenerator) EmitTypeConst(w io.Writer) { doTemplate(` // TODO EmitTypeConst `, w, g.AdjCfg, g) } // --- TypedNode interface satisfaction ---> func (g mapGenerator) EmitTypedNodeMethodType(w io.Writer) { doTemplate(` func ({{ .Type | TypeSymbol }}) Type() schema.Type { return nil /*TODO:typelit*/ } `, w, g.AdjCfg, g) } func (g mapGenerator) EmitTypedNodeMethodRepresentation(w io.Writer) { emitTypicalTypedNodeMethodRepresentation(w, g.AdjCfg, g) } // --- Node interface satisfaction ---> func (g mapGenerator) EmitNodeType(w io.Writer) { // No additional types needed. Methods all attach to the native type. } func (g mapGenerator) EmitNodeTypeAssertions(w io.Writer) { emitNodeTypeAssertions_typical(w, g.AdjCfg, g) } func (g mapGenerator) EmitNodeMethodLookupByString(w io.Writer) { // What should be coercible in which directions (and how surprising that is) is an interesting question. // Most of the answer comes from considering what needs to be possible when working with PathSegment: // we *must* be able to accept a string in a PathSegment and be able to use it to navigate a map -- even if the map has complex keys. // For that to work out, it means if the key type doesn't have a string type kind, we must be willing to reach into its representation and use the fromString there. // If the key type *does* have a string kind at the type level, we'll use that; no need to consider going through the representation. doTemplate(` func (n {{ .Type | TypeSymbol }}) LookupByString(k string) (datamodel.Node, error) { var k2 _{{ .Type.KeyType | TypeSymbol }} {{- if eq .Type.KeyType.TypeKind.String "String" }} if err := (_{{ .Type.KeyType | TypeSymbol }}__Prototype{}).fromString(&k2, k); err != nil { return nil, err // TODO wrap in some kind of ErrInvalidKey } {{- else}} if err := (_{{ .Type.KeyType | TypeSymbol }}__ReprPrototype{}).fromString(&k2, k); err != nil { return nil, err // TODO wrap in some kind of ErrInvalidKey } {{- end}} v, exists := n.m[k2] if !exists { return nil, datamodel.ErrNotExists{Segment: datamodel.PathSegmentOfString(k)} } {{- if .Type.ValueIsNullable }} if v.m == schema.Maybe_Null { return datamodel.Null, nil } return {{ if not (MaybeUsesPtr .Type.ValueType) }}&{{end}}v.v, nil {{- else}} return v, nil {{- end}} } `, w, g.AdjCfg, g) } func (g mapGenerator) EmitNodeMethodLookupByNode(w io.Writer) { // LookupByNode will proceed by cast if it can; or simply error if that doesn't work. // There's no attempt to turn the node (or its repr) into a string and then reify that into a key; // if you used a Node here, you should've meant it. // REVIEW: by comparison structs will coerce anything stringish silently...! so we should figure out if that inconsistency is acceptable, and at least document it if so. doTemplate(` func (n {{ .Type | TypeSymbol }}) LookupByNode(k datamodel.Node) (datamodel.Node, error) { k2, ok := k.({{ .Type.KeyType | TypeSymbol }}) if !ok { panic("todo invalid key type error") // 'schema.ErrInvalidKey{TypeName:"{{ .PkgName }}.{{ .Type.Name }}", Key:&_String{k}}' doesn't quite cut it: need room to explain the type, and it's not guaranteed k can be turned into a string at all } v, exists := n.m[*k2] if !exists { return nil, datamodel.ErrNotExists{Segment: datamodel.PathSegmentOfString(k2.String())} } {{- if .Type.ValueIsNullable }} if v.m == schema.Maybe_Null { return datamodel.Null, nil } return {{ if not (MaybeUsesPtr .Type.ValueType) }}&{{end}}v.v, nil {{- else}} return v, nil {{- end}} } `, w, g.AdjCfg, g) } func (g mapGenerator) EmitNodeMethodMapIterator(w io.Writer) { doTemplate(` func (n {{ .Type | TypeSymbol }}) MapIterator() datamodel.MapIterator { return &_{{ .Type | TypeSymbol }}__MapItr{n, 0} } type _{{ .Type | TypeSymbol }}__MapItr struct { n {{ .Type | TypeSymbol }} idx int } func (itr *_{{ .Type | TypeSymbol }}__MapItr) Next() (k datamodel.Node, v datamodel.Node, _ error) { if itr.idx >= len(itr.n.t) { return nil, nil, datamodel.ErrIteratorOverread{} } x := &itr.n.t[itr.idx] k = &x.k {{- if .Type.ValueIsNullable }} switch x.v.m { case schema.Maybe_Null: v = datamodel.Null case schema.Maybe_Value: v = {{ if not (MaybeUsesPtr .Type.ValueType) }}&{{end}}x.v.v } {{- else}} v = &x.v {{- end}} itr.idx++ return } func (itr *_{{ .Type | TypeSymbol }}__MapItr) Done() bool { return itr.idx >= len(itr.n.t) } `, w, g.AdjCfg, g) } func (g mapGenerator) EmitNodeMethodLength(w io.Writer) { doTemplate(` func (n {{ .Type | TypeSymbol }}) Length() int64 { return int64(len(n.t)) } `, w, g.AdjCfg, g) } func (g mapGenerator) EmitNodeMethodPrototype(w io.Writer) { emitNodeMethodPrototype_typical(w, g.AdjCfg, g) } func (g mapGenerator) EmitNodePrototypeType(w io.Writer) { emitNodePrototypeType_typical(w, g.AdjCfg, g) } // --- NodeBuilder and NodeAssembler ---> func (g mapGenerator) GetNodeBuilderGenerator() NodeBuilderGenerator { return mapBuilderGenerator{ g.AdjCfg, mixins.MapAssemblerTraits{ PkgName: g.PkgName, TypeName: g.TypeName, AppliedPrefix: "_" + g.AdjCfg.TypeSymbol(g.Type) + "__", }, g.PkgName, g.Type, } } type mapBuilderGenerator struct { AdjCfg *AdjunctCfg mixins.MapAssemblerTraits PkgName string Type *schema.TypeMap } func (mapBuilderGenerator) IsRepr() bool { return false } // hint used in some generalized templates. func (g mapBuilderGenerator) EmitNodeBuilderType(w io.Writer) { emitEmitNodeBuilderType_typical(w, g.AdjCfg, g) } func (g mapBuilderGenerator) EmitNodeBuilderMethods(w io.Writer) { emitNodeBuilderMethods_typical(w, g.AdjCfg, g) } func (g mapBuilderGenerator) EmitNodeAssemblerType(w io.Writer) { // - 'w' is the "**w**ip" pointer. // - 'm' is the **m**aybe which communicates our completeness to the parent if we're a child assembler. // - 'state' is what it says on the tin. this is used for the map state (the broad transitions between null, start-map, and finish are handled by 'm' for consistency.) // - there's no equivalent of the 'f' (**f**ocused next) field in struct assemblers -- that's implicitly the last row of the 'w.t'. // // - 'cm' is **c**hild **m**aybe and is used for the completion message from children. // It's used for values if values aren't allowed to be nullable and thus don't have their own per-value maybe slot we can use. // It's always used for key assembly, since keys are never allowed to be nullable and thus etc. // - 'ka' and 'va' are the key assembler and value assembler respectively. // Perhaps surprisingly, we can get away with using the assemblers for each type just straight up, no wrappers necessary; // All of the required magic is handled through maybe pointers and some tidy methods used during state transitions. doTemplate(` type _{{ .Type | TypeSymbol }}__Assembler struct { w *_{{ .Type | TypeSymbol }} m *schema.Maybe state maState cm schema.Maybe ka _{{ .Type.KeyType | TypeSymbol }}__Assembler va _{{ .Type.ValueType | TypeSymbol }}__Assembler } func (na *_{{ .Type | TypeSymbol }}__Assembler) reset() { na.state = maState_initial na.ka.reset() na.va.reset() } `, w, g.AdjCfg, g) } func (g mapBuilderGenerator) EmitNodeAssemblerMethodBeginMap(w io.Writer) { emitNodeAssemblerMethodBeginMap_mapoid(w, g.AdjCfg, g) } func (g mapBuilderGenerator) EmitNodeAssemblerMethodAssignNull(w io.Writer) { emitNodeAssemblerMethodAssignNull_recursive(w, g.AdjCfg, g) } func (g mapBuilderGenerator) EmitNodeAssemblerMethodAssignNode(w io.Writer) { emitNodeAssemblerMethodAssignNode_mapoid(w, g.AdjCfg, g) } func (g mapBuilderGenerator) EmitNodeAssemblerOtherBits(w io.Writer) { emitNodeAssemblerHelper_mapoid_keyTidyHelper(w, g.AdjCfg, g) emitNodeAssemblerHelper_mapoid_valueTidyHelper(w, g.AdjCfg, g) emitNodeAssemblerHelper_mapoid_mapAssemblerMethods(w, g.AdjCfg, g) } ================================================ FILE: schema/gen/go/genMapReprMap.go ================================================ package gengo import ( "io" "github.com/ipld/go-ipld-prime/schema" "github.com/ipld/go-ipld-prime/schema/gen/go/mixins" ) var _ TypeGenerator = &mapReprMapGenerator{} func NewMapReprMapGenerator(pkgName string, typ *schema.TypeMap, adjCfg *AdjunctCfg) TypeGenerator { return mapReprMapGenerator{ mapGenerator{ adjCfg, mixins.MapTraits{ PkgName: pkgName, TypeName: string(typ.Name()), TypeSymbol: adjCfg.TypeSymbol(typ), }, pkgName, typ, }, } } type mapReprMapGenerator struct { mapGenerator } func (g mapReprMapGenerator) GetRepresentationNodeGen() NodeGenerator { return mapReprMapReprGenerator{ g.AdjCfg, mixins.MapTraits{ PkgName: g.PkgName, TypeName: string(g.Type.Name()) + ".Repr", TypeSymbol: "_" + g.AdjCfg.TypeSymbol(g.Type) + "__Repr", }, g.PkgName, g.Type, } } type mapReprMapReprGenerator struct { AdjCfg *AdjunctCfg mixins.MapTraits PkgName string Type *schema.TypeMap } func (mapReprMapReprGenerator) IsRepr() bool { return true } // hint used in some generalized templates. func (g mapReprMapReprGenerator) EmitNodeType(w io.Writer) { // Even though this is a "natural" representation... we need a new type here, // because maps are recursive, and so all our functions that access // children need to remember to return the representation node of those child values. // It's still structurally the same, though (and we'll be able to cast in the methodset pattern). // Error-thunking methods also have a different string in their error, so those are unique even if they don't seem particularly interesting. doTemplate(` type _{{ .Type | TypeSymbol }}__Repr _{{ .Type | TypeSymbol }} `, w, g.AdjCfg, g) } func (g mapReprMapReprGenerator) EmitNodeTypeAssertions(w io.Writer) { doTemplate(` var _ datamodel.Node = &_{{ .Type | TypeSymbol }}__Repr{} `, w, g.AdjCfg, g) } func (g mapReprMapReprGenerator) EmitNodeMethodLookupByString(w io.Writer) { doTemplate(` func (nr *_{{ .Type | TypeSymbol }}__Repr) LookupByString(k string) (datamodel.Node, error) { v, err := ({{ .Type | TypeSymbol }})(nr).LookupByString(k) if err != nil || v == datamodel.Null { return v, err } return v.({{ .Type.ValueType | TypeSymbol}}).Representation(), nil } `, w, g.AdjCfg, g) } func (g mapReprMapReprGenerator) EmitNodeMethodLookupByNode(w io.Writer) { // Null is also already a branch in the method we're calling; hopefully the compiler inlines and sees this and DTRT. // REVIEW: these unchecked casts are definitely safe at compile time, but I'm not sure if the compiler considers that provable, // so we should investigate if there's any runtime checks injected here that waste time. If so: write this with more gsloc to avoid :( doTemplate(` func (nr *_{{ .Type | TypeSymbol }}__Repr) LookupByNode(k datamodel.Node) (datamodel.Node, error) { v, err := ({{ .Type | TypeSymbol }})(nr).LookupByNode(k) if err != nil || v == datamodel.Null { return v, err } return v.({{ .Type.ValueType | TypeSymbol}}).Representation(), nil } `, w, g.AdjCfg, g) } func (g mapReprMapReprGenerator) EmitNodeMethodMapIterator(w io.Writer) { // FUTURE: trying to get this to share the preallocated memory if we get iterators wedged into their node slab will be ... fun. doTemplate(` func (nr *_{{ .Type | TypeSymbol }}__Repr) MapIterator() datamodel.MapIterator { return &_{{ .Type | TypeSymbol }}__ReprMapItr{({{ .Type | TypeSymbol }})(nr), 0} } type _{{ .Type | TypeSymbol }}__ReprMapItr _{{ .Type | TypeSymbol }}__MapItr func (itr *_{{ .Type | TypeSymbol }}__ReprMapItr) Next() (k datamodel.Node, v datamodel.Node, err error) { k, v, err = (*_{{ .Type | TypeSymbol }}__MapItr)(itr).Next() if err != nil || v == datamodel.Null { return } return k, v.({{ .Type.ValueType | TypeSymbol}}).Representation(), nil } func (itr *_{{ .Type | TypeSymbol }}__ReprMapItr) Done() bool { return (*_{{ .Type | TypeSymbol }}__MapItr)(itr).Done() } `, w, g.AdjCfg, g) } func (g mapReprMapReprGenerator) EmitNodeMethodLength(w io.Writer) { doTemplate(` func (rn *_{{ .Type | TypeSymbol }}__Repr) Length() int64 { return int64(len(rn.t)) } `, w, g.AdjCfg, g) } func (g mapReprMapReprGenerator) EmitNodeMethodPrototype(w io.Writer) { emitNodeMethodPrototype_typical(w, g.AdjCfg, g) } func (g mapReprMapReprGenerator) EmitNodePrototypeType(w io.Writer) { emitNodePrototypeType_typical(w, g.AdjCfg, g) } // --- NodeBuilder and NodeAssembler ---> func (g mapReprMapReprGenerator) GetNodeBuilderGenerator() NodeBuilderGenerator { return mapReprMapReprBuilderGenerator{ g.AdjCfg, mixins.MapAssemblerTraits{ PkgName: g.PkgName, TypeName: g.TypeName, AppliedPrefix: "_" + g.AdjCfg.TypeSymbol(g.Type) + "__Repr", }, g.PkgName, g.Type, } } type mapReprMapReprBuilderGenerator struct { AdjCfg *AdjunctCfg mixins.MapAssemblerTraits PkgName string Type *schema.TypeMap } func (mapReprMapReprBuilderGenerator) IsRepr() bool { return true } // hint used in some generalized templates. func (g mapReprMapReprBuilderGenerator) EmitNodeBuilderType(w io.Writer) { emitEmitNodeBuilderType_typical(w, g.AdjCfg, g) } func (g mapReprMapReprBuilderGenerator) EmitNodeBuilderMethods(w io.Writer) { emitNodeBuilderMethods_typical(w, g.AdjCfg, g) } func (g mapReprMapReprBuilderGenerator) EmitNodeAssemblerType(w io.Writer) { // - 'w' is the "**w**ip" pointer. // - 'm' is the **m**aybe which communicates our completeness to the parent if we're a child assembler. // - 'state' is what it says on the tin. this is used for the map state (the broad transitions between null, start-map, and finish are handled by 'm' for consistency.) // - there's no equivalent of the 'f' (**f**ocused next) field in struct assemblers -- that's implicitly the last row of the 'w.t'. // // - 'cm' is **c**hild **m**aybe and is used for the completion message from children. // It's used for values if values aren't allowed to be nullable and thus don't have their own per-value maybe slot we can use. // It's always used for key assembly, since keys are never allowed to be nullable and thus etc. // - 'ka' and 'va' are the key assembler and value assembler respectively. // Perhaps surprisingly, we can get away with using the assemblers for each type just straight up, no wrappers necessary; // All of the required magic is handled through maybe pointers and some tidy methods used during state transitions. // // Note that this textually similar to the type-level assembler, but because it embeds the repr assembler for the child types, // it might be *significantly* different in size and memory layout in that trailing part of the struct. doTemplate(` type _{{ .Type | TypeSymbol }}__ReprAssembler struct { w *_{{ .Type | TypeSymbol }} m *schema.Maybe state maState cm schema.Maybe ka _{{ .Type.KeyType | TypeSymbol }}__ReprAssembler va _{{ .Type.ValueType | TypeSymbol }}__ReprAssembler } func (na *_{{ .Type | TypeSymbol }}__ReprAssembler) reset() { na.state = maState_initial na.ka.reset() na.va.reset() } `, w, g.AdjCfg, g) } func (g mapReprMapReprBuilderGenerator) EmitNodeAssemblerMethodBeginMap(w io.Writer) { emitNodeAssemblerMethodBeginMap_mapoid(w, g.AdjCfg, g) } func (g mapReprMapReprBuilderGenerator) EmitNodeAssemblerMethodAssignNull(w io.Writer) { emitNodeAssemblerMethodAssignNull_recursive(w, g.AdjCfg, g) } func (g mapReprMapReprBuilderGenerator) EmitNodeAssemblerMethodAssignNode(w io.Writer) { emitNodeAssemblerMethodAssignNode_mapoid(w, g.AdjCfg, g) } func (g mapReprMapReprBuilderGenerator) EmitNodeAssemblerOtherBits(w io.Writer) { emitNodeAssemblerHelper_mapoid_keyTidyHelper(w, g.AdjCfg, g) emitNodeAssemblerHelper_mapoid_valueTidyHelper(w, g.AdjCfg, g) emitNodeAssemblerHelper_mapoid_mapAssemblerMethods(w, g.AdjCfg, g) } ================================================ FILE: schema/gen/go/genString.go ================================================ package gengo import ( "io" "github.com/ipld/go-ipld-prime/schema" "github.com/ipld/go-ipld-prime/schema/gen/go/mixins" ) type stringGenerator struct { AdjCfg *AdjunctCfg mixins.StringTraits PkgName string Type *schema.TypeString } func (stringGenerator) IsRepr() bool { return false } // hint used in some generalized templates. // --- native content and specializations ---> func (g stringGenerator) EmitNativeType(w io.Writer) { emitNativeType_scalar(w, g.AdjCfg, g) } func (g stringGenerator) EmitNativeAccessors(w io.Writer) { emitNativeAccessors_scalar(w, g.AdjCfg, g) } func (g stringGenerator) EmitNativeBuilder(w io.Writer) { // Generate a single-step construction function -- this is easy to do for a scalar, // and all representations of scalar kind can be expected to have a method like this. // The function is attached to the NodePrototype for convenient namespacing; // it needs no new memory, so it would be inappropriate to attach to the builder or assembler. // The function is directly used internally by anything else that might involve recursive destructuring on the same scalar kind // (for example, structs using stringjoin strategies that have one of this type as a field, etc). // FUTURE: should engage validation flow. doTemplate(` func (_{{ .Type | TypeSymbol }}__Prototype) fromString(w *_{{ .Type | TypeSymbol }}, v string) error { *w = _{{ .Type | TypeSymbol }}{v} return nil } `, w, g.AdjCfg, g) // And generate a publicly exported version of that single-step constructor, too. // (Just don't expose the details about allocation, because you can't meaningfully use that from outside the package.) emitNativeBuilder_scalar(w, g.AdjCfg, g) } func (g stringGenerator) EmitNativeMaybe(w io.Writer) { emitNativeMaybe(w, g.AdjCfg, g) } // --- type info ---> func (g stringGenerator) EmitTypeConst(w io.Writer) { doTemplate(` // TODO EmitTypeConst `, w, g.AdjCfg, g) } // --- TypedNode interface satisfaction ---> func (g stringGenerator) EmitTypedNodeMethodType(w io.Writer) { doTemplate(` func ({{ .Type | TypeSymbol }}) Type() schema.Type { return nil /*TODO:typelit*/ } `, w, g.AdjCfg, g) } func (g stringGenerator) EmitTypedNodeMethodRepresentation(w io.Writer) { emitTypicalTypedNodeMethodRepresentation(w, g.AdjCfg, g) } // --- Node interface satisfaction ---> func (g stringGenerator) EmitNodeType(w io.Writer) { // No additional types needed. Methods all attach to the native type. } func (g stringGenerator) EmitNodeTypeAssertions(w io.Writer) { emitNodeTypeAssertions_typical(w, g.AdjCfg, g) } func (g stringGenerator) EmitNodeMethodAsString(w io.Writer) { emitNodeMethodAsKind_scalar(w, g.AdjCfg, g) } func (g stringGenerator) EmitNodeMethodPrototype(w io.Writer) { emitNodeMethodPrototype_typical(w, g.AdjCfg, g) } func (g stringGenerator) EmitNodePrototypeType(w io.Writer) { emitNodePrototypeType_typical(w, g.AdjCfg, g) } // --- NodeBuilder and NodeAssembler ---> func (g stringGenerator) GetNodeBuilderGenerator() NodeBuilderGenerator { return stringBuilderGenerator{ g.AdjCfg, mixins.StringAssemblerTraits{ PkgName: g.PkgName, TypeName: g.TypeName, AppliedPrefix: "_" + g.AdjCfg.TypeSymbol(g.Type) + "__", }, g.PkgName, g.Type, } } type stringBuilderGenerator struct { AdjCfg *AdjunctCfg mixins.StringAssemblerTraits PkgName string Type *schema.TypeString } func (stringBuilderGenerator) IsRepr() bool { return false } // hint used in some generalized templates. func (g stringBuilderGenerator) EmitNodeBuilderType(w io.Writer) { emitEmitNodeBuilderType_typical(w, g.AdjCfg, g) } func (g stringBuilderGenerator) EmitNodeBuilderMethods(w io.Writer) { emitNodeBuilderMethods_typical(w, g.AdjCfg, g) } func (g stringBuilderGenerator) EmitNodeAssemblerType(w io.Writer) { emitNodeAssemblerType_scalar(w, g.AdjCfg, g) } func (g stringBuilderGenerator) EmitNodeAssemblerMethodAssignNull(w io.Writer) { emitNodeAssemblerMethodAssignNull_scalar(w, g.AdjCfg, g) } func (g stringBuilderGenerator) EmitNodeAssemblerMethodAssignString(w io.Writer) { emitNodeAssemblerMethodAssignKind_scalar(w, g.AdjCfg, g) } func (g stringBuilderGenerator) EmitNodeAssemblerMethodAssignNode(w io.Writer) { emitNodeAssemblerMethodAssignNode_scalar(w, g.AdjCfg, g) } func (g stringBuilderGenerator) EmitNodeAssemblerOtherBits(w io.Writer) { // Nothing needed here for string kinds. } ================================================ FILE: schema/gen/go/genStringReprString.go ================================================ package gengo import ( "io" "github.com/ipld/go-ipld-prime/schema" "github.com/ipld/go-ipld-prime/schema/gen/go/mixins" ) var _ TypeGenerator = &stringReprStringGenerator{} func NewStringReprStringGenerator(pkgName string, typ *schema.TypeString, adjCfg *AdjunctCfg) TypeGenerator { return stringReprStringGenerator{ stringGenerator{ adjCfg, mixins.StringTraits{ PkgName: pkgName, TypeName: string(typ.Name()), TypeSymbol: adjCfg.TypeSymbol(typ), }, pkgName, typ, }, } } type stringReprStringGenerator struct { stringGenerator } func (g stringReprStringGenerator) GetRepresentationNodeGen() NodeGenerator { return stringReprStringReprGenerator{ g.AdjCfg, g.Type, } } type stringReprStringReprGenerator struct { AdjCfg *AdjunctCfg Type *schema.TypeString } func (g stringReprStringReprGenerator) EmitNodeType(w io.Writer) { // Since this is a "natural" representation... there's just a type alias here. // No new functions are necessary. doTemplate(` type _{{ .Type | TypeSymbol }}__Repr = _{{ .Type | TypeSymbol }} `, w, g.AdjCfg, g) } func (g stringReprStringReprGenerator) EmitNodeTypeAssertions(w io.Writer) { doTemplate(` var _ datamodel.Node = &_{{ .Type | TypeSymbol }}__Repr{} `, w, g.AdjCfg, g) } func (stringReprStringReprGenerator) EmitNodeMethodKind(io.Writer) {} func (stringReprStringReprGenerator) EmitNodeMethodLookupByString(io.Writer) {} func (stringReprStringReprGenerator) EmitNodeMethodLookupByNode(io.Writer) {} func (stringReprStringReprGenerator) EmitNodeMethodLookupByIndex(io.Writer) {} func (stringReprStringReprGenerator) EmitNodeMethodLookupBySegment(io.Writer) {} func (stringReprStringReprGenerator) EmitNodeMethodMapIterator(io.Writer) {} func (stringReprStringReprGenerator) EmitNodeMethodListIterator(io.Writer) {} func (stringReprStringReprGenerator) EmitNodeMethodLength(io.Writer) {} func (stringReprStringReprGenerator) EmitNodeMethodIsAbsent(io.Writer) {} func (stringReprStringReprGenerator) EmitNodeMethodIsNull(io.Writer) {} func (stringReprStringReprGenerator) EmitNodeMethodAsBool(io.Writer) {} func (stringReprStringReprGenerator) EmitNodeMethodAsInt(io.Writer) {} func (stringReprStringReprGenerator) EmitNodeMethodAsFloat(io.Writer) {} func (stringReprStringReprGenerator) EmitNodeMethodAsString(io.Writer) {} func (stringReprStringReprGenerator) EmitNodeMethodAsBytes(io.Writer) {} func (stringReprStringReprGenerator) EmitNodeMethodAsLink(io.Writer) {} func (stringReprStringReprGenerator) EmitNodeMethodPrototype(io.Writer) {} func (g stringReprStringReprGenerator) EmitNodePrototypeType(w io.Writer) { // Since this is a "natural" representation... there's just a type alias here. // No new functions are necessary. doTemplate(` type _{{ .Type | TypeSymbol }}__ReprPrototype = _{{ .Type | TypeSymbol }}__Prototype `, w, g.AdjCfg, g) } func (g stringReprStringReprGenerator) GetNodeBuilderGenerator() NodeBuilderGenerator { return stringReprStringReprBuilderGenerator(g) } type stringReprStringReprBuilderGenerator struct { AdjCfg *AdjunctCfg Type *schema.TypeString } func (stringReprStringReprBuilderGenerator) EmitNodeBuilderType(io.Writer) {} func (g stringReprStringReprBuilderGenerator) EmitNodeBuilderMethods(w io.Writer) {} func (g stringReprStringReprBuilderGenerator) EmitNodeAssemblerType(w io.Writer) { // Since this is a "natural" representation... there's just a type alias here. // No new functions are necessary. doTemplate(` type _{{ .Type | TypeSymbol }}__ReprAssembler = _{{ .Type | TypeSymbol }}__Assembler `, w, g.AdjCfg, g) } func (stringReprStringReprBuilderGenerator) EmitNodeAssemblerMethodBeginMap(io.Writer) {} func (stringReprStringReprBuilderGenerator) EmitNodeAssemblerMethodBeginList(io.Writer) {} func (stringReprStringReprBuilderGenerator) EmitNodeAssemblerMethodAssignNull(io.Writer) {} func (stringReprStringReprBuilderGenerator) EmitNodeAssemblerMethodAssignBool(io.Writer) {} func (stringReprStringReprBuilderGenerator) EmitNodeAssemblerMethodAssignInt(io.Writer) {} func (stringReprStringReprBuilderGenerator) EmitNodeAssemblerMethodAssignFloat(io.Writer) {} func (stringReprStringReprBuilderGenerator) EmitNodeAssemblerMethodAssignString(io.Writer) {} func (stringReprStringReprBuilderGenerator) EmitNodeAssemblerMethodAssignBytes(io.Writer) {} func (stringReprStringReprBuilderGenerator) EmitNodeAssemblerMethodAssignLink(io.Writer) {} func (stringReprStringReprBuilderGenerator) EmitNodeAssemblerMethodAssignNode(io.Writer) {} func (stringReprStringReprBuilderGenerator) EmitNodeAssemblerMethodPrototype(io.Writer) {} func (stringReprStringReprBuilderGenerator) EmitNodeAssemblerOtherBits(io.Writer) {} ================================================ FILE: schema/gen/go/genStruct.go ================================================ package gengo import ( "io" "github.com/ipld/go-ipld-prime/schema" "github.com/ipld/go-ipld-prime/schema/gen/go/mixins" ) type structGenerator struct { AdjCfg *AdjunctCfg mixins.MapTraits PkgName string Type *schema.TypeStruct } func (structGenerator) IsRepr() bool { return false } // hint used in some generalized templates. // --- native content and specializations ---> func (g structGenerator) EmitNativeType(w io.Writer) { doTemplate(` {{- if Comments -}} // {{ .Type | TypeSymbol }} matches the IPLD Schema type "{{ .Type.Name }}". It has {{ .Type.TypeKind }} type-kind, and may be interrogated like {{ .Kind }} kind. {{- end}} type {{ .Type | TypeSymbol }} = *_{{ .Type | TypeSymbol }} type _{{ .Type | TypeSymbol }} struct { {{- range $field := .Type.Fields}} {{ $field | FieldSymbolLower }} _{{ $field.Type | TypeSymbol }}{{if $field.IsMaybe }}__Maybe{{end}} {{- end}} } `, w, g.AdjCfg, g) } func (g structGenerator) EmitNativeAccessors(w io.Writer) { doTemplate(` {{- $type := .Type -}} {{- /* ranging modifies dot, unhelpfully */ -}} {{- range $field := .Type.Fields }} func (n _{{ $type | TypeSymbol }}) Field{{ $field | FieldSymbolUpper }}() {{ if $field.IsMaybe }}Maybe{{end}}{{ $field.Type | TypeSymbol }} { return &n.{{ $field | FieldSymbolLower }} } {{- end}} `, w, g.AdjCfg, g) } func (g structGenerator) EmitNativeBuilder(w io.Writer) { // Unclear what, if anything, goes here. } func (g structGenerator) EmitNativeMaybe(w io.Writer) { emitNativeMaybe(w, g.AdjCfg, g) } // --- type info ---> func (g structGenerator) EmitTypeConst(w io.Writer) { doTemplate(` // TODO EmitTypeConst `, w, g.AdjCfg, g) } // --- TypedNode interface satisfaction ---> func (g structGenerator) EmitTypedNodeMethodType(w io.Writer) { doTemplate(` func ({{ .Type | TypeSymbol }}) Type() schema.Type { return nil /*TODO:typelit*/ } `, w, g.AdjCfg, g) } func (g structGenerator) EmitTypedNodeMethodRepresentation(w io.Writer) { emitTypicalTypedNodeMethodRepresentation(w, g.AdjCfg, g) } // --- Node interface satisfaction ---> func (g structGenerator) EmitNodeType(w io.Writer) { // No additional types needed. Methods all attach to the native type. // We do, however, want some constants for our fields; // they'll make iterators able to work faster. So let's emit those. doTemplate(` var ( {{- $type := .Type -}} {{- /* ranging modifies dot, unhelpfully */ -}} {{- range $field := .Type.Fields }} fieldName__{{ $type | TypeSymbol }}_{{ $field | FieldSymbolUpper }} = _String{"{{ $field.Name }}"} {{- end }} ) `, w, g.AdjCfg, g) } func (g structGenerator) EmitNodeTypeAssertions(w io.Writer) { emitNodeTypeAssertions_typical(w, g.AdjCfg, g) } func (g structGenerator) EmitNodeMethodLookupByString(w io.Writer) { doTemplate(` func (n {{ .Type | TypeSymbol }}) LookupByString(key string) (datamodel.Node, error) { switch key { {{- range $field := .Type.Fields }} case "{{ $field.Name }}": {{- if $field.IsOptional }} if n.{{ $field | FieldSymbolLower }}.m == schema.Maybe_Absent { return datamodel.Absent, nil } {{- end}} {{- if $field.IsNullable }} if n.{{ $field | FieldSymbolLower }}.m == schema.Maybe_Null { return datamodel.Null, nil } {{- end}} {{- if $field.IsMaybe }} return {{if not (MaybeUsesPtr $field.Type) }}&{{end}}n.{{ $field | FieldSymbolLower }}.v, nil {{- else}} return &n.{{ $field | FieldSymbolLower }}, nil {{- end}} {{- end}} default: return nil, schema.ErrNoSuchField{Type: nil /*TODO*/, Field: datamodel.PathSegmentOfString(key)} } } `, w, g.AdjCfg, g) } func (g structGenerator) EmitNodeMethodLookupByNode(w io.Writer) { doTemplate(` func (n {{ .Type | TypeSymbol }}) LookupByNode(key datamodel.Node) (datamodel.Node, error) { ks, err := key.AsString() if err != nil { return nil, err } return n.LookupByString(ks) } `, w, g.AdjCfg, g) } func (g structGenerator) EmitNodeMethodMapIterator(w io.Writer) { // Note that the typed iterator will report absent fields. // The representation iterator (if has one) however will skip those. doTemplate(` func (n {{ .Type | TypeSymbol }}) MapIterator() datamodel.MapIterator { return &_{{ .Type | TypeSymbol }}__MapItr{n, 0} } type _{{ .Type | TypeSymbol }}__MapItr struct { n {{ .Type | TypeSymbol }} idx int } func (itr *_{{ .Type | TypeSymbol }}__MapItr) Next() (k datamodel.Node, v datamodel.Node, _ error) { {{- if not .Type.Fields }} return nil, nil, datamodel.ErrIteratorOverread{} {{ else -}} if itr.idx >= {{ len .Type.Fields }} { return nil, nil, datamodel.ErrIteratorOverread{} } switch itr.idx { {{- $type := .Type -}} {{- /* ranging modifies dot, unhelpfully */ -}} {{- range $i, $field := .Type.Fields }} case {{ $i }}: k = &fieldName__{{ $type | TypeSymbol }}_{{ $field | FieldSymbolUpper }} {{- if $field.IsOptional }} if itr.n.{{ $field | FieldSymbolLower }}.m == schema.Maybe_Absent { v = datamodel.Absent break } {{- end}} {{- if $field.IsNullable }} if itr.n.{{ $field | FieldSymbolLower }}.m == schema.Maybe_Null { v = datamodel.Null break } {{- end}} {{- if $field.IsMaybe }} v = {{if not (MaybeUsesPtr $field.Type) }}&{{end}}itr.n.{{ $field | FieldSymbolLower}}.v {{- else}} v = &itr.n.{{ $field | FieldSymbolLower}} {{- end}} {{- end}} default: panic("unreachable") } itr.idx++ return {{- end}} } func (itr *_{{ .Type | TypeSymbol }}__MapItr) Done() bool { return itr.idx >= {{ len .Type.Fields }} } `, w, g.AdjCfg, g) } func (g structGenerator) EmitNodeMethodLength(w io.Writer) { doTemplate(` func ({{ .Type | TypeSymbol }}) Length() int64 { return {{ len .Type.Fields }} } `, w, g.AdjCfg, g) } func (g structGenerator) EmitNodeMethodPrototype(w io.Writer) { emitNodeMethodPrototype_typical(w, g.AdjCfg, g) } func (g structGenerator) EmitNodePrototypeType(w io.Writer) { emitNodePrototypeType_typical(w, g.AdjCfg, g) } // --- NodeBuilder and NodeAssembler ---> func (g structGenerator) GetNodeBuilderGenerator() NodeBuilderGenerator { return structBuilderGenerator{ g.AdjCfg, mixins.MapAssemblerTraits{ PkgName: g.PkgName, TypeName: g.TypeName, AppliedPrefix: "_" + g.AdjCfg.TypeSymbol(g.Type) + "__", }, g.PkgName, g.Type, } } type structBuilderGenerator struct { AdjCfg *AdjunctCfg mixins.MapAssemblerTraits PkgName string Type *schema.TypeStruct } func (structBuilderGenerator) IsRepr() bool { return false } // hint used in some generalized templates. func (g structBuilderGenerator) EmitNodeBuilderType(w io.Writer) { emitEmitNodeBuilderType_typical(w, g.AdjCfg, g) } func (g structBuilderGenerator) EmitNodeBuilderMethods(w io.Writer) { emitNodeBuilderMethods_typical(w, g.AdjCfg, g) } func (g structBuilderGenerator) EmitNodeAssemblerType(w io.Writer) { // - 'w' is the "**w**ip" pointer. // - 'm' is the **m**aybe which communicates our completeness to the parent if we're a child assembler. // - 'state' is what it says on the tin. this is used for the map state (the broad transitions between null, start-map, and finish are handled by 'm' for consistency.) // - 's' is a bitfield for what's been **s**et. // - 'f' is the **f**ocused field that will be assembled next. // // - 'cm' is **c**hild **m**aybe and is used for the completion message from children that aren't allowed to be nullable (for those that are, their own maybe.m is used). // ('cm' could be elided for structs where all fields are maybes. trivial but not yet implemented.) // - the 'ca_*' fields embed **c**hild **a**ssemblers -- these are embedded so we can yield pointers to them without causing new allocations. doTemplate(` type _{{ .Type | TypeSymbol }}__Assembler struct { w *_{{ .Type | TypeSymbol }} m *schema.Maybe state maState s int f int cm schema.Maybe {{range $field := .Type.Fields -}} ca_{{ $field | FieldSymbolLower }} _{{ $field.Type | TypeSymbol }}__Assembler {{end -}} } func (na *_{{ .Type | TypeSymbol }}__Assembler) reset() { na.state = maState_initial na.s = 0 {{- range $field := .Type.Fields }} na.ca_{{ $field | FieldSymbolLower }}.reset() {{- end}} } var ( {{- $type := .Type -}} {{- /* ranging modifies dot, unhelpfully */ -}} {{- range $i, $field := .Type.Fields }} fieldBit__{{ $type | TypeSymbol }}_{{ $field | FieldSymbolUpper }} = 1 << {{ $i }} {{- end}} fieldBits__{{ $type | TypeSymbol }}_sufficient = 0 {{- range $i, $field := .Type.Fields }}{{if not $field.IsOptional }} + 1 << {{ $i }}{{end}}{{end}} ) `, w, g.AdjCfg, g) } func (g structBuilderGenerator) EmitNodeAssemblerMethodBeginMap(w io.Writer) { emitNodeAssemblerMethodBeginMap_strictoid(w, g.AdjCfg, g) } func (g structBuilderGenerator) EmitNodeAssemblerMethodAssignNull(w io.Writer) { emitNodeAssemblerMethodAssignNull_recursive(w, g.AdjCfg, g) } func (g structBuilderGenerator) EmitNodeAssemblerMethodAssignNode(w io.Writer) { // AssignNode goes through three phases: // 1. is it null? Jump over to AssignNull (which may or may not reject it). // 2. is it our own type? Handle specially -- we might be able to do efficient things. // 3. is it the right kind to morph into us? Do so. // // We do not set m=midvalue in phase 3 -- it shouldn't matter unless you're trying to pull off concurrent access, which is wrong and unsafe regardless. doTemplate(` func (na *_{{ .Type | TypeSymbol }}__Assembler) AssignNode(v datamodel.Node) error { if v.IsNull() { return na.AssignNull() } if v2, ok := v.(*_{{ .Type | TypeSymbol }}); ok { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") case midvalue: panic("invalid state: cannot assign null into an assembler that's already begun working on recursive structures!") } {{- if .Type | MaybeUsesPtr }} if na.w == nil { na.w = v2 *na.m = schema.Maybe_Value return nil } {{- end}} *na.w = *v2 *na.m = schema.Maybe_Value return nil } if v.Kind() != datamodel.Kind_Map { return datamodel.ErrWrongKind{TypeName: "{{ .PkgName }}.{{ .Type.Name }}", MethodName: "AssignNode", AppropriateKind: datamodel.KindSet_JustMap, ActualKind: v.Kind()} } itr := v.MapIterator() for !itr.Done() { k, v, err := itr.Next() if err != nil { return err } if err := na.AssembleKey().AssignNode(k); err != nil { return err } if err := na.AssembleValue().AssignNode(v); err != nil { return err } } return na.Finish() } `, w, g.AdjCfg, g) } func (g structBuilderGenerator) EmitNodeAssemblerOtherBits(w io.Writer) { g.emitMapAssemblerChildTidyHelper(w) g.emitMapAssemblerMethods(w) g.emitKeyAssembler(w) } func (g structBuilderGenerator) emitMapAssemblerChildTidyHelper(w io.Writer) { // This function attempts to clean up the state machine to acknolwedge child assembly finish. // If the child was finished and we just collected it, return true and update state to maState_initial. // Otherwise, if it wasn't done, return false; // and the caller is almost certain to emit an error momentarily. // The function will only be called when the current state is maState_midValue. // (In general, the idea is that if the user is doing things correctly, // this function will only be called when the child is in fact finished.) // Most of the logic here is about nullables and not optionals, // because if you're an optional that's absent, you never got to value assembly. // There's still one branch for optionals, though, because they have a different residence for 'm' just as nullables do. // Child assemblers are expected to control their own state machines; // for values that have maybes, we never change their maybe state again, so the usual logic should hold; // for values that don't have maybes (and thus share 'cm')... // We don't bother to nil their 'm' pointer; the worst that can happen is an over-held assembler for that field // can make a bizarre and broken transition for a subsequent field, which will result in very ugly errors, but isn't unsafe per se. // We do nil their 'w' pointer, though: we don't want a set to that able to leak in later if we're on the way to Finish! doTemplate(` func (ma *_{{ .Type | TypeSymbol }}__Assembler) valueFinishTidy() bool { switch ma.f { {{- range $i, $field := .Type.Fields }} case {{ $i }}: {{- if $field.IsNullable }} switch ma.w.{{ $field | FieldSymbolLower }}.m { case schema.Maybe_Null: ma.state = maState_initial return true case schema.Maybe_Value: {{- if (MaybeUsesPtr $field.Type) }} ma.w.{{ $field | FieldSymbolLower }}.v = ma.ca_{{ $field | FieldSymbolLower }}.w {{- end}} ma.state = maState_initial return true default: return false } {{- else if $field.IsOptional }} switch ma.w.{{ $field | FieldSymbolLower }}.m { case schema.Maybe_Value: {{- if (MaybeUsesPtr $field.Type) }} ma.w.{{ $field | FieldSymbolLower }}.v = ma.ca_{{ $field | FieldSymbolLower }}.w {{- end}} ma.state = maState_initial return true default: return false } {{- else}} switch ma.cm { case schema.Maybe_Value: ma.ca_{{ $field | FieldSymbolLower }}.w = nil ma.cm = schema.Maybe_Absent ma.state = maState_initial return true default: return false } {{- end}} {{- end}} default: panic("unreachable") } } `, w, g.AdjCfg, g) } func (g structBuilderGenerator) emitMapAssemblerMethods(w io.Writer) { // FUTURE: some of the setup of the child assemblers could probably be DRY'd up. doTemplate(` func (ma *_{{ .Type | TypeSymbol }}__Assembler) AssembleEntry(k string) (datamodel.NodeAssembler, error) { switch ma.state { case maState_initial: // carry on case maState_midKey: panic("invalid state: AssembleEntry cannot be called when in the middle of assembling another key") case maState_expectValue: panic("invalid state: AssembleEntry cannot be called when expecting start of value assembly") case maState_midValue: if !ma.valueFinishTidy() { panic("invalid state: AssembleEntry cannot be called when in the middle of assembling a value") } // if tidy success: carry on case maState_finished: panic("invalid state: AssembleEntry cannot be called on an assembler that's already finished") } {{- $type := .Type -}} {{- /* ranging modifies dot, unhelpfully */ -}} {{- if .Type.Fields }} switch k { {{- range $i, $field := .Type.Fields }} case "{{ $field.Name }}": if ma.s & fieldBit__{{ $type | TypeSymbol }}_{{ $field | FieldSymbolUpper }} != 0 { return nil, datamodel.ErrRepeatedMapKey{Key: &fieldName__{{ $type | TypeSymbol }}_{{ $field | FieldSymbolUpper }}} } ma.s += fieldBit__{{ $type | TypeSymbol }}_{{ $field | FieldSymbolUpper }} ma.state = maState_midValue ma.f = {{ $i }} {{- if $field.IsMaybe }} ma.ca_{{ $field | FieldSymbolLower }}.w = {{if not (MaybeUsesPtr $field.Type) }}&{{end}}ma.w.{{ $field | FieldSymbolLower }}.v ma.ca_{{ $field | FieldSymbolLower }}.m = &ma.w.{{ $field | FieldSymbolLower }}.m {{- if $field.IsNullable }} ma.w.{{ $field | FieldSymbolLower }}.m = allowNull {{- end}} {{- else}} ma.ca_{{ $field | FieldSymbolLower }}.w = &ma.w.{{ $field | FieldSymbolLower }} ma.ca_{{ $field | FieldSymbolLower }}.m = &ma.cm {{- end}} return &ma.ca_{{ $field | FieldSymbolLower }}, nil {{- end}} } {{- end}} return nil, schema.ErrInvalidKey{TypeName:"{{ .PkgName }}.{{ .Type.Name }}", Key:&_String{k}} } func (ma *_{{ .Type | TypeSymbol }}__Assembler) AssembleKey() datamodel.NodeAssembler { switch ma.state { case maState_initial: // carry on case maState_midKey: panic("invalid state: AssembleKey cannot be called when in the middle of assembling another key") case maState_expectValue: panic("invalid state: AssembleKey cannot be called when expecting start of value assembly") case maState_midValue: if !ma.valueFinishTidy() { panic("invalid state: AssembleKey cannot be called when in the middle of assembling a value") } // if tidy success: carry on case maState_finished: panic("invalid state: AssembleKey cannot be called on an assembler that's already finished") } ma.state = maState_midKey return (*_{{ .Type | TypeSymbol }}__KeyAssembler)(ma) } func (ma *_{{ .Type | TypeSymbol }}__Assembler) AssembleValue() datamodel.NodeAssembler { switch ma.state { case maState_initial: panic("invalid state: AssembleValue cannot be called when no key is primed") case maState_midKey: panic("invalid state: AssembleValue cannot be called when in the middle of assembling a key") case maState_expectValue: // carry on case maState_midValue: panic("invalid state: AssembleValue cannot be called when in the middle of assembling another value") case maState_finished: panic("invalid state: AssembleValue cannot be called on an assembler that's already finished") } ma.state = maState_midValue switch ma.f { {{- range $i, $field := .Type.Fields }} case {{ $i }}: {{- if $field.IsMaybe }} ma.ca_{{ $field | FieldSymbolLower }}.w = {{if not (MaybeUsesPtr $field.Type) }}&{{end}}ma.w.{{ $field | FieldSymbolLower }}.v ma.ca_{{ $field | FieldSymbolLower }}.m = &ma.w.{{ $field | FieldSymbolLower }}.m {{- if $field.IsNullable }} ma.w.{{ $field | FieldSymbolLower }}.m = allowNull {{- end}} {{- else}} ma.ca_{{ $field | FieldSymbolLower }}.w = &ma.w.{{ $field | FieldSymbolLower }} ma.ca_{{ $field | FieldSymbolLower }}.m = &ma.cm {{- end}} return &ma.ca_{{ $field | FieldSymbolLower }} {{- end}} default: panic("unreachable") } } func (ma *_{{ .Type | TypeSymbol }}__Assembler) Finish() error { switch ma.state { case maState_initial: // carry on case maState_midKey: panic("invalid state: Finish cannot be called when in the middle of assembling a key") case maState_expectValue: panic("invalid state: Finish cannot be called when expecting start of value assembly") case maState_midValue: if !ma.valueFinishTidy() { panic("invalid state: Finish cannot be called when in the middle of assembling a value") } // if tidy success: carry on case maState_finished: panic("invalid state: Finish cannot be called on an assembler that's already finished") } if ma.s & fieldBits__{{ $type | TypeSymbol }}_sufficient != fieldBits__{{ $type | TypeSymbol }}_sufficient { err := schema.ErrMissingRequiredField{Missing: make([]string, 0)} {{- range $i, $field := .Type.Fields }} {{- if not $field.IsMaybe}} if ma.s & fieldBit__{{ $type | TypeSymbol }}_{{ $field | FieldSymbolUpper }} == 0 { err.Missing = append(err.Missing, "{{ $field.Name }}") } {{- end}} {{- end}} return err } ma.state = maState_finished *ma.m = schema.Maybe_Value return nil } func (ma *_{{ .Type | TypeSymbol }}__Assembler) KeyPrototype() datamodel.NodePrototype { return _String__Prototype{} } func (ma *_{{ .Type | TypeSymbol }}__Assembler) ValuePrototype(k string) datamodel.NodePrototype { panic("todo structbuilder mapassembler valueprototype") } `, w, g.AdjCfg, g) } func (g structBuilderGenerator) emitKeyAssembler(w io.Writer) { doTemplate(` type _{{ .Type | TypeSymbol }}__KeyAssembler _{{ .Type | TypeSymbol }}__Assembler `, w, g.AdjCfg, g) stubs := mixins.StringAssemblerTraits{ PkgName: g.PkgName, TypeName: g.TypeName + ".KeyAssembler", AppliedPrefix: "_" + g.AdjCfg.TypeSymbol(g.Type) + "__Key", } // This key assembler can disregard any idea of complex keys because it's a struct! // Struct field names must be strings (and quite simple ones at that). stubs.EmitNodeAssemblerMethodBeginMap(w) stubs.EmitNodeAssemblerMethodBeginList(w) stubs.EmitNodeAssemblerMethodAssignNull(w) stubs.EmitNodeAssemblerMethodAssignBool(w) stubs.EmitNodeAssemblerMethodAssignInt(w) stubs.EmitNodeAssemblerMethodAssignFloat(w) doTemplate(` func (ka *_{{ .Type | TypeSymbol }}__KeyAssembler) AssignString(k string) error { if ka.state != maState_midKey { panic("misuse: KeyAssembler held beyond its valid lifetime") } switch k { {{- $type := .Type -}} {{- /* ranging modifies dot, unhelpfully */ -}} {{- range $i, $field := .Type.Fields }} case "{{ $field.Name }}": if ka.s & fieldBit__{{ $type | TypeSymbol }}_{{ $field | FieldSymbolUpper }} != 0 { return datamodel.ErrRepeatedMapKey{Key: &fieldName__{{ $type | TypeSymbol }}_{{ $field | FieldSymbolUpper }}} } ka.s += fieldBit__{{ $type | TypeSymbol }}_{{ $field | FieldSymbolUpper }} ka.state = maState_expectValue ka.f = {{ $i }} return nil {{- end}} default: return schema.ErrInvalidKey{TypeName:"{{ .PkgName }}.{{ .Type.Name }}", Key:&_String{k}} } } `, w, g.AdjCfg, g) stubs.EmitNodeAssemblerMethodAssignBytes(w) stubs.EmitNodeAssemblerMethodAssignLink(w) doTemplate(` func (ka *_{{ .Type | TypeSymbol }}__KeyAssembler) AssignNode(v datamodel.Node) error { if v2, err := v.AsString(); err != nil { return err } else { return ka.AssignString(v2) } } func (_{{ .Type | TypeSymbol }}__KeyAssembler) Prototype() datamodel.NodePrototype { return _String__Prototype{} } `, w, g.AdjCfg, g) } ================================================ FILE: schema/gen/go/genStructReprMap.go ================================================ package gengo import ( "io" "strconv" "github.com/ipld/go-ipld-prime/schema" "github.com/ipld/go-ipld-prime/schema/gen/go/mixins" ) var _ TypeGenerator = &structReprMapGenerator{} func NewStructReprMapGenerator(pkgName string, typ *schema.TypeStruct, adjCfg *AdjunctCfg) TypeGenerator { return structReprMapGenerator{ structGenerator{ adjCfg, mixins.MapTraits{ PkgName: pkgName, TypeName: string(typ.Name()), TypeSymbol: adjCfg.TypeSymbol(typ), }, pkgName, typ, }, } } type structReprMapGenerator struct { structGenerator } func (g structReprMapGenerator) GetRepresentationNodeGen() NodeGenerator { return structReprMapReprGenerator{ g.AdjCfg, mixins.MapTraits{ PkgName: g.PkgName, TypeName: string(g.Type.Name()) + ".Repr", TypeSymbol: "_" + g.AdjCfg.TypeSymbol(g.Type) + "__Repr", }, g.PkgName, g.Type, } } type structReprMapReprGenerator struct { AdjCfg *AdjunctCfg mixins.MapTraits PkgName string Type *schema.TypeStruct } func (structReprMapReprGenerator) IsRepr() bool { return true } // hint used in some generalized templates. func (g structReprMapReprGenerator) EmitNodeType(w io.Writer) { // The type is structurally the same, but will have a different set of methods. doTemplate(` type _{{ .Type | TypeSymbol }}__Repr _{{ .Type | TypeSymbol }} `, w, g.AdjCfg, g) // We do also want some constants for our fields; // they'll make iterators able to work faster. // These might be the same strings as the type-level field names // (in fact, they are, unless renames are used)... but that's fine. // We get simpler code by just doing this unconditionally. doTemplate(` var ( {{- $type := .Type -}} {{- /* ranging modifies dot, unhelpfully */ -}} {{- range $field := .Type.Fields }} fieldName__{{ $type | TypeSymbol }}_{{ $field | FieldSymbolUpper }}_serial = _String{"{{ $field | $type.RepresentationStrategy.GetFieldKey }}"} {{- end }} ) `, w, g.AdjCfg, g) } func (g structReprMapReprGenerator) EmitNodeTypeAssertions(w io.Writer) { doTemplate(` var _ datamodel.Node = &_{{ .Type | TypeSymbol }}__Repr{} `, w, g.AdjCfg, g) } func (g structReprMapReprGenerator) EmitNodeMethodLookupByString(w io.Writer) { // Similar to the type-level method, except any absent fields also return ErrNotExists. doTemplate(` func (n *_{{ .Type | TypeSymbol }}__Repr) LookupByString(key string) (datamodel.Node, error) { switch key { {{- range $field := .Type.Fields }} case "{{ $field | $field.Parent.RepresentationStrategy.GetFieldKey }}": {{- if $field.IsOptional }} if n.{{ $field | FieldSymbolLower }}.m == schema.Maybe_Absent { return datamodel.Absent, datamodel.ErrNotExists{Segment: datamodel.PathSegmentOfString(key)} } {{- end}} {{- if $field.IsNullable }} if n.{{ $field | FieldSymbolLower }}.m == schema.Maybe_Null { return datamodel.Null, nil } {{- end}} {{- if $field.IsMaybe }} return n.{{ $field | FieldSymbolLower }}.v.Representation(), nil {{- else}} return n.{{ $field | FieldSymbolLower }}.Representation(), nil {{- end}} {{- end}} default: return nil, schema.ErrNoSuchField{Type: nil /*TODO*/, Field: datamodel.PathSegmentOfString(key)} } } `, w, g.AdjCfg, g) } func (g structReprMapReprGenerator) EmitNodeMethodLookupByNode(w io.Writer) { doTemplate(` func (n *_{{ .Type | TypeSymbol }}__Repr) LookupByNode(key datamodel.Node) (datamodel.Node, error) { ks, err := key.AsString() if err != nil { return nil, err } return n.LookupByString(ks) } `, w, g.AdjCfg, g) } func (g structReprMapReprGenerator) EmitNodeMethodMapIterator(w io.Writer) { // The 'idx' int is what field we'll yield next. // Note that this iterator doesn't mention fields that are absent. // This makes things a bit trickier -- especially the 'Done' predicate, // since it may have to do lookahead if there's any optionals at the end of the structure! // It also means 'idx' can jump ahead by more than one per Next call in order to skip over absent fields. // TODO : support for implicits is still future work. // First: Determine if there are any optionals at all. // If there are none, some control flow symbols need to not be emitted. fields := g.Type.Fields() haveOptionals := false for _, field := range fields { if field.IsOptional() { haveOptionals = true break } } // Second: Count how many trailing fields are optional. // The 'Done' predicate gets more complex when in the trailing optionals. fieldCount := len(fields) beginTrailingOptionalField := fieldCount for i := fieldCount - 1; i >= 0; i-- { if !fields[i].IsOptional() { break } beginTrailingOptionalField = i } haveTrailingOptionals := beginTrailingOptionalField < fieldCount // Now: finally we can get on with the actual templating. doTemplate(` func (n *_{{ .Type | TypeSymbol }}__Repr) MapIterator() datamodel.MapIterator { {{- if .HaveTrailingOptionals }} end := {{ len .Type.Fields }}`+ func() string { // this next part was too silly in templates due to lack of reverse ranging. v := "\n" for i := fieldCount - 1; i >= beginTrailingOptionalField; i-- { v += "\t\t\tif n." + g.AdjCfg.FieldSymbolLower(fields[i]) + ".m == schema.Maybe_Absent {\n" v += "\t\t\t\tend = " + strconv.Itoa(i) + "\n" v += "\t\t\t} else {\n" v += "\t\t\t\tgoto done\n" v += "\t\t\t}\n" } return v }()+`done: return &_{{ .Type | TypeSymbol }}__ReprMapItr{n, 0, end} {{- else}} return &_{{ .Type | TypeSymbol }}__ReprMapItr{n, 0} {{- end}} } type _{{ .Type | TypeSymbol }}__ReprMapItr struct { n *_{{ .Type | TypeSymbol }}__Repr idx int {{if .HaveTrailingOptionals }}end int{{end}} } func (itr *_{{ .Type | TypeSymbol }}__ReprMapItr) Next() (k datamodel.Node, v datamodel.Node, _ error) { {{- if not .Type.Fields }} {{- /* TODO: deduplicate all these methods which just error */ -}} return nil, nil, datamodel.ErrIteratorOverread{} {{ else -}} {{ if .HaveOptionals }}advance:{{end -}} if itr.idx >= {{ len .Type.Fields }} { return nil, nil, datamodel.ErrIteratorOverread{} } switch itr.idx { {{- $type := .Type -}} {{- /* ranging modifies dot, unhelpfully */ -}} {{- range $i, $field := .Type.Fields }} case {{ $i }}: k = &fieldName__{{ $type | TypeSymbol }}_{{ $field | FieldSymbolUpper }}_serial {{- if $field.IsOptional }} if itr.n.{{ $field | FieldSymbolLower }}.m == schema.Maybe_Absent { itr.idx++ goto advance } {{- end}} {{- if $field.IsNullable }} if itr.n.{{ $field | FieldSymbolLower }}.m == schema.Maybe_Null { v = datamodel.Null break } {{- end}} {{- if $field.IsMaybe }} v = itr.n.{{ $field | FieldSymbolLower}}.v.Representation() {{- else}} v = itr.n.{{ $field | FieldSymbolLower}}.Representation() {{- end}} {{- end}} default: panic("unreachable") } itr.idx++ return {{- end}} } {{- if .HaveTrailingOptionals }} func (itr *_{{ .Type | TypeSymbol }}__ReprMapItr) Done() bool { return itr.idx >= itr.end } {{- else}} func (itr *_{{ .Type | TypeSymbol }}__ReprMapItr) Done() bool { return itr.idx >= {{ len .Type.Fields }} } {{- end}} `, w, g.AdjCfg, struct { Type *schema.TypeStruct HaveOptionals bool HaveTrailingOptionals bool BeginTrailingOptionalField int }{ g.Type, haveOptionals, haveTrailingOptionals, beginTrailingOptionalField, }) } func (g structReprMapReprGenerator) EmitNodeMethodLength(w io.Writer) { // This is fun: it has to count down for any unset optional fields. // TODO : support for implicits is still future work. doTemplate(` func (rn *_{{ .Type | TypeSymbol }}__Repr) Length() int64 { l := {{ len .Type.Fields }} {{- range $field := .Type.Fields }} {{- if $field.IsOptional }} if rn.{{ $field | FieldSymbolLower }}.m == schema.Maybe_Absent { l-- } {{- end}} {{- end}} return int64(l) } `, w, g.AdjCfg, g) } func (g structReprMapReprGenerator) EmitNodeMethodPrototype(w io.Writer) { emitNodeMethodPrototype_typical(w, g.AdjCfg, g) } func (g structReprMapReprGenerator) EmitNodePrototypeType(w io.Writer) { emitNodePrototypeType_typical(w, g.AdjCfg, g) } // --- NodeBuilder and NodeAssembler ---> func (g structReprMapReprGenerator) GetNodeBuilderGenerator() NodeBuilderGenerator { return structReprMapReprBuilderGenerator{ g.AdjCfg, mixins.MapAssemblerTraits{ PkgName: g.PkgName, TypeName: g.TypeName, AppliedPrefix: "_" + g.AdjCfg.TypeSymbol(g.Type) + "__Repr", }, g.PkgName, g.Type, } } type structReprMapReprBuilderGenerator struct { AdjCfg *AdjunctCfg mixins.MapAssemblerTraits PkgName string Type *schema.TypeStruct } func (structReprMapReprBuilderGenerator) IsRepr() bool { return true } // hint used in some generalized templates. func (g structReprMapReprBuilderGenerator) EmitNodeBuilderType(w io.Writer) { emitEmitNodeBuilderType_typical(w, g.AdjCfg, g) } func (g structReprMapReprBuilderGenerator) EmitNodeBuilderMethods(w io.Writer) { emitNodeBuilderMethods_typical(w, g.AdjCfg, g) } func (g structReprMapReprBuilderGenerator) EmitNodeAssemblerType(w io.Writer) { // - 'w' is the "**w**ip" pointer. // - 'm' is the **m**aybe which communicates our completeness to the parent if we're a child assembler. // - 'state' is what it says on the tin. this is used for the map state (the broad transitions between null, start-map, and finish are handled by 'm' for consistency.) // - 's' is a bitfield for what's been **s**et. // - 'f' is the **f**ocused field that will be assembled next. // // - 'cm' is **c**hild **m**aybe and is used for the completion message from children that aren't allowed to be nullable (for those that are, their own maybe.m is used). // - the 'ca_*' fields embed **c**hild **a**ssemblers -- these are embedded so we can yield pointers to them without causing new allocations. doTemplate(` type _{{ .Type | TypeSymbol }}__ReprAssembler struct { w *_{{ .Type | TypeSymbol }} m *schema.Maybe state maState s int f int cm schema.Maybe {{range $field := .Type.Fields -}} ca_{{ $field | FieldSymbolLower }} _{{ $field.Type | TypeSymbol }}__ReprAssembler {{end -}} } func (na *_{{ .Type | TypeSymbol }}__ReprAssembler) reset() { na.state = maState_initial na.s = 0 {{- range $field := .Type.Fields }} na.ca_{{ $field | FieldSymbolLower }}.reset() {{- end}} } `, w, g.AdjCfg, g) } func (g structReprMapReprBuilderGenerator) EmitNodeAssemblerMethodBeginMap(w io.Writer) { emitNodeAssemblerMethodBeginMap_strictoid(w, g.AdjCfg, g) } func (g structReprMapReprBuilderGenerator) EmitNodeAssemblerMethodAssignNull(w io.Writer) { emitNodeAssemblerMethodAssignNull_recursive(w, g.AdjCfg, g) } func (g structReprMapReprBuilderGenerator) EmitNodeAssemblerMethodAssignNode(w io.Writer) { // AssignNode goes through three phases: // 1. is it null? Jump over to AssignNull (which may or may not reject it). // 2. is it our own type? Handle specially -- we might be able to do efficient things. // 3. is it the right kind to morph into us? Do so. // // We do not set m=midvalue in phase 3 -- it shouldn't matter unless you're trying to pull off concurrent access, which is wrong and unsafe regardless. doTemplate(` func (na *_{{ .Type | TypeSymbol }}__ReprAssembler) AssignNode(v datamodel.Node) error { if v.IsNull() { return na.AssignNull() } if v2, ok := v.(*_{{ .Type | TypeSymbol }}); ok { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") case midvalue: panic("invalid state: cannot assign null into an assembler that's already begun working on recursive structures!") } {{- if .Type | MaybeUsesPtr }} if na.w == nil { na.w = v2 *na.m = schema.Maybe_Value return nil } {{- end}} *na.w = *v2 *na.m = schema.Maybe_Value return nil } if v.Kind() != datamodel.Kind_Map { return datamodel.ErrWrongKind{TypeName: "{{ .PkgName }}.{{ .Type.Name }}.Repr", MethodName: "AssignNode", AppropriateKind: datamodel.KindSet_JustMap, ActualKind: v.Kind()} } itr := v.MapIterator() for !itr.Done() { k, v, err := itr.Next() if err != nil { return err } if err := na.AssembleKey().AssignNode(k); err != nil { return err } if err := na.AssembleValue().AssignNode(v); err != nil { return err } } return na.Finish() } `, w, g.AdjCfg, g) } func (g structReprMapReprBuilderGenerator) EmitNodeAssemblerOtherBits(w io.Writer) { g.emitMapAssemblerChildTidyHelper(w) g.emitMapAssemblerMethods(w) g.emitKeyAssembler(w) } func (g structReprMapReprBuilderGenerator) emitMapAssemblerChildTidyHelper(w io.Writer) { // This is exactly the same as the matching method on the type-level assembler; // everything that differs happens to be hidden behind the 'f' indirection, which is numeric. doTemplate(` func (ma *_{{ .Type | TypeSymbol }}__ReprAssembler) valueFinishTidy() bool { switch ma.f { {{- range $i, $field := .Type.Fields }} case {{ $i }}: {{- if $field.IsNullable }} switch ma.w.{{ $field | FieldSymbolLower }}.m { case schema.Maybe_Null: ma.state = maState_initial return true case schema.Maybe_Value: {{- if (MaybeUsesPtr $field.Type) }} ma.w.{{ $field | FieldSymbolLower }}.v = ma.ca_{{ $field | FieldSymbolLower }}.w {{- end}} ma.state = maState_initial return true default: return false } {{- else if $field.IsOptional }} switch ma.w.{{ $field | FieldSymbolLower }}.m { case schema.Maybe_Value: {{- if (MaybeUsesPtr $field.Type) }} ma.w.{{ $field | FieldSymbolLower }}.v = ma.ca_{{ $field | FieldSymbolLower }}.w {{- end}} ma.state = maState_initial return true default: return false } {{- else}} switch ma.cm { case schema.Maybe_Value: {{- /* while defense in depth here might avoid some 'wat' outcomes, it's not strictly necessary for safety */ -}} {{- /* ma.ca_{{ $field | FieldSymbolLower }}.w = nil */ -}} {{- /* ma.ca_{{ $field | FieldSymbolLower }}.m = nil */ -}} ma.cm = schema.Maybe_Absent ma.state = maState_initial return true default: return false } {{- end}} {{- end}} default: panic("unreachable") } } `, w, g.AdjCfg, g) } func (g structReprMapReprBuilderGenerator) emitMapAssemblerMethods(w io.Writer) { // FUTURE: some of the setup of the child assemblers could probably be DRY'd up. doTemplate(` func (ma *_{{ .Type | TypeSymbol }}__ReprAssembler) AssembleEntry(k string) (datamodel.NodeAssembler, error) { switch ma.state { case maState_initial: // carry on case maState_midKey: panic("invalid state: AssembleEntry cannot be called when in the middle of assembling another key") case maState_expectValue: panic("invalid state: AssembleEntry cannot be called when expecting start of value assembly") case maState_midValue: if !ma.valueFinishTidy() { panic("invalid state: AssembleEntry cannot be called when in the middle of assembling a value") } // if tidy success: carry on case maState_finished: panic("invalid state: AssembleEntry cannot be called on an assembler that's already finished") } {{- $type := .Type -}} {{- /* ranging modifies dot, unhelpfully */ -}} {{- if .Type.Fields }} switch k { {{- range $i, $field := .Type.Fields }} case "{{ $field | $type.RepresentationStrategy.GetFieldKey }}": if ma.s & fieldBit__{{ $type | TypeSymbol }}_{{ $field | FieldSymbolUpper }} != 0 { return nil, datamodel.ErrRepeatedMapKey{Key: &fieldName__{{ $type | TypeSymbol }}_{{ $field | FieldSymbolUpper }}_serial} } ma.s += fieldBit__{{ $type | TypeSymbol }}_{{ $field | FieldSymbolUpper }} ma.state = maState_midValue ma.f = {{ $i }} {{- if $field.IsMaybe }} ma.ca_{{ $field | FieldSymbolLower }}.w = {{if not (MaybeUsesPtr $field.Type) }}&{{end}}ma.w.{{ $field | FieldSymbolLower }}.v ma.ca_{{ $field | FieldSymbolLower }}.m = &ma.w.{{ $field | FieldSymbolLower }}.m {{if $field.IsNullable }}ma.w.{{ $field | FieldSymbolLower }}.m = allowNull{{end}} {{- else}} ma.ca_{{ $field | FieldSymbolLower }}.w = &ma.w.{{ $field | FieldSymbolLower }} ma.ca_{{ $field | FieldSymbolLower }}.m = &ma.cm {{- end}} return &ma.ca_{{ $field | FieldSymbolLower }}, nil {{- end}} default: } {{- end}} return nil, schema.ErrInvalidKey{TypeName:"{{ .PkgName }}.{{ .Type.Name }}.Repr", Key:&_String{k}} } func (ma *_{{ .Type | TypeSymbol }}__ReprAssembler) AssembleKey() datamodel.NodeAssembler { switch ma.state { case maState_initial: // carry on case maState_midKey: panic("invalid state: AssembleKey cannot be called when in the middle of assembling another key") case maState_expectValue: panic("invalid state: AssembleKey cannot be called when expecting start of value assembly") case maState_midValue: if !ma.valueFinishTidy() { panic("invalid state: AssembleKey cannot be called when in the middle of assembling a value") } // if tidy success: carry on case maState_finished: panic("invalid state: AssembleKey cannot be called on an assembler that's already finished") } ma.state = maState_midKey return (*_{{ .Type | TypeSymbol }}__ReprKeyAssembler)(ma) } func (ma *_{{ .Type | TypeSymbol }}__ReprAssembler) AssembleValue() datamodel.NodeAssembler { switch ma.state { case maState_initial: panic("invalid state: AssembleValue cannot be called when no key is primed") case maState_midKey: panic("invalid state: AssembleValue cannot be called when in the middle of assembling a key") case maState_expectValue: // carry on case maState_midValue: panic("invalid state: AssembleValue cannot be called when in the middle of assembling another value") case maState_finished: panic("invalid state: AssembleValue cannot be called on an assembler that's already finished") } ma.state = maState_midValue switch ma.f { {{- range $i, $field := .Type.Fields }} case {{ $i }}: {{- if $field.IsMaybe }} ma.ca_{{ $field | FieldSymbolLower }}.w = {{if not (MaybeUsesPtr $field.Type) }}&{{end}}ma.w.{{ $field | FieldSymbolLower }}.v ma.ca_{{ $field | FieldSymbolLower }}.m = &ma.w.{{ $field | FieldSymbolLower }}.m {{if $field.IsNullable }}ma.w.{{ $field | FieldSymbolLower }}.m = allowNull{{end}} {{- else}} ma.ca_{{ $field | FieldSymbolLower }}.w = &ma.w.{{ $field | FieldSymbolLower }} ma.ca_{{ $field | FieldSymbolLower }}.m = &ma.cm {{- end}} return &ma.ca_{{ $field | FieldSymbolLower }} {{- end}} default: panic("unreachable") } } func (ma *_{{ .Type | TypeSymbol }}__ReprAssembler) Finish() error { switch ma.state { case maState_initial: // carry on case maState_midKey: panic("invalid state: Finish cannot be called when in the middle of assembling a key") case maState_expectValue: panic("invalid state: Finish cannot be called when expecting start of value assembly") case maState_midValue: if !ma.valueFinishTidy() { panic("invalid state: Finish cannot be called when in the middle of assembling a value") } // if tidy success: carry on case maState_finished: panic("invalid state: Finish cannot be called on an assembler that's already finished") } if ma.s & fieldBits__{{ $type | TypeSymbol }}_sufficient != fieldBits__{{ $type | TypeSymbol }}_sufficient { err := schema.ErrMissingRequiredField{Missing: make([]string, 0)} {{- range $i, $field := .Type.Fields }} {{- if not $field.IsMaybe}} if ma.s & fieldBit__{{ $type | TypeSymbol }}_{{ $field | FieldSymbolUpper }} == 0 { {{- if $field | $type.RepresentationStrategy.FieldHasRename }} err.Missing = append(err.Missing, "{{ $field.Name }} (serial:\"{{ $field | $type.RepresentationStrategy.GetFieldKey }}\")") {{- else}} err.Missing = append(err.Missing, "{{ $field.Name }}") {{- end}} } {{- end}} {{- end}} return err } ma.state = maState_finished *ma.m = schema.Maybe_Value return nil } func (ma *_{{ .Type | TypeSymbol }}__ReprAssembler) KeyPrototype() datamodel.NodePrototype { return _String__Prototype{} } func (ma *_{{ .Type | TypeSymbol }}__ReprAssembler) ValuePrototype(k string) datamodel.NodePrototype { panic("todo structbuilder mapassembler repr valueprototype") } `, w, g.AdjCfg, g) } func (g structReprMapReprBuilderGenerator) emitKeyAssembler(w io.Writer) { doTemplate(` type _{{ .Type | TypeSymbol }}__ReprKeyAssembler _{{ .Type | TypeSymbol }}__ReprAssembler `, w, g.AdjCfg, g) stubs := mixins.StringAssemblerTraits{ PkgName: g.PkgName, TypeName: g.TypeName + ".KeyAssembler", // ".Repr" is already in `g.TypeName`, so don't stutter the "Repr" part. AppliedPrefix: "_" + g.AdjCfg.TypeSymbol(g.Type) + "__ReprKey", } // This key assembler can disregard any idea of complex keys because it's at the representation level! // Map keys must always be plain strings at the representation level. stubs.EmitNodeAssemblerMethodBeginMap(w) stubs.EmitNodeAssemblerMethodBeginList(w) stubs.EmitNodeAssemblerMethodAssignNull(w) stubs.EmitNodeAssemblerMethodAssignBool(w) stubs.EmitNodeAssemblerMethodAssignInt(w) stubs.EmitNodeAssemblerMethodAssignFloat(w) doTemplate(` func (ka *_{{ .Type | TypeSymbol }}__ReprKeyAssembler) AssignString(k string) error { if ka.state != maState_midKey { panic("misuse: KeyAssembler held beyond its valid lifetime") } {{- if .Type.Fields }} switch k { {{- $type := .Type -}} {{- /* ranging modifies dot, unhelpfully */ -}} {{- range $i, $field := .Type.Fields }} case "{{ $field | $type.RepresentationStrategy.GetFieldKey }}": if ka.s & fieldBit__{{ $type | TypeSymbol }}_{{ $field | FieldSymbolUpper }} != 0 { return datamodel.ErrRepeatedMapKey{Key: &fieldName__{{ $type | TypeSymbol }}_{{ $field | FieldSymbolUpper }}_serial} } ka.s += fieldBit__{{ $type | TypeSymbol }}_{{ $field | FieldSymbolUpper }} ka.state = maState_expectValue ka.f = {{ $i }} return nil {{- end }} } {{- end }} return schema.ErrInvalidKey{TypeName:"{{ .PkgName }}.{{ .Type.Name }}.Repr", Key:&_String{k}} } `, w, g.AdjCfg, g) stubs.EmitNodeAssemblerMethodAssignBytes(w) stubs.EmitNodeAssemblerMethodAssignLink(w) doTemplate(` func (ka *_{{ .Type | TypeSymbol }}__ReprKeyAssembler) AssignNode(v datamodel.Node) error { if v2, err := v.AsString(); err != nil { return err } else { return ka.AssignString(v2) } } func (_{{ .Type | TypeSymbol }}__ReprKeyAssembler) Prototype() datamodel.NodePrototype { return _String__Prototype{} } `, w, g.AdjCfg, g) } ================================================ FILE: schema/gen/go/genStructReprStringjoin.go ================================================ package gengo import ( "io" "github.com/ipld/go-ipld-prime/schema" "github.com/ipld/go-ipld-prime/schema/gen/go/mixins" ) var _ TypeGenerator = &structReprStringjoinGenerator{} func NewStructReprStringjoinGenerator(pkgName string, typ *schema.TypeStruct, adjCfg *AdjunctCfg) TypeGenerator { return structReprStringjoinGenerator{ structGenerator{ adjCfg, mixins.MapTraits{ PkgName: pkgName, TypeName: string(typ.Name()), TypeSymbol: adjCfg.TypeSymbol(typ), }, pkgName, typ, }, } } type structReprStringjoinGenerator struct { structGenerator } func (g structReprStringjoinGenerator) GetRepresentationNodeGen() NodeGenerator { return structReprStringjoinReprGenerator{ g.AdjCfg, mixins.StringTraits{ PkgName: g.PkgName, TypeName: string(g.Type.Name()) + ".Repr", TypeSymbol: "_" + g.AdjCfg.TypeSymbol(g.Type) + "__Repr", }, g.PkgName, g.Type, } } type structReprStringjoinReprGenerator struct { AdjCfg *AdjunctCfg mixins.StringTraits PkgName string Type *schema.TypeStruct } func (structReprStringjoinReprGenerator) IsRepr() bool { return true } // hint used in some generalized templates. func (g structReprStringjoinReprGenerator) EmitNodeType(w io.Writer) { // The type is structurally the same, but will have a different set of methods. doTemplate(` type _{{ .Type | TypeSymbol }}__Repr _{{ .Type | TypeSymbol }} `, w, g.AdjCfg, g) } func (g structReprStringjoinReprGenerator) EmitNodeTypeAssertions(w io.Writer) { doTemplate(` var _ datamodel.Node = &_{{ .Type | TypeSymbol }}__Repr{} `, w, g.AdjCfg, g) } func (g structReprStringjoinReprGenerator) EmitNodeMethodAsString(w io.Writer) { // Prerequisites: // - every field must be a string, or have string representation. // - this should've been checked when compiling the type system info. // - we're willing to imply a base-10 atoi/itoa for ints (but it's not currently supported). // - there are NO sanity checks that your value doesn't contain the delimiter // - you need to do this in validation hooks or some other way // - optional or nullable fields are not supported with this representation strategy. // - this should've been checked when compiling the type system info. // - if support for this is added in the future, you can bet all optionals // will be required to be *either* in a row at the start, or in a row at the end. // (a 'direction' property might also be needed, so behavior is defined if every field is optional.) // // A speciated String method is also generated here. // (Organization questionable: if this was at type level, it'd be in the 'EmitNativeAccessors' block, // but we don't have that in the NodeGenerator interface so we don't have it here. Maybe that's a mistake.) // // A String method is *also* generated on the type-level node. // This might be worth consistency review... // It's a practical necessity in areas like stringifying for key error messages if used in map keys, for example. doTemplate(` func (n *_{{ .Type | TypeSymbol }}__Repr) AsString() (string, error) { return n.String(), nil } func (n *_{{ .Type | TypeSymbol }}__Repr) String() string { return {{ "" }} {{- $type := .Type -}} {{- /* ranging modifies dot, unhelpfully */ -}} {{- range $i, $field := .Type.Fields }} {{- if $i }} + "{{ $type.RepresentationStrategy.GetDelim }}" + {{end -}} (*_{{ $field.Type | TypeSymbol }}__Repr)(&n.{{ $field | FieldSymbolLower }}).String() {{- end}} } func (n {{ .Type | TypeSymbol }}) String() string { return (*_{{ .Type | TypeSymbol }}__Repr)(n).String() } `, w, g.AdjCfg, g) } func (g structReprStringjoinReprGenerator) EmitNodeMethodPrototype(w io.Writer) { emitNodeMethodPrototype_typical(w, g.AdjCfg, g) } func (g structReprStringjoinReprGenerator) EmitNodePrototypeType(w io.Writer) { emitNodePrototypeType_typical(w, g.AdjCfg, g) } // --- NodeBuilder and NodeAssembler ---> func (g structReprStringjoinReprGenerator) GetNodeBuilderGenerator() NodeBuilderGenerator { return structReprStringjoinReprBuilderGenerator{ g.AdjCfg, mixins.StringAssemblerTraits{ PkgName: g.PkgName, TypeName: g.TypeName, AppliedPrefix: "_" + g.AdjCfg.TypeSymbol(g.Type) + "__Repr", }, g.PkgName, g.Type, } } type structReprStringjoinReprBuilderGenerator struct { AdjCfg *AdjunctCfg mixins.StringAssemblerTraits PkgName string Type *schema.TypeStruct } func (structReprStringjoinReprBuilderGenerator) IsRepr() bool { return true } // hint used in some generalized templates. func (g structReprStringjoinReprBuilderGenerator) EmitNodeBuilderType(w io.Writer) { emitEmitNodeBuilderType_typical(w, g.AdjCfg, g) } func (g structReprStringjoinReprBuilderGenerator) EmitNodeBuilderMethods(w io.Writer) { emitNodeBuilderMethods_typical(w, g.AdjCfg, g) // Generate a single-step construction function -- this is easy to do for a scalar, // and all representations of scalar kind can be expected to have a method like this. // The function is attached to the NodePrototype for convenient namespacing; // it needs no new memory, so it would be inappropriate to attach to the builder or assembler. // The function is directly used internally by anything else that might involve recursive destructuring on the same scalar kind // (for example, structs using stringjoin strategies that have one of this type as a field, etc). // Since we're a representation of scalar kind, and can recurse, // we ourselves presume this plain construction method must also exist for all our members. // REVIEW: We could make an immut-safe version of this and export it on the NodePrototype too, as `FromString(string)`. // FUTURE: should engage validation flow. doTemplate(` func (_{{ .Type | TypeSymbol }}__ReprPrototype) fromString(w *_{{ .Type | TypeSymbol }}, v string) error { ss, err := mixins.SplitExact(v, "{{ .Type.RepresentationStrategy.GetDelim }}", {{ len .Type.Fields }}) if err != nil { return schema.ErrUnmatchable{TypeName:"{{ .PkgName }}.{{ .Type.Name }}.Repr", Reason: err} } {{- $dot := . -}} {{- /* ranging modifies dot, unhelpfully */ -}} {{- range $i, $field := .Type.Fields }} if err := (_{{ $field.Type | TypeSymbol }}__ReprPrototype{}).fromString(&w.{{ $field | FieldSymbolLower }}, ss[{{ $i }}]); err != nil { return schema.ErrUnmatchable{TypeName:"{{ $dot.PkgName }}.{{ $dot.Type.Name }}.Repr", Reason: err} } {{- end}} return nil } `, w, g.AdjCfg, g) } func (g structReprStringjoinReprBuilderGenerator) EmitNodeAssemblerType(w io.Writer) { doTemplate(` type _{{ .Type | TypeSymbol }}__ReprAssembler struct { w *_{{ .Type | TypeSymbol }} m *schema.Maybe } func (na *_{{ .Type | TypeSymbol }}__ReprAssembler) reset() {} `, w, g.AdjCfg, g) } func (g structReprStringjoinReprBuilderGenerator) EmitNodeAssemblerMethodAssignNull(w io.Writer) { emitNodeAssemblerMethodAssignNull_scalar(w, g.AdjCfg, g) } func (g structReprStringjoinReprBuilderGenerator) EmitNodeAssemblerMethodAssignString(w io.Writer) { // This method contains a branch to support MaybeUsesPtr because new memory may need to be allocated. // This allocation only happens if the 'w' ptr is nil, which means we're being used on a Maybe; // otherwise, the 'w' ptr should already be set, and we fill that memory location without allocating, as usual. doTemplate(` func (na *_{{ .Type | TypeSymbol }}__ReprAssembler) AssignString(v string) error { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") } {{- if .Type | MaybeUsesPtr }} if na.w == nil { na.w = &_{{ .Type | TypeSymbol }}{} } {{- end}} if err := (_{{ .Type | TypeSymbol }}__ReprPrototype{}).fromString(na.w, v); err != nil { return err } *na.m = schema.Maybe_Value return nil } `, w, g.AdjCfg, g) } func (g structReprStringjoinReprBuilderGenerator) EmitNodeAssemblerMethodAssignNode(w io.Writer) { // AssignNode goes through three phases: // 1. is it null? Jump over to AssignNull (which may or may not reject it). // 2. is it our own type? Handle specially -- we might be able to do efficient things. // 3. is it the right kind to morph into us? Do so. doTemplate(` func (na *_{{ .Type | TypeSymbol }}__ReprAssembler) AssignNode(v datamodel.Node) error { if v.IsNull() { return na.AssignNull() } if v2, ok := v.(*_{{ .Type | TypeSymbol }}); ok { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") } {{- if .Type | MaybeUsesPtr }} if na.w == nil { na.w = v2 *na.m = schema.Maybe_Value return nil } {{- end}} *na.w = *v2 *na.m = schema.Maybe_Value return nil } if v2, err := v.AsString(); err != nil { return err } else { return na.AssignString(v2) } } `, w, g.AdjCfg, g) } func (g structReprStringjoinReprBuilderGenerator) EmitNodeAssemblerOtherBits(w io.Writer) { // None for this. } ================================================ FILE: schema/gen/go/genStructReprTuple.go ================================================ package gengo import ( "io" "strconv" "github.com/ipld/go-ipld-prime/schema" "github.com/ipld/go-ipld-prime/schema/gen/go/mixins" ) var _ TypeGenerator = &structReprTupleGenerator{} // Optional fields for tuple representation are only allowed at the end, and contiguously. // Present fields are matched greedily: if the struct has five fields, // and the last two are optional, and there's four values, then they will be mapped onto the first four fields, period. // In theory, it would be possible to support a variety of fancier modes, configurably; // in practice, let's not: the ROI would be atrocious: // few people seem to want this; // the implementation complexity would rise dramatically; // and the next nearest substitutes for such behavior are already available, and cheap (and also sturdier). // It would make about as much sense to support implicits as it does trailing optionals, // which means we probably should consider that someday, // but it's not implemented today. func NewStructReprTupleGenerator(pkgName string, typ *schema.TypeStruct, adjCfg *AdjunctCfg) TypeGenerator { return structReprTupleGenerator{ structGenerator{ adjCfg, mixins.MapTraits{ PkgName: pkgName, TypeName: string(typ.Name()), TypeSymbol: adjCfg.TypeSymbol(typ), }, pkgName, typ, }, } } type structReprTupleGenerator struct { structGenerator } func (g structReprTupleGenerator) GetRepresentationNodeGen() NodeGenerator { return structReprTupleReprGenerator{ g.AdjCfg, mixins.ListTraits{ PkgName: g.PkgName, TypeName: string(g.Type.Name()) + ".Repr", TypeSymbol: "_" + g.AdjCfg.TypeSymbol(g.Type) + "__Repr", }, g.PkgName, g.Type, } } type structReprTupleReprGenerator struct { AdjCfg *AdjunctCfg mixins.ListTraits PkgName string Type *schema.TypeStruct } func (structReprTupleReprGenerator) IsRepr() bool { return true } // hint used in some generalized templates. func (g structReprTupleReprGenerator) EmitNodeType(w io.Writer) { // The type is structurally the same, but will have a different set of methods. doTemplate(` type _{{ .Type | TypeSymbol }}__Repr _{{ .Type | TypeSymbol }} `, w, g.AdjCfg, g) } func (g structReprTupleReprGenerator) EmitNodeTypeAssertions(w io.Writer) { doTemplate(` var _ datamodel.Node = &_{{ .Type | TypeSymbol }}__Repr{} `, w, g.AdjCfg, g) } func (g structReprTupleReprGenerator) EmitNodeMethodLookupByIndex(w io.Writer) { doTemplate(` func (n *_{{ .Type | TypeSymbol }}__Repr) LookupByIndex(idx int64) (datamodel.Node, error) { switch idx { {{- range $i, $field := .Type.Fields }} case {{ $i }}: {{- if $field.IsOptional }} if n.{{ $field | FieldSymbolLower }}.m == schema.Maybe_Absent { return datamodel.Absent, datamodel.ErrNotExists{Segment: datamodel.PathSegmentOfInt(idx)} } {{- end}} {{- if $field.IsNullable }} if n.{{ $field | FieldSymbolLower }}.m == schema.Maybe_Null { return datamodel.Null, nil } {{- end}} {{- if $field.IsMaybe }} return n.{{ $field | FieldSymbolLower }}.v.Representation(), nil {{- else}} return n.{{ $field | FieldSymbolLower }}.Representation(), nil {{- end}} {{- end}} default: return nil, schema.ErrNoSuchField{Type: nil /*TODO*/, Field: datamodel.PathSegmentOfInt(idx)} } } `, w, g.AdjCfg, g) } func (g structReprTupleReprGenerator) EmitNodeMethodLookupByNode(w io.Writer) { doTemplate(` func (n *_{{ .Type | TypeSymbol }}__Repr) LookupByNode(key datamodel.Node) (datamodel.Node, error) { ki, err := key.AsInt() if err != nil { return nil, err } return n.LookupByIndex(ki) } `, w, g.AdjCfg, g) } func (g structReprTupleReprGenerator) EmitNodeMethodListIterator(w io.Writer) { // DRY: much of this precalcuation about doneness is common with the map representation. // (or at least: it is for now: the addition of support for implicits in the map representation may bamboozle that.) // Some of the templating also experiences the `.HaveTrailingOptionals` branching, // but not quite as much as the map representation: since we always know those come at the end // (and in particular, once we hit one absent, we're done!), some simplifications can be made. // The 'idx' int is what field we'll yield next. // Note that this iterator doesn't mention fields that are absent. // This makes things a bit trickier -- especially the 'Done' predicate, // since it may have to do lookahead if there's any optionals at the end of the structure! // Count how many trailing fields are optional. // The 'Done' predicate gets more complex when in the trailing optionals. fields := g.Type.Fields() fieldCount := len(fields) beginTrailingOptionalField := fieldCount for i := fieldCount - 1; i >= 0; i-- { if !fields[i].IsOptional() { break } beginTrailingOptionalField = i } haveTrailingOptionals := beginTrailingOptionalField < fieldCount // Now: finally we can get on with the actual templating. doTemplate(` func (n *_{{ .Type | TypeSymbol }}__Repr) ListIterator() datamodel.ListIterator { {{- if .HaveTrailingOptionals }} end := {{ len .Type.Fields }}`+ func() string { // this next part was too silly in templates due to lack of reverse ranging. v := "\n" for i := fieldCount - 1; i >= beginTrailingOptionalField; i-- { v += "\t\t\tif n." + g.AdjCfg.FieldSymbolLower(fields[i]) + ".m == schema.Maybe_Absent {\n" v += "\t\t\t\tend = " + strconv.Itoa(i) + "\n" v += "\t\t\t} else {\n" v += "\t\t\t\tgoto done\n" v += "\t\t\t}\n" } return v }()+`done: return &_{{ .Type | TypeSymbol }}__ReprListItr{n, 0, end} {{- else}} return &_{{ .Type | TypeSymbol }}__ReprListItr{n, 0} {{- end}} } type _{{ .Type | TypeSymbol }}__ReprListItr struct { n *_{{ .Type | TypeSymbol }}__Repr idx int {{if .HaveTrailingOptionals }}end int{{end}} } func (itr *_{{ .Type | TypeSymbol }}__ReprListItr) Next() (idx int64, v datamodel.Node, err error) { if itr.idx >= {{ len .Type.Fields }} { return -1, nil, datamodel.ErrIteratorOverread{} } switch itr.idx { {{- range $i, $field := .Type.Fields }} case {{ $i }}: idx = int64(itr.idx) {{- if $field.IsOptional }} if itr.n.{{ $field | FieldSymbolLower }}.m == schema.Maybe_Absent { return -1, nil, datamodel.ErrIteratorOverread{} } {{- end}} {{- if $field.IsNullable }} if itr.n.{{ $field | FieldSymbolLower }}.m == schema.Maybe_Null { v = datamodel.Null break } {{- end}} {{- if $field.IsMaybe }} v = itr.n.{{ $field | FieldSymbolLower}}.v.Representation() {{- else}} v = itr.n.{{ $field | FieldSymbolLower}}.Representation() {{- end}} {{- end}} default: panic("unreachable") } itr.idx++ return } {{- if .HaveTrailingOptionals }} func (itr *_{{ .Type | TypeSymbol }}__ReprListItr) Done() bool { return itr.idx >= itr.end } {{- else}} func (itr *_{{ .Type | TypeSymbol }}__ReprListItr) Done() bool { return itr.idx >= {{ len .Type.Fields }} } {{- end}} `, w, g.AdjCfg, struct { Type *schema.TypeStruct HaveTrailingOptionals bool }{ g.Type, haveTrailingOptionals, }) } func (g structReprTupleReprGenerator) EmitNodeMethodLength(w io.Writer) { // This is fun: it has to count down for any unset optional fields. doTemplate(` func (rn *_{{ .Type | TypeSymbol }}__Repr) Length() int64 { l := {{ len .Type.Fields }} {{- range $field := .Type.Fields }} {{- if $field.IsOptional }} if rn.{{ $field | FieldSymbolLower }}.m == schema.Maybe_Absent { l-- } {{- end}} {{- end}} return int64(l) } `, w, g.AdjCfg, g) } func (g structReprTupleReprGenerator) EmitNodeMethodPrototype(w io.Writer) { emitNodeMethodPrototype_typical(w, g.AdjCfg, g) } func (g structReprTupleReprGenerator) EmitNodePrototypeType(w io.Writer) { emitNodePrototypeType_typical(w, g.AdjCfg, g) } // --- NodeBuilder and NodeAssembler ---> func (g structReprTupleReprGenerator) GetNodeBuilderGenerator() NodeBuilderGenerator { return structReprTupleReprBuilderGenerator{ g.AdjCfg, mixins.ListAssemblerTraits{ PkgName: g.PkgName, TypeName: g.TypeName, AppliedPrefix: "_" + g.AdjCfg.TypeSymbol(g.Type) + "__Repr", }, g.PkgName, g.Type, } } type structReprTupleReprBuilderGenerator struct { AdjCfg *AdjunctCfg mixins.ListAssemblerTraits PkgName string Type *schema.TypeStruct } func (structReprTupleReprBuilderGenerator) IsRepr() bool { return true } // hint used in some generalized templates. func (g structReprTupleReprBuilderGenerator) EmitNodeBuilderType(w io.Writer) { emitEmitNodeBuilderType_typical(w, g.AdjCfg, g) } func (g structReprTupleReprBuilderGenerator) EmitNodeBuilderMethods(w io.Writer) { emitNodeBuilderMethods_typical(w, g.AdjCfg, g) } func (g structReprTupleReprBuilderGenerator) EmitNodeAssemblerType(w io.Writer) { // - 'w' is the "**w**ip" pointer. // - 'm' is the **m**aybe which communicates our completeness to the parent if we're a child assembler. // - 'state' is what it says on the tin. this is used for the list state (the broad transitions between null, start-list, and finish are handled by 'm' for consistency with other types). // - contrasted to the map representation, there's no 's' bitfield for what's been **s**et -- because we know things must proceed in order, it would be redundant with 'f'. // - 'f' is the **f**ocused field that will be assembled next. // // - 'cm' is **c**hild **m**aybe and is used for the completion message from children that aren't allowed to be nullable (for those that are, their own maybe.m is used). // - the 'ca_*' fields embed **c**hild **a**ssemblers -- these are embedded so we can yield pointers to them without causing new allocations. // // Note that this textually similar to the type-level assembler, but because it embeds the repr assembler for the child types, // it might be *significantly* different in size and memory layout in that trailing part of the struct. doTemplate(` type _{{ .Type | TypeSymbol }}__ReprAssembler struct { w *_{{ .Type | TypeSymbol }} m *schema.Maybe state laState f int cm schema.Maybe {{range $field := .Type.Fields -}} ca_{{ $field | FieldSymbolLower }} _{{ $field.Type | TypeSymbol }}__ReprAssembler {{end -}} } func (na *_{{ .Type | TypeSymbol }}__ReprAssembler) reset() { na.state = laState_initial na.f = 0 {{- range $field := .Type.Fields }} na.ca_{{ $field | FieldSymbolLower }}.reset() {{- end}} } `, w, g.AdjCfg, g) } func (g structReprTupleReprBuilderGenerator) EmitNodeAssemblerMethodBeginList(w io.Writer) { // Future: This could do something strict with the sizehint; it currently ignores it. doTemplate(` func (na *_{{ .Type | TypeSymbol }}__ReprAssembler) BeginList(int64) (datamodel.ListAssembler, error) { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") case midvalue: panic("invalid state: it makes no sense to 'begin' twice on the same assembler!") } *na.m = midvalue {{- if .Type | MaybeUsesPtr }} if na.w == nil { na.w = &_{{ .Type | TypeSymbol }}{} } {{- end}} return na, nil } `, w, g.AdjCfg, g) } func (g structReprTupleReprBuilderGenerator) EmitNodeAssemblerMethodAssignNull(w io.Writer) { emitNodeAssemblerMethodAssignNull_recursive(w, g.AdjCfg, g) } func (g structReprTupleReprBuilderGenerator) EmitNodeAssemblerMethodAssignNode(w io.Writer) { emitNodeAssemblerMethodAssignNode_listoid(w, g.AdjCfg, g) } func (g structReprTupleReprBuilderGenerator) EmitNodeAssemblerOtherBits(w io.Writer) { g.emitListAssemblerChildTidyHelper(w) g.emitListAssemblerChildListAssemblerMethods(w) } func (g structReprTupleReprBuilderGenerator) emitListAssemblerChildTidyHelper(w io.Writer) { doTemplate(` func (la *_{{ .Type | TypeSymbol }}__ReprAssembler) valueFinishTidy() bool { switch la.f { {{- range $i, $field := .Type.Fields }} case {{ $i }}: {{- if $field.IsMaybe }} switch la.w.{{ $field | FieldSymbolLower }}.m { case schema.Maybe_Value: {{- if (MaybeUsesPtr $field.Type) }} la.w.{{ $field | FieldSymbolLower }}.v = la.ca_{{ $field | FieldSymbolLower }}.w {{- end}} la.state = laState_initial la.f++ return true {{- else}} switch la.cm { case schema.Maybe_Value: la.cm = schema.Maybe_Absent la.state = laState_initial la.f++ return true {{- end}} {{- if $field.IsNullable }} case schema.Maybe_Null: la.state = laState_initial la.f++ return true {{- end}} default: return false } {{- end}} default: panic("unreachable") } } `, w, g.AdjCfg, g) } func (g structReprTupleReprBuilderGenerator) emitListAssemblerChildListAssemblerMethods(w io.Writer) { doTemplate(` func (la *_{{ .Type | TypeSymbol }}__ReprAssembler) AssembleValue() datamodel.NodeAssembler { switch la.state { case laState_initial: // carry on case laState_midValue: if !la.valueFinishTidy() { panic("invalid state: AssembleValue cannot be called when still in the middle of assembling the previous value") } // if tidy success: carry on case laState_finished: panic("invalid state: AssembleValue cannot be called on an assembler that's already finished") } if la.f >= {{ len .Type.Fields }} { return _ErrorThunkAssembler{schema.ErrNoSuchField{Type: nil /*TODO*/, Field: datamodel.PathSegmentOfInt({{ len .Type.Fields }})}} } la.state = laState_midValue switch la.f { {{- range $i, $field := .Type.Fields }} case {{ $i }}: {{- if $field.IsMaybe }} la.ca_{{ $field | FieldSymbolLower }}.w = {{if not (MaybeUsesPtr $field.Type) }}&{{end}}la.w.{{ $field | FieldSymbolLower }}.v la.ca_{{ $field | FieldSymbolLower }}.m = &la.w.{{ $field | FieldSymbolLower }}.m {{- if $field.IsNullable }} la.w.{{ $field | FieldSymbolLower }}.m = allowNull {{- end}} {{- else}} la.ca_{{ $field | FieldSymbolLower }}.w = &la.w.{{ $field | FieldSymbolLower }} la.ca_{{ $field | FieldSymbolLower }}.m = &la.cm {{- end}} return &la.ca_{{ $field | FieldSymbolLower }} {{- end}} default: panic("unreachable") } } `, w, g.AdjCfg, g) // Surprisingly, the Finish method doesn't have anything to do regarding any trailing optionals: // if they weren't assigned yet, their Maybe state is still the zero value: absent. And that's correct. // DRY: okay, this finish component is actually identical, both textually and in terms of linking, to lists. This we should actually extract. doTemplate(` func (la *_{{ .Type | TypeSymbol }}__ReprAssembler) Finish() error { switch la.state { case laState_initial: // carry on case laState_midValue: if !la.valueFinishTidy() { panic("invalid state: Finish cannot be called when in the middle of assembling a value") } // if tidy success: carry on case laState_finished: panic("invalid state: Finish cannot be called on an assembler that's already finished") } la.state = laState_finished *la.m = schema.Maybe_Value return nil } `, w, g.AdjCfg, g) doTemplate(` func (la *_{{ .Type | TypeSymbol }}__ReprAssembler) ValuePrototype(_ int64) datamodel.NodePrototype { panic("todo structbuilder tuplerepr valueprototype") } `, w, g.AdjCfg, g) } ================================================ FILE: schema/gen/go/genUnion.go ================================================ package gengo import ( "io" "github.com/ipld/go-ipld-prime/schema" "github.com/ipld/go-ipld-prime/schema/gen/go/mixins" ) // The generator for unions is a bit more wild than most others: // it has at three major branches for how its internals are laid out: // // - all possible children are embedded. // - all possible children are pointers... in which case we collapse to one interface resident. // (n.b. this does give up some inlining potential as well as gives up on alloc amortization, but it does make resident memory size minimal.) // - some children are emebedded and some are pointers, and of the latter set, they may be either in one interface field or several discrete pointers. // (discrete fields of pointer type makes inlining possible in some paths, whereas an interface field blocks it). // // ... We're not doing that last one at all right now. The pareto-prevalence of these concerns is extremely low compared to the effort required. // But the first two are both very reasonable, and both are often wanted. // // These choices are made from adjunct config (which should make sense, because they're clearly all "golang" details -- not type semantics). // We still tackle all the generation for all these strategies this in one file, // because all of the interfaces we export are the same, regardless of the internals (and it just seems easiest to do this way). type unionGenerator struct { AdjCfg *AdjunctCfg mixins.MapTraits PkgName string Type *schema.TypeUnion } func (unionGenerator) IsRepr() bool { return false } // hint used in some generalized templates. // --- native content and specializations ---> func (g unionGenerator) EmitNativeType(w io.Writer) { // We generate *two* types: a struct which acts as the union node, // and also an interface which covers the members (and has an unexported marker function to make sure the set can't be extended). // // The interface *mostly* isn't used... except for in the return type of a speciated function which can be used to do golang-native type switches. // // The interface also includes a requirement for an errorless primitive access method (such as `String() string`) // if our representation strategy is one that has that semantic (e.g., stringprefix repr does). // // A note about index: in all cases the index of a member type is used, we increment it by one, to avoid using zero. // We do this because it's desirable to reserve the zero in the 'tag' field (if we generate one) as a sentinel value // (see further comments in the EmitNodeAssemblerType function); // and since we do it in that one case, it's just as well to do it uniformly. doTemplate(` {{- if Comments -}} // {{ .Type | TypeSymbol }} matches the IPLD Schema type "{{ .Type.Name }}". // {{ .Type | TypeSymbol }} has {{ .Type.TypeKind }} typekind, which means its data model behaviors are that of a {{ .Kind }} kind. {{- end}} type {{ .Type | TypeSymbol }} = *_{{ .Type | TypeSymbol }} type _{{ .Type | TypeSymbol }} struct { {{- if (eq (.AdjCfg.UnionMemlayout .Type) "embedAll") }} tag uint {{- range $i, $member := .Type.Members }} x{{ add $i 1 }} _{{ $member | TypeSymbol }} {{- end}} {{- else if (eq (.AdjCfg.UnionMemlayout .Type) "interface") }} x _{{ .Type | TypeSymbol }}__iface {{- end}} } type _{{ .Type | TypeSymbol }}__iface interface { _{{ .Type | TypeSymbol }}__member() {{- if (eq (.Type.RepresentationStrategy | printf "%T") "schema.UnionRepresentation_Stringprefix") }} String() string {{- end}} } {{- range $member := .Type.Members }} func (_{{ $member | TypeSymbol }}) _{{ dot.Type | TypeSymbol }}__member() {} {{- end}} `, w, g.AdjCfg, g) } func (g unionGenerator) EmitNativeAccessors(w io.Writer) { doTemplate(` func (n _{{ .Type | TypeSymbol }}) AsInterface() _{{ .Type | TypeSymbol }}__iface { {{- if (eq (.AdjCfg.UnionMemlayout .Type) "embedAll") }} switch n.tag { {{- range $i, $member := .Type.Members }} case {{ add $i 1 }}: return &n.x{{ add $i 1 }} {{- end}} default: panic("invalid union state; how did you create this object?") } {{- else if (eq (.AdjCfg.UnionMemlayout .Type) "interface") }} return n.x {{- end}} } `, w, g.AdjCfg, g) } func (g unionGenerator) EmitNativeBuilder(w io.Writer) { // Unclear as yet what should go here. } func (g unionGenerator) EmitNativeMaybe(w io.Writer) { emitNativeMaybe(w, g.AdjCfg, g) } // --- type info ---> func (g unionGenerator) EmitTypeConst(w io.Writer) { doTemplate(` // TODO EmitTypeConst `, w, g.AdjCfg, g) } // --- TypedNode interface satisfaction ---> func (g unionGenerator) EmitTypedNodeMethodType(w io.Writer) { doTemplate(` func ({{ .Type | TypeSymbol }}) Type() schema.Type { return nil /*TODO:typelit*/ } `, w, g.AdjCfg, g) } func (g unionGenerator) EmitTypedNodeMethodRepresentation(w io.Writer) { emitTypicalTypedNodeMethodRepresentation(w, g.AdjCfg, g) } // --- Node interface satisfaction ---> func (g unionGenerator) EmitNodeType(w io.Writer) { // No additional types needed. Methods all attach to the native type. // We do, however, want some constants for our member names; // they'll make iterators able to work faster. So let's emit those. // These are a bit perplexing, because they're... type names. // However, oddly enough, we don't have type names available *as nodes* anywhere else centrally available, // so... we generate some values for them here with scoped identifiers and get on with it. // Maybe this could be elided with future work. doTemplate(` var ( {{- range $member := .Type.Members }} memberName__{{ dot.Type | TypeSymbol }}_{{ $member.Name }} = _String{"{{ $member.Name }}"} {{- end }} ) `, w, g.AdjCfg, g) } func (g unionGenerator) EmitNodeTypeAssertions(w io.Writer) { emitNodeTypeAssertions_typical(w, g.AdjCfg, g) } func (g unionGenerator) EmitNodeMethodLookupByString(w io.Writer) { doTemplate(` func (n {{ .Type | TypeSymbol }}) LookupByString(key string) (datamodel.Node, error) { switch key { {{- range $i, $member := .Type.Members }} case "{{ $member.Name }}": {{- if (eq (dot.AdjCfg.UnionMemlayout dot.Type) "embedAll") }} if n.tag != {{ add $i 1 }} { return nil, datamodel.ErrNotExists{Segment: datamodel.PathSegmentOfString(key)} } return &n.x{{ add $i 1 }}, nil {{- else if (eq (dot.AdjCfg.UnionMemlayout dot.Type) "interface") }} if n2, ok := n.x.({{ $member | TypeSymbol }}); ok { return n2, nil } else { return nil, datamodel.ErrNotExists{Segment: datamodel.PathSegmentOfString(key)} } {{- end}} {{- end}} default: return nil, schema.ErrNoSuchField{Type: nil /*TODO*/, Field: datamodel.PathSegmentOfString(key)} } } `, w, g.AdjCfg, g) } func (g unionGenerator) EmitNodeMethodLookupByNode(w io.Writer) { doTemplate(` func (n {{ .Type | TypeSymbol }}) LookupByNode(key datamodel.Node) (datamodel.Node, error) { ks, err := key.AsString() if err != nil { return nil, err } return n.LookupByString(ks) } `, w, g.AdjCfg, g) } func (g unionGenerator) EmitNodeMethodMapIterator(w io.Writer) { // This is kind of a hilarious "iterator": it has to count all the way up to... 1. doTemplate(` func (n {{ .Type | TypeSymbol }}) MapIterator() datamodel.MapIterator { return &_{{ .Type | TypeSymbol }}__MapItr{n, false} } type _{{ .Type | TypeSymbol }}__MapItr struct { n {{ .Type | TypeSymbol }} done bool } func (itr *_{{ .Type | TypeSymbol }}__MapItr) Next() (k datamodel.Node, v datamodel.Node, _ error) { if itr.done { return nil, nil, datamodel.ErrIteratorOverread{} } {{- if (eq (.AdjCfg.UnionMemlayout .Type) "embedAll") }} switch itr.n.tag { {{- range $i, $member := .Type.Members }} case {{ add $i 1 }}: k, v = &memberName__{{ dot.Type | TypeSymbol }}_{{ $member.Name }}, &itr.n.x{{ add $i 1 }} {{- end}} {{- else if (eq (.AdjCfg.UnionMemlayout .Type) "interface") }} switch n2 := itr.n.x.(type) { {{- range $member := .Type.Members }} case {{ $member | TypeSymbol }}: k, v = &memberName__{{ dot.Type | TypeSymbol }}_{{ $member.Name }}, n2 {{- end}} {{- end}} default: panic("unreachable") } itr.done = true return } func (itr *_{{ .Type | TypeSymbol }}__MapItr) Done() bool { return itr.done } `, w, g.AdjCfg, g) } func (g unionGenerator) EmitNodeMethodLength(w io.Writer) { doTemplate(` func ({{ .Type | TypeSymbol }}) Length() int64 { return 1 } `, w, g.AdjCfg, g) } func (g unionGenerator) EmitNodeMethodPrototype(w io.Writer) { emitNodeMethodPrototype_typical(w, g.AdjCfg, g) } func (g unionGenerator) EmitNodePrototypeType(w io.Writer) { emitNodePrototypeType_typical(w, g.AdjCfg, g) } // --- NodeBuilder and NodeAssembler ---> func (g unionGenerator) GetNodeBuilderGenerator() NodeBuilderGenerator { return unionBuilderGenerator{ g.AdjCfg, mixins.MapAssemblerTraits{ PkgName: g.PkgName, TypeName: g.TypeName, AppliedPrefix: "_" + g.AdjCfg.TypeSymbol(g.Type) + "__", }, g.PkgName, g.Type, } } type unionBuilderGenerator struct { AdjCfg *AdjunctCfg mixins.MapAssemblerTraits PkgName string Type *schema.TypeUnion } func (unionBuilderGenerator) IsRepr() bool { return false } // hint used in some generalized templates. func (g unionBuilderGenerator) EmitNodeBuilderType(w io.Writer) { emitEmitNodeBuilderType_typical(w, g.AdjCfg, g) } func (g unionBuilderGenerator) EmitNodeBuilderMethods(w io.Writer) { emitNodeBuilderMethods_typical(w, g.AdjCfg, g) } func (g unionBuilderGenerator) EmitNodeAssemblerType(w io.Writer) { // Assemblers for unions are not unlikely those for structs or maps: // // - 'w' is the "**w**ip" pointer. // - 'm' is the pointer to a **m**aybe which communicates our completeness to the parent if we're a child assembler. // Like any other structure, a union can be nullable in the context of some enclosing object, and we'll have the usual branches for handling that in our various Assign methods. // - 'state' is what it says on the tin. Unions use maState to sequence the transitions between a new assembler, the map having been started, key insertions, value insertions, and finish. // Most of this is just like the way struct and map use maState. // However, we also need to guard to make sure a second entry never begins; after the first, finish is the *only* valid transition. // In structs, this is done using the "set" bitfield; in maps, the state resides in the wip map itself. // Unions are more like the latter: depending on which memory layout we're using, either the `na.w.tag` value, or, a non-nil `na.w.x`, is indicative that one key has been entered. // (The zero value for `na.w.tag` is reserved, and all for this reason. // - There is no additional state need to store "focus" (in contrast to structs); // information during the AssembleValue phase about which member is selected is also just handled in `na.w.tag`, or, in the type info of `na.w.x`, again depending on memory layout strategy. // (This is subverted a bit by the 'ca' field, however... which effectively mirrors `na.w.tag`, and is only active in the resetting process, but is necessary because it outlives its twin inside 'w'.) // // - 'cm' is **c**hild **m**aybe and is used for the completion message from children. // - 'ca*' fields embed **c**hild **a**ssemblers -- these are embedded so we can yield pointers to them during recursion into child value assembly without causing new allocations. // In unions, only one of these will every be used! However, we don't know *which one* in advance, so, we have to embed them all. // (It's ironic to note that if the golang compiler had an understanding of unions itself (either tagged or untagged would suffice), we could compile this down into *much* more minimal amounts of resident memory reservation. Alas!) // The 'ca*' fields are pointers (and allocated on demand) instead of embeds for unions with memlayout=interface mode. (Arguably, this is overloading that config; PRs for more granular configurability welcome.) // - 'ca' (with no further suffix) identifies which child assembler was previously used. // This is for minimizing the amount of work that resetting has to do: it will only recurse into resetting that child assembler. doTemplate(` type _{{ .Type | TypeSymbol }}__Assembler struct { w *_{{ .Type | TypeSymbol }} m *schema.Maybe state maState cm schema.Maybe {{- range $i, $member := .Type.Members }} ca{{ add $i 1 }} {{ if (eq (dot.AdjCfg.UnionMemlayout dot.Type) "interface") }}*{{end}}_{{ $member | TypeSymbol }}__Assembler {{end -}} ca uint } `, w, g.AdjCfg, g) // Reset methods for unions are a tad more involved than for most other assemblers: // we only want to bother to reset whichever child assembler (if any) we actually used last. // We *could* blithely reset *all* child assemblers every time; but, trading an extra bit of state in our assembler // for the privilege of trimming off a potentially sizable amount of unnecessary zeroing efforts seems preferable. // Also, although go syntax makes it not textually obvious here, note that it's possible for the child assemblers to be either pointers or embeds: // on consequence of this is that just zeroing this struct would be both unreliable and undesirable in the pointer case // (it would leave orphan child assemblers that might still have pointers into us, which could be guarded against but is nonetheless is considerably scary in complexity; // and it would also mean that we can't keep ahold of the child assemblers across resets and thus amortize allocations, which... is the whole reason the reset system exists in the first place). doTemplate(` func (na *_{{ .Type | TypeSymbol }}__Assembler) reset() { na.state = maState_initial switch na.ca { case 0: return {{- range $i, $member := .Type.Members }} case {{ add $i 1 }}: na.ca{{ add $i 1 }}.reset() {{end -}} default: panic("unreachable") } na.ca = 0 na.cm = schema.Maybe_Absent } `, w, g.AdjCfg, g) } func (g unionBuilderGenerator) EmitNodeAssemblerMethodBeginMap(w io.Writer) { emitNodeAssemblerMethodBeginMap_strictoid(w, g.AdjCfg, g) } func (g unionBuilderGenerator) EmitNodeAssemblerMethodAssignNull(w io.Writer) { // It might sound a bit odd to call a union "recursive", since it's so very trivially so (no fan-out), // but it's functionally accurate: the generated method should include a branch for the 'midvalue' state. emitNodeAssemblerMethodAssignNull_recursive(w, g.AdjCfg, g) } func (g unionBuilderGenerator) EmitNodeAssemblerMethodAssignNode(w io.Writer) { // AssignNode goes through three phases: // 1. is it null? Jump over to AssignNull (which may or may not reject it). // 2. is it our own type? Handle specially -- we might be able to do efficient things. // 3. is it the right kind to morph into us? Do so. // // We do not set m=midvalue in phase 3 -- it shouldn't matter unless you're trying to pull off concurrent access, which is wrong and unsafe regardless. // // DRY: this turns out to be textually identical to the method for structs! (At least, for now. It could/should probably be optimized to get to the point faster in phase 3.) doTemplate(` func (na *_{{ .Type | TypeSymbol }}__Assembler) AssignNode(v datamodel.Node) error { if v.IsNull() { return na.AssignNull() } if v2, ok := v.(*_{{ .Type | TypeSymbol }}); ok { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") case midvalue: panic("invalid state: cannot assign null into an assembler that's already begun working on recursive structures!") } {{- if .Type | MaybeUsesPtr }} if na.w == nil { na.w = v2 *na.m = schema.Maybe_Value return nil } {{- end}} *na.w = *v2 *na.m = schema.Maybe_Value return nil } if v.Kind() != datamodel.Kind_Map { return datamodel.ErrWrongKind{TypeName: "{{ .PkgName }}.{{ .Type.Name }}", MethodName: "AssignNode", AppropriateKind: datamodel.KindSet_JustMap, ActualKind: v.Kind()} } itr := v.MapIterator() for !itr.Done() { k, v, err := itr.Next() if err != nil { return err } if err := na.AssembleKey().AssignNode(k); err != nil { return err } if err := na.AssembleValue().AssignNode(v); err != nil { return err } } return na.Finish() } `, w, g.AdjCfg, g) } func (g unionBuilderGenerator) EmitNodeAssemblerOtherBits(w io.Writer) { g.emitMapAssemblerChildTidyHelper(w) g.emitMapAssemblerMethods(w) g.emitKeyAssembler(w) } func (g unionBuilderGenerator) emitMapAssemblerChildTidyHelper(w io.Writer) { // This function attempts to clean up the state machine to acknolwedge child assembly finish. // If the child was finished and we just collected it, return true and update state to maState_initial. // Otherwise, if it wasn't done, return false; // and the caller is almost certain to emit an error momentarily. // The function will only be called when the current state is maState_midValue. // (In general, the idea is that if the user is doing things correctly, // this function will only be called when the child is in fact finished.) // This is a *lot* simpler than the tidy behaviors needed for any of the other recursive kinds: // unions don't allow either nullable nor optional members, so there's no need to process anything except Maybe_Value state, // and the lack of need to consider nullable nor optionals also means we never need to worry about moving memory in the case of MaybeUsePtr modes. // (FUTURE: this may get a bit more conditional if we support members that are of unit ype and have null as a representation. Unsure how that would work out exactly, but should be possible.) // We don't bother to nil the child assembler's 'w' pointer: it's not necessary, // because we'll never "share" 'cm' (as some systems, like maps and lists, do) or change its value (short of the whole assembler resetting), // and therefore we should be able to rely on the child assembler to be reasonable and never start acting again after finish. // (This *does* mean some care is required in the reset logic: we have to be absolutely sure that resetting propagates to all child assemblers, // even if they're in other regions of the heap; otherwise, they might end up still holding actionable 'w' and 'm' pointers into bad times!) // (If you want to compare this to the logic in struct assemblers: it's similar to how only children that don't have maybes need an active 'w' nil'ing; // but the salient reason there isn't "because the don't have maybes"; it's "because they have a potentially-reused 'cm'". We don't have the former; but we *also* don't have the latter, for other reasons.) doTemplate(` func (ma *_{{ .Type | TypeSymbol }}__Assembler) valueFinishTidy() bool { switch ma.cm { case schema.Maybe_Value: {{- /* nothing to do for memlayout=embedAll; the tag is already set and memory already in place. */ -}} {{- /* nothing to do for memlayout=interface either; same story, the values are already in place. */ -}} ma.state = maState_initial return true default: return false } } `, w, g.AdjCfg, g) } func (g unionBuilderGenerator) emitMapAssemblerMethods(w io.Writer) { // DRY: I did an interesting thing here: the `switch ma.state` block remains textually identical to the one for structs, // even though the branch by valueFinishTidy could jump directly to an error state. // That same semantic error state gets checked separately a few lines later in a different mechanism. // The later check is needed either way (the assembler needs to *keep* erroring if some derp calls AssembleEntry *again* after a previous call already did the tidy and got rejected), // but we could arguably save a step there. It would probably trade more assembly size for the cycles saved, too, though. // Ah, tradeoffs. I think the textually simple approach here is probably in fact the best. But it could be done differently, yes. // Note that calling AssembleEntry again when it's not for the first entry *returns* an error; it doesn't panic. // This is subtle but important: trying to add more data than is acceptable is a data mismatch, not a system misuse, and must error accordingly politely. doTemplate(` func (ma *_{{ .Type | TypeSymbol }}__Assembler) AssembleEntry(k string) (datamodel.NodeAssembler, error) { switch ma.state { case maState_initial: // carry on case maState_midKey: panic("invalid state: AssembleEntry cannot be called when in the middle of assembling another key") case maState_expectValue: panic("invalid state: AssembleEntry cannot be called when expecting start of value assembly") case maState_midValue: if !ma.valueFinishTidy() { panic("invalid state: AssembleEntry cannot be called when in the middle of assembling a value") } // if tidy success: carry on for the moment, but we'll still be erroring shortly. case maState_finished: panic("invalid state: AssembleEntry cannot be called on an assembler that's already finished") } if ma.ca != 0 { return nil, schema.ErrNotUnionStructure{TypeName:"{{ .PkgName }}.{{ .Type.Name }}", Detail: "cannot add another entry -- a union can only contain one thing!"} } {{- if .Type.Members }} switch k { {{- range $i, $member := .Type.Members }} case "{{ $member.Name }}": ma.state = maState_midValue ma.ca = {{ add $i 1 }} {{- if (eq (dot.AdjCfg.UnionMemlayout dot.Type) "embedAll") }} ma.w.tag = {{ add $i 1 }} ma.ca{{ add $i 1 }}.w = &ma.w.x{{ add $i 1 }} ma.ca{{ add $i 1 }}.m = &ma.cm return &ma.ca{{ add $i 1 }}, nil {{- else if (eq (dot.AdjCfg.UnionMemlayout dot.Type) "interface") }} x := &_{{ $member | TypeSymbol}}{} ma.w.x = x if ma.ca{{ add $i 1 }} == nil { ma.ca{{ add $i 1 }} = &_{{ $member | TypeSymbol }}__Assembler{} } ma.ca{{ add $i 1 }}.w = x ma.ca{{ add $i 1 }}.m = &ma.cm return ma.ca{{ add $i 1 }}, nil {{- end}} {{- end}} {{- end}} } return nil, schema.ErrInvalidKey{TypeName:"{{ .PkgName }}.{{ .Type.Name }}", Key:&_String{k}} } `, w, g.AdjCfg, g) // AssembleKey has a similar DRY note as the AssembleEntry above had. // One misfortune in this method: we may know that we're doomed to errors because the caller is trying to start a second entry, // but we can't report it from this method: we have to sit on our tongue, slide to midKey state (even though we're doomed!), // and let the keyAssembler return the error later. // This sucks, but panicking wouldn't be correct (see remarks about error vs panic on the AssembleEntry method), // and we don't want to make this call unchainable for everyone everywhere, either, so it can't be rewritten to have an immediate error return. // The transition to midKey state is particularly irritating because it means this assembler will be perma-wedged; but I see no alternative. doTemplate(` func (ma *_{{ .Type | TypeSymbol }}__Assembler) AssembleKey() datamodel.NodeAssembler { switch ma.state { case maState_initial: // carry on case maState_midKey: panic("invalid state: AssembleKey cannot be called when in the middle of assembling another key") case maState_expectValue: panic("invalid state: AssembleKey cannot be called when expecting start of value assembly") case maState_midValue: if !ma.valueFinishTidy() { panic("invalid state: AssembleKey cannot be called when in the middle of assembling a value") } // if tidy success: carry on for the moment, but we'll still be erroring shortly... or rather, the keyassembler will be. case maState_finished: panic("invalid state: AssembleKey cannot be called on an assembler that's already finished") } ma.state = maState_midKey return (*_{{ .Type | TypeSymbol }}__KeyAssembler)(ma) } `, w, g.AdjCfg, g) // As with structs, the responsibilties of this are similar to AssembleEntry, but with some of the burden split into the key assembler (which should have acted earlier), // and some of the logical continuity bounces through state in the form of 'ma.ca'. // The potential to DRY up some of this should be plentiful, but it's a bit heady. doTemplate(` func (ma *_{{ .Type | TypeSymbol }}__Assembler) AssembleValue() datamodel.NodeAssembler { switch ma.state { case maState_initial: panic("invalid state: AssembleValue cannot be called when no key is primed") case maState_midKey: panic("invalid state: AssembleValue cannot be called when in the middle of assembling a key") case maState_expectValue: // carry on case maState_midValue: panic("invalid state: AssembleValue cannot be called when in the middle of assembling another value") case maState_finished: panic("invalid state: AssembleValue cannot be called on an assembler that's already finished") } ma.state = maState_midValue switch ma.ca { {{- range $i, $member := .Type.Members }} case {{ add $i 1 }}: {{- if (eq (dot.AdjCfg.UnionMemlayout dot.Type) "embedAll") }} ma.ca{{ add $i 1 }}.w = &ma.w.x{{ add $i 1 }} ma.ca{{ add $i 1 }}.m = &ma.cm return &ma.ca{{ add $i 1 }} {{- else if (eq (dot.AdjCfg.UnionMemlayout dot.Type) "interface") }} x := &_{{ $member | TypeSymbol}}{} ma.w.x = x if ma.ca{{ add $i 1 }} == nil { ma.ca{{ add $i 1 }} = &_{{ $member | TypeSymbol }}__Assembler{} } ma.ca{{ add $i 1 }}.w = x ma.ca{{ add $i 1 }}.m = &ma.cm return ma.ca{{ add $i 1 }} {{- end}} {{- end}} default: panic("unreachable") } } `, w, g.AdjCfg, g) // Finish checks are nice and easy. Is the maState in the right place now and was a 'ca' ever marked? // If yes and yes, then together with the rules elsewhere, we must've processed and accepted exactly one entry; perfect. doTemplate(` func (ma *_{{ .Type | TypeSymbol }}__Assembler) Finish() error { switch ma.state { case maState_initial: // carry on case maState_midKey: panic("invalid state: Finish cannot be called when in the middle of assembling a key") case maState_expectValue: panic("invalid state: Finish cannot be called when expecting start of value assembly") case maState_midValue: if !ma.valueFinishTidy() { panic("invalid state: Finish cannot be called when in the middle of assembling a value") } // if tidy success: carry on case maState_finished: panic("invalid state: Finish cannot be called on an assembler that's already finished") } if ma.ca == 0 { return schema.ErrNotUnionStructure{TypeName:"{{ .PkgName }}.{{ .Type.Name }}", Detail: "a union must have exactly one entry (not none)!"} } ma.state = maState_finished *ma.m = schema.Maybe_Value return nil } `, w, g.AdjCfg, g) doTemplate(` func (ma *_{{ .Type | TypeSymbol }}__Assembler) KeyPrototype() datamodel.NodePrototype { return _String__Prototype{} } func (ma *_{{ .Type | TypeSymbol }}__Assembler) ValuePrototype(k string) datamodel.NodePrototype { switch k { {{- range $i, $member := .Type.Members }} case "{{ $member.Name }}": return _{{ $member | TypeSymbol }}__Prototype{} {{- end}} default: return nil } } `, w, g.AdjCfg, g) } func (g unionBuilderGenerator) emitKeyAssembler(w io.Writer) { doTemplate(` type _{{ .Type | TypeSymbol }}__KeyAssembler _{{ .Type | TypeSymbol }}__Assembler `, w, g.AdjCfg, g) stubs := mixins.StringAssemblerTraits{ PkgName: g.PkgName, TypeName: g.TypeName + ".KeyAssembler", AppliedPrefix: "_" + g.AdjCfg.TypeSymbol(g.Type) + "__Key", } // This key assembler can disregard any idea of complex keys because we're fronting for a union! // Union member names must be strings (and quite simple ones at that). stubs.EmitNodeAssemblerMethodBeginMap(w) stubs.EmitNodeAssemblerMethodBeginList(w) stubs.EmitNodeAssemblerMethodAssignNull(w) stubs.EmitNodeAssemblerMethodAssignBool(w) stubs.EmitNodeAssemblerMethodAssignInt(w) stubs.EmitNodeAssemblerMethodAssignFloat(w) doTemplate(` func (ka *_{{ .Type | TypeSymbol }}__KeyAssembler) AssignString(k string) error { if ka.state != maState_midKey { panic("misuse: KeyAssembler held beyond its valid lifetime") } if ka.ca != 0 { return schema.ErrNotUnionStructure{TypeName:"{{ .PkgName }}.{{ .Type.Name }}", Detail: "cannot add another entry -- a union can only contain one thing!"} } switch k { {{- range $i, $member := .Type.Members }} case "{{ $member.Name }}": ka.ca = {{ add $i 1 }} {{- if (eq (dot.AdjCfg.UnionMemlayout dot.Type) "embedAll") }} ka.w.tag = {{ add $i 1 }} {{- end}} ka.state = maState_expectValue return nil {{- end}} } return schema.ErrInvalidKey{TypeName:"{{ .PkgName }}.{{ .Type.Name }}", Key:&_String{k}} // TODO: error quality: ErrInvalidUnionDiscriminant ? } `, w, g.AdjCfg, g) stubs.EmitNodeAssemblerMethodAssignBytes(w) stubs.EmitNodeAssemblerMethodAssignLink(w) doTemplate(` func (ka *_{{ .Type | TypeSymbol }}__KeyAssembler) AssignNode(v datamodel.Node) error { if v2, err := v.AsString(); err != nil { return err } else { return ka.AssignString(v2) } } func (_{{ .Type | TypeSymbol }}__KeyAssembler) Prototype() datamodel.NodePrototype { return _String__Prototype{} } `, w, g.AdjCfg, g) } ================================================ FILE: schema/gen/go/genUnionReprKeyed.go ================================================ package gengo import ( "io" "github.com/ipld/go-ipld-prime/schema" "github.com/ipld/go-ipld-prime/schema/gen/go/mixins" ) var _ TypeGenerator = &unionReprKeyedGenerator{} // General observation: many things about the keyed representation of unions is *very* similar to the type-level code, // because the type level code effective does espouse keyed-style behavior (just with type names as the keys). // Be advised that this similarity does not hold at *all* true of any of the other representation modes of unions! func NewUnionReprKeyedGenerator(pkgName string, typ *schema.TypeUnion, adjCfg *AdjunctCfg) TypeGenerator { return unionReprKeyedGenerator{ unionGenerator{ adjCfg, mixins.MapTraits{ PkgName: pkgName, TypeName: string(typ.Name()), TypeSymbol: adjCfg.TypeSymbol(typ), }, pkgName, typ, }, } } type unionReprKeyedGenerator struct { unionGenerator } func (g unionReprKeyedGenerator) GetRepresentationNodeGen() NodeGenerator { return unionReprKeyedReprGenerator{ g.AdjCfg, mixins.MapTraits{ PkgName: g.PkgName, TypeName: string(g.Type.Name()) + ".Repr", TypeSymbol: "_" + g.AdjCfg.TypeSymbol(g.Type) + "__Repr", }, g.PkgName, g.Type, } } type unionReprKeyedReprGenerator struct { AdjCfg *AdjunctCfg mixins.MapTraits PkgName string Type *schema.TypeUnion } func (unionReprKeyedReprGenerator) IsRepr() bool { return true } // hint used in some generalized templates. func (g unionReprKeyedReprGenerator) EmitNodeType(w io.Writer) { // The type is structurally the same, but will have a different set of methods. doTemplate(` type _{{ .Type | TypeSymbol }}__Repr _{{ .Type | TypeSymbol }} `, w, g.AdjCfg, g) // We do also want some constants for our discriminant values; // they'll make iterators able to work faster. doTemplate(` var ( {{- range $member := .Type.Members }} memberName__{{ dot.Type | TypeSymbol }}_{{ $member.Name }}_serial = _String{"{{ $member | dot.Type.RepresentationStrategy.GetDiscriminant }}"} {{- end }} ) `, w, g.AdjCfg, g) } func (g unionReprKeyedReprGenerator) EmitNodeTypeAssertions(w io.Writer) { doTemplate(` var _ datamodel.Node = &_{{ .Type | TypeSymbol }}__Repr{} `, w, g.AdjCfg, g) } func (g unionReprKeyedReprGenerator) EmitNodeMethodLookupByString(w io.Writer) { // Similar to the type-level method, except uses discriminant values as keys instead of the member type names. doTemplate(` func (n *_{{ .Type | TypeSymbol }}__Repr) LookupByString(key string) (datamodel.Node, error) { switch key { {{- range $i, $member := .Type.Members }} case "{{ $member | dot.Type.RepresentationStrategy.GetDiscriminant }}": {{- if (eq (dot.AdjCfg.UnionMemlayout dot.Type) "embedAll") }} if n.tag != {{ add $i 1 }} { return nil, datamodel.ErrNotExists{Segment: datamodel.PathSegmentOfString(key)} } return n.x{{ add $i 1 }}.Representation(), nil {{- else if (eq (dot.AdjCfg.UnionMemlayout dot.Type) "interface") }} if n2, ok := n.x.({{ $member | TypeSymbol }}); ok { return n2.Representation(), nil } else { return nil, datamodel.ErrNotExists{Segment: datamodel.PathSegmentOfString(key)} } {{- end}} {{- end}} default: return nil, schema.ErrNoSuchField{Type: nil /*TODO*/, Field: datamodel.PathSegmentOfString(key)} } } `, w, g.AdjCfg, g) } func (g unionReprKeyedReprGenerator) EmitNodeMethodLookupByNode(w io.Writer) { doTemplate(` func (n *_{{ .Type | TypeSymbol }}__Repr) LookupByNode(key datamodel.Node) (datamodel.Node, error) { ks, err := key.AsString() if err != nil { return nil, err } return n.LookupByString(ks) } `, w, g.AdjCfg, g) } func (g unionReprKeyedReprGenerator) EmitNodeMethodMapIterator(w io.Writer) { // Similar to the type-level method, except yields discriminant values as keys instead of the member type names. doTemplate(` func (n *_{{ .Type | TypeSymbol }}__Repr) MapIterator() datamodel.MapIterator { return &_{{ .Type | TypeSymbol }}__ReprMapItr{n, false} } type _{{ .Type | TypeSymbol }}__ReprMapItr struct { n *_{{ .Type | TypeSymbol }}__Repr done bool } func (itr *_{{ .Type | TypeSymbol }}__ReprMapItr) Next() (k datamodel.Node, v datamodel.Node, _ error) { if itr.done { return nil, nil, datamodel.ErrIteratorOverread{} } {{- if (eq (.AdjCfg.UnionMemlayout .Type) "embedAll") }} switch itr.n.tag { {{- range $i, $member := .Type.Members }} case {{ add $i 1 }}: k, v = &memberName__{{ dot.Type | TypeSymbol }}_{{ $member.Name }}_serial, itr.n.x{{ add $i 1 }}.Representation() {{- end}} {{- else if (eq (.AdjCfg.UnionMemlayout .Type) "interface") }} switch n2 := itr.n.x.(type) { {{- range $member := .Type.Members }} case {{ $member | TypeSymbol }}: k, v = &memberName__{{ dot.Type | TypeSymbol }}_{{ $member.Name }}_serial, n2.Representation() {{- end}} {{- end}} default: panic("unreachable") } itr.done = true return } func (itr *_{{ .Type | TypeSymbol }}__ReprMapItr) Done() bool { return itr.done } `, w, g.AdjCfg, g) } func (g unionReprKeyedReprGenerator) EmitNodeMethodLength(w io.Writer) { doTemplate(` func (_{{ .Type | TypeSymbol }}__Repr) Length() int64 { return 1 } `, w, g.AdjCfg, g) } func (g unionReprKeyedReprGenerator) EmitNodeMethodPrototype(w io.Writer) { emitNodeMethodPrototype_typical(w, g.AdjCfg, g) } func (g unionReprKeyedReprGenerator) EmitNodePrototypeType(w io.Writer) { emitNodePrototypeType_typical(w, g.AdjCfg, g) } // --- NodeBuilder and NodeAssembler ---> func (g unionReprKeyedReprGenerator) GetNodeBuilderGenerator() NodeBuilderGenerator { return unionReprKeyedReprBuilderGenerator{ g.AdjCfg, mixins.MapAssemblerTraits{ PkgName: g.PkgName, TypeName: g.TypeName, AppliedPrefix: "_" + g.AdjCfg.TypeSymbol(g.Type) + "__Repr", }, g.PkgName, g.Type, } } type unionReprKeyedReprBuilderGenerator struct { AdjCfg *AdjunctCfg mixins.MapAssemblerTraits PkgName string Type *schema.TypeUnion } func (unionReprKeyedReprBuilderGenerator) IsRepr() bool { return true } // hint used in some generalized templates. func (g unionReprKeyedReprBuilderGenerator) EmitNodeBuilderType(w io.Writer) { emitEmitNodeBuilderType_typical(w, g.AdjCfg, g) } func (g unionReprKeyedReprBuilderGenerator) EmitNodeBuilderMethods(w io.Writer) { emitNodeBuilderMethods_typical(w, g.AdjCfg, g) } func (g unionReprKeyedReprBuilderGenerator) EmitNodeAssemblerType(w io.Writer) { // Nearly identical to the type-level system, except it embeds the Repr variant of child assemblers // (which is a very minor difference textually, but means this structure can end up with a pretty wildly different resident memory size than the type-level one). doTemplate(` type _{{ .Type | TypeSymbol }}__ReprAssembler struct { w *_{{ .Type | TypeSymbol }} m *schema.Maybe state maState cm schema.Maybe {{- range $i, $member := .Type.Members }} ca{{ add $i 1 }} {{ if (eq (dot.AdjCfg.UnionMemlayout dot.Type) "interface") }}*{{end}}_{{ $member | TypeSymbol }}__ReprAssembler {{end -}} ca uint } `, w, g.AdjCfg, g) // Reset methods: also nearly identical to the type-level ones. doTemplate(` func (na *_{{ .Type | TypeSymbol }}__ReprAssembler) reset() { na.state = maState_initial switch na.ca { case 0: return {{- range $i, $member := .Type.Members }} case {{ add $i 1 }}: na.ca{{ add $i 1 }}.reset() {{end -}} default: panic("unreachable") } na.ca = 0 na.cm = schema.Maybe_Absent } `, w, g.AdjCfg, g) } func (g unionReprKeyedReprBuilderGenerator) EmitNodeAssemblerMethodBeginMap(w io.Writer) { emitNodeAssemblerMethodBeginMap_strictoid(w, g.AdjCfg, g) } func (g unionReprKeyedReprBuilderGenerator) EmitNodeAssemblerMethodAssignNull(w io.Writer) { // It might sound a bit odd to call a union "recursive", since it's so very trivially so (no fan-out), // but it's functionally accurate: the generated method should include a branch for the 'midvalue' state. emitNodeAssemblerMethodAssignNull_recursive(w, g.AdjCfg, g) } func (g unionReprKeyedReprBuilderGenerator) EmitNodeAssemblerMethodAssignNode(w io.Writer) { // DRY: this is once again not-coincidentally very nearly equal to the type-level method. Would be good to dedup them... after we do the get-to-the-point-in-phase-3 improvement. doTemplate(` func (na *_{{ .Type | TypeSymbol }}__ReprAssembler) AssignNode(v datamodel.Node) error { if v.IsNull() { return na.AssignNull() } if v2, ok := v.(*_{{ .Type | TypeSymbol }}); ok { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") case midvalue: panic("invalid state: cannot assign null into an assembler that's already begun working on recursive structures!") } {{- if .Type | MaybeUsesPtr }} if na.w == nil { na.w = v2 *na.m = schema.Maybe_Value return nil } {{- end}} *na.w = *v2 *na.m = schema.Maybe_Value return nil } if v.Kind() != datamodel.Kind_Map { return datamodel.ErrWrongKind{TypeName: "{{ .PkgName }}.{{ .Type.Name }}.Repr", MethodName: "AssignNode", AppropriateKind: datamodel.KindSet_JustMap, ActualKind: v.Kind()} } itr := v.MapIterator() for !itr.Done() { k, v, err := itr.Next() if err != nil { return err } if err := na.AssembleKey().AssignNode(k); err != nil { return err } if err := na.AssembleValue().AssignNode(v); err != nil { return err } } return na.Finish() } `, w, g.AdjCfg, g) } func (g unionReprKeyedReprBuilderGenerator) EmitNodeAssemblerOtherBits(w io.Writer) { g.emitMapAssemblerChildTidyHelper(w) g.emitMapAssemblerMethods(w) g.emitKeyAssembler(w) } func (g unionReprKeyedReprBuilderGenerator) emitMapAssemblerChildTidyHelper(w io.Writer) { // Nearly identical to the type-level equivalent. doTemplate(` func (ma *_{{ .Type | TypeSymbol }}__ReprAssembler) valueFinishTidy() bool { switch ma.cm { case schema.Maybe_Value: {{- /* nothing to do for memlayout=embedAll; the tag is already set and memory already in place. */ -}} {{- /* nothing to do for memlayout=interface either; same story, the values are already in place. */ -}} ma.state = maState_initial return true default: return false } } `, w, g.AdjCfg, g) } func (g unionReprKeyedReprBuilderGenerator) emitMapAssemblerMethods(w io.Writer) { // All of these: shamelessly similar to the type-level equivalent, modulo a few appearances of "Repr". // Alright, and also the "discriminant values as keys instead of the member type names" thing. // DRY: the number of times these `ma.state` switches are appearing is truly intense! This is starting to look like one of them most important things to shrink the GSLOC/ASM size of! doTemplate(` func (ma *_{{ .Type | TypeSymbol }}__ReprAssembler) AssembleEntry(k string) (datamodel.NodeAssembler, error) { switch ma.state { case maState_initial: // carry on case maState_midKey: panic("invalid state: AssembleEntry cannot be called when in the middle of assembling another key") case maState_expectValue: panic("invalid state: AssembleEntry cannot be called when expecting start of value assembly") case maState_midValue: if !ma.valueFinishTidy() { panic("invalid state: AssembleEntry cannot be called when in the middle of assembling a value") } // if tidy success: carry on for the moment, but we'll still be erroring shortly. case maState_finished: panic("invalid state: AssembleEntry cannot be called on an assembler that's already finished") } if ma.ca != 0 { return nil, schema.ErrNotUnionStructure{TypeName:"{{ .PkgName }}.{{ .Type.Name }}.Repr", Detail: "cannot add another entry -- a union can only contain one thing!"} } {{- if .Type.Members }} switch k { {{- range $i, $member := .Type.Members }} case "{{ $member | dot.Type.RepresentationStrategy.GetDiscriminant }}": ma.state = maState_midValue ma.ca = {{ add $i 1 }} {{- if (eq (dot.AdjCfg.UnionMemlayout dot.Type) "embedAll") }} ma.w.tag = {{ add $i 1 }} ma.ca{{ add $i 1 }}.w = &ma.w.x{{ add $i 1 }} ma.ca{{ add $i 1 }}.m = &ma.cm return &ma.ca{{ add $i 1 }}, nil {{- else if (eq (dot.AdjCfg.UnionMemlayout dot.Type) "interface") }} x := &_{{ $member | TypeSymbol }}{} ma.w.x = x if ma.ca{{ add $i 1 }} == nil { ma.ca{{ add $i 1 }} = &_{{ $member | TypeSymbol }}__ReprAssembler{} } ma.ca{{ add $i 1 }}.w = x ma.ca{{ add $i 1 }}.m = &ma.cm return ma.ca{{ add $i 1 }}, nil {{- end}} {{- end}} } {{- end}} return nil, schema.ErrInvalidKey{TypeName:"{{ .PkgName }}.{{ .Type.Name }}.Repr", Key:&_String{k}} } `, w, g.AdjCfg, g) doTemplate(` func (ma *_{{ .Type | TypeSymbol }}__ReprAssembler) AssembleKey() datamodel.NodeAssembler { switch ma.state { case maState_initial: // carry on case maState_midKey: panic("invalid state: AssembleKey cannot be called when in the middle of assembling another key") case maState_expectValue: panic("invalid state: AssembleKey cannot be called when expecting start of value assembly") case maState_midValue: if !ma.valueFinishTidy() { panic("invalid state: AssembleKey cannot be called when in the middle of assembling a value") } // if tidy success: carry on for the moment, but we'll still be erroring shortly... or rather, the keyassembler will be. case maState_finished: panic("invalid state: AssembleKey cannot be called on an assembler that's already finished") } ma.state = maState_midKey return (*_{{ .Type | TypeSymbol }}__ReprKeyAssembler)(ma) } `, w, g.AdjCfg, g) doTemplate(` func (ma *_{{ .Type | TypeSymbol }}__ReprAssembler) AssembleValue() datamodel.NodeAssembler { switch ma.state { case maState_initial: panic("invalid state: AssembleValue cannot be called when no key is primed") case maState_midKey: panic("invalid state: AssembleValue cannot be called when in the middle of assembling a key") case maState_expectValue: // carry on case maState_midValue: panic("invalid state: AssembleValue cannot be called when in the middle of assembling another value") case maState_finished: panic("invalid state: AssembleValue cannot be called on an assembler that's already finished") } ma.state = maState_midValue switch ma.ca { {{- range $i, $member := .Type.Members }} case {{ add $i 1 }}: {{- if (eq (dot.AdjCfg.UnionMemlayout dot.Type) "embedAll") }} ma.ca{{ add $i 1 }}.w = &ma.w.x{{ add $i 1 }} ma.ca{{ add $i 1 }}.m = &ma.cm return &ma.ca{{ add $i 1 }} {{- else if (eq (dot.AdjCfg.UnionMemlayout dot.Type) "interface") }} x := &_{{ $member | TypeSymbol }}{} ma.w.x = x if ma.ca{{ add $i 1 }} == nil { ma.ca{{ add $i 1 }} = &_{{ $member | TypeSymbol }}__ReprAssembler{} } ma.ca{{ add $i 1 }}.w = x ma.ca{{ add $i 1 }}.m = &ma.cm return ma.ca{{ add $i 1 }} {{- end}} {{- end}} default: panic("unreachable") } } `, w, g.AdjCfg, g) doTemplate(` func (ma *_{{ .Type | TypeSymbol }}__ReprAssembler) Finish() error { switch ma.state { case maState_initial: // carry on case maState_midKey: panic("invalid state: Finish cannot be called when in the middle of assembling a key") case maState_expectValue: panic("invalid state: Finish cannot be called when expecting start of value assembly") case maState_midValue: if !ma.valueFinishTidy() { panic("invalid state: Finish cannot be called when in the middle of assembling a value") } // if tidy success: carry on case maState_finished: panic("invalid state: Finish cannot be called on an assembler that's already finished") } if ma.ca == 0 { return schema.ErrNotUnionStructure{TypeName:"{{ .PkgName }}.{{ .Type.Name }}.Repr", Detail: "a union must have exactly one entry (not none)!"} } ma.state = maState_finished *ma.m = schema.Maybe_Value return nil } `, w, g.AdjCfg, g) doTemplate(` func (ma *_{{ .Type | TypeSymbol }}__ReprAssembler) KeyPrototype() datamodel.NodePrototype { return _String__Prototype{} } func (ma *_{{ .Type | TypeSymbol }}__ReprAssembler) ValuePrototype(k string) datamodel.NodePrototype { switch k { {{- range $i, $member := .Type.Members }} case "{{ $member.Name }}": return _{{ $member | TypeSymbol }}__ReprPrototype{} {{- end}} default: return nil } } `, w, g.AdjCfg, g) } func (g unionReprKeyedReprBuilderGenerator) emitKeyAssembler(w io.Writer) { doTemplate(` type _{{ .Type | TypeSymbol }}__ReprKeyAssembler _{{ .Type | TypeSymbol }}__ReprAssembler `, w, g.AdjCfg, g) stubs := mixins.StringAssemblerTraits{ PkgName: g.PkgName, TypeName: g.TypeName + ".KeyAssembler", // ".Repr" is already in `g.TypeName`, so don't stutter the "Repr" part. AppliedPrefix: "_" + g.AdjCfg.TypeSymbol(g.Type) + "__ReprKey", } // This key assembler can disregard any idea of complex keys because we know that our discriminants are just strings! stubs.EmitNodeAssemblerMethodBeginMap(w) stubs.EmitNodeAssemblerMethodBeginList(w) stubs.EmitNodeAssemblerMethodAssignNull(w) stubs.EmitNodeAssemblerMethodAssignBool(w) stubs.EmitNodeAssemblerMethodAssignInt(w) stubs.EmitNodeAssemblerMethodAssignFloat(w) doTemplate(` func (ka *_{{ .Type | TypeSymbol }}__ReprKeyAssembler) AssignString(k string) error { if ka.state != maState_midKey { panic("misuse: KeyAssembler held beyond its valid lifetime") } if ka.ca != 0 { return schema.ErrNotUnionStructure{TypeName:"{{ .PkgName }}.{{ .Type.Name }}.Repr", Detail: "cannot add another entry -- a union can only contain one thing!"} } switch k { {{- range $i, $member := .Type.Members }} case "{{ $member | dot.Type.RepresentationStrategy.GetDiscriminant }}": ka.ca = {{ add $i 1 }} {{- if (eq (dot.AdjCfg.UnionMemlayout dot.Type) "embedAll") }} ka.w.tag = {{ add $i 1 }} {{- end}} ka.state = maState_expectValue return nil {{- end}} } return schema.ErrInvalidKey{TypeName:"{{ .PkgName }}.{{ .Type.Name }}.Repr", Key:&_String{k}} // TODO: error quality: ErrInvalidUnionDiscriminant ? } `, w, g.AdjCfg, g) stubs.EmitNodeAssemblerMethodAssignBytes(w) stubs.EmitNodeAssemblerMethodAssignLink(w) doTemplate(` func (ka *_{{ .Type | TypeSymbol }}__ReprKeyAssembler) AssignNode(v datamodel.Node) error { if v2, err := v.AsString(); err != nil { return err } else { return ka.AssignString(v2) } } func (_{{ .Type | TypeSymbol }}__ReprKeyAssembler) Prototype() datamodel.NodePrototype { return _String__Prototype{} } `, w, g.AdjCfg, g) } ================================================ FILE: schema/gen/go/genUnionReprKinded.go ================================================ package gengo import ( "io" "github.com/ipld/go-ipld-prime/schema" "github.com/ipld/go-ipld-prime/schema/gen/go/mixins" ) var _ TypeGenerator = &unionKindedGenerator{} // Kinded union representations are quite wild: their behavior varies almost completely per inhabitant, // and their implementation is generally delegating directly to something else, // rather than having an intermediate node (like most unions do, and like the type-level view of this same value will). // // This also means any error values can be a little weird: // sometimes they'll have the union's type name, but sometimes they'll have the inhabitant's type name instead; // this depends on whether the error is an ErrWrongKind that was found while checking the method for appropriateness on the union's inhabitant // versus if the error came from the union inhabitant itself after delegation occurred. func NewUnionReprKindedGenerator(pkgName string, typ *schema.TypeUnion, adjCfg *AdjunctCfg) TypeGenerator { return unionKindedGenerator{ unionGenerator{ adjCfg, mixins.MapTraits{ PkgName: pkgName, TypeName: string(typ.Name()), TypeSymbol: adjCfg.TypeSymbol(typ), }, pkgName, typ, }, } } type unionKindedGenerator struct { unionGenerator } func (g unionKindedGenerator) GetRepresentationNodeGen() NodeGenerator { return unionKindedReprGenerator{ g.AdjCfg, g.PkgName, g.Type, } } type unionKindedReprGenerator struct { // Note that there's no MapTraits (or any other FooTraits) mixin in this one! // This is no accident: *None* of them apply! AdjCfg *AdjunctCfg PkgName string Type *schema.TypeUnion } func (unionKindedReprGenerator) IsRepr() bool { return true } // hint used in some generalized templates. func (g unionKindedReprGenerator) EmitNodeType(w io.Writer) { // The type is structurally the same, but will have a different set of methods. doTemplate(` type _{{ .Type | TypeSymbol }}__Repr _{{ .Type | TypeSymbol }} `, w, g.AdjCfg, g) } func (g unionKindedReprGenerator) EmitNodeTypeAssertions(w io.Writer) { doTemplate(` var _ datamodel.Node = &_{{ .Type | TypeSymbol }}__Repr{} `, w, g.AdjCfg, g) } func (g unionKindedReprGenerator) EmitNodeMethodKind(w io.Writer) { doTemplate(` func (n *_{{ .Type | TypeSymbol }}__Repr) Kind() datamodel.Kind { {{- if (eq (.AdjCfg.UnionMemlayout .Type) "embedAll") }} switch n.tag { {{- range $i, $member := .Type.Members }} case {{ add $i 1 }}: return {{ $member.RepresentationBehavior | KindSymbol }} {{- end}} {{- else if (eq (.AdjCfg.UnionMemlayout .Type) "interface") }} switch n.x.(type) { {{- range $i, $member := .Type.Members }} case {{ $member | TypeSymbol }}: return {{ $member.RepresentationBehavior | KindSymbol }} {{- end}} {{- end}} default: panic("unreachable") } } `, w, g.AdjCfg, g) } func kindedUnionNodeMethodTemplateMunge( methodName string, // for error messages methodSig string, // output literally someSwitchClause string, // template condition for if *any* switch clause should be present condClause string, // template condition for the member this should match on when in the range retClause string, // clause returning the thing (how to delegate methodsig, generally) appropriateKind string, // for error messages nopeSentinel string, // for error return paths; generally the zero value for the first return type. nopeSentinelOnly bool, // true if this method has no error return, just the sentinel. ) string { // We really could just... call the methods directly (and elide the switch entirely all the time), in the case of the "interface" implementation strategy. // We don't, though, because that would deprive us of getting the union type's name in the wrong-kind errors... // and in addition to that being sadface in general, it would be downright unacceptable if that behavior varied based on implementation strategy. // // This error text doesn't tell us what the member kind is. This might read weirdly. // It's possible we could try to cram a description of the inhabitant into the "TypeName" since it's stringy; but unclear if that's a good idea either. // These template concatenations have evolved into a mess very quickly. This entire thing should be replaced. // String concatenations of template clauses is an atrociously unhygenic way to compose things; // it looked like we could limp by with it for a while, but it's gotten messier faster than expected. errorClause := `return ` + nopeSentinel if !nopeSentinelOnly { errorClause += `, datamodel.ErrWrongKind{TypeName: "{{ .PkgName }}.{{ .Type.Name }}.Repr", MethodName: "` + methodName + `", AppropriateKind: ` + appropriateKind + `, ActualKind: n.Kind()}` } return ` func (n *_{{ .Type | TypeSymbol }}__Repr) ` + methodSig + ` { ` + someSwitchClause + ` {{- if (eq (.AdjCfg.UnionMemlayout .Type) "embedAll") }} switch n.tag { {{- range $i, $member := .Type.Members }} ` + condClause + ` case {{ add $i 1 }}: return n.x{{ add $i 1 }}.Representation()` + retClause + ` {{- end}} {{- end}} {{- else if (eq (.AdjCfg.UnionMemlayout .Type) "interface") }} switch n2 := n.x.(type) { {{- range $i, $member := .Type.Members }} ` + condClause + ` case {{ $member | TypeSymbol }}: return n2.Representation()` + retClause + ` {{- end}} {{- end}} {{- end}} default: {{- end}} ` + errorClause + ` ` + someSwitchClause + ` } {{- end}} } ` } func (g unionKindedReprGenerator) EmitNodeMethodLookupByString(w io.Writer) { doTemplate(kindedUnionNodeMethodTemplateMunge( `LookupByString`, `LookupByString(key string) (datamodel.Node, error)`, `{{- if .Type.RepresentationStrategy.GetMember (Kind "map") }}`, `{{- if eq $member.RepresentationBehavior.String "map" }}`, `.LookupByString(key)`, `datamodel.KindSet_JustMap`, `nil`, false, ), w, g.AdjCfg, g) } func (g unionKindedReprGenerator) EmitNodeMethodLookupByIndex(w io.Writer) { doTemplate(kindedUnionNodeMethodTemplateMunge( `LookupByIndex`, `LookupByIndex(idx int64) (datamodel.Node, error)`, `{{- if .Type.RepresentationStrategy.GetMember (Kind "list") }}`, `{{- if eq $member.RepresentationBehavior.String "list" }}`, `.LookupByIndex(idx)`, `datamodel.KindSet_JustList`, `nil`, false, ), w, g.AdjCfg, g) } func (g unionKindedReprGenerator) EmitNodeMethodLookupByNode(w io.Writer) { doTemplate(kindedUnionNodeMethodTemplateMunge( `LookupByNode`, `LookupByNode(key datamodel.Node) (datamodel.Node, error)`, `{{- if or (.Type.RepresentationStrategy.GetMember (Kind "map")) (.Type.RepresentationStrategy.GetMember (Kind "list")) }}`, `{{- if or (eq $member.RepresentationBehavior.String "map") (eq $member.RepresentationBehavior.String "list") }}`, `.LookupByNode(key)`, `datamodel.KindSet_Recursive`, `nil`, false, ), w, g.AdjCfg, g) } func (g unionKindedReprGenerator) EmitNodeMethodLookupBySegment(w io.Writer) { doTemplate(kindedUnionNodeMethodTemplateMunge( `LookupBySegment`, `LookupBySegment(seg datamodel.PathSegment) (datamodel.Node, error)`, `{{- if or (.Type.RepresentationStrategy.GetMember (Kind "map")) (.Type.RepresentationStrategy.GetMember (Kind "list")) }}`, `{{- if or (eq $member.RepresentationBehavior.String "map") (eq $member.RepresentationBehavior.String "list") }}`, `.LookupBySegment(seg)`, `datamodel.KindSet_Recursive`, `nil`, false, ), w, g.AdjCfg, g) } func (g unionKindedReprGenerator) EmitNodeMethodMapIterator(w io.Writer) { doTemplate(kindedUnionNodeMethodTemplateMunge( `MapIterator`, `MapIterator() datamodel.MapIterator`, `{{- if .Type.RepresentationStrategy.GetMember (Kind "map") }}`, `{{- if eq $member.RepresentationBehavior.String "map" }}`, `.MapIterator()`, `datamodel.KindSet_JustMap`, `nil`, true, ), w, g.AdjCfg, g) } func (g unionKindedReprGenerator) EmitNodeMethodListIterator(w io.Writer) { doTemplate(kindedUnionNodeMethodTemplateMunge( `ListIterator`, `ListIterator() datamodel.ListIterator`, `{{- if .Type.RepresentationStrategy.GetMember (Kind "list") }}`, `{{- if eq $member.RepresentationBehavior.String "list" }}`, `.ListIterator()`, `datamodel.KindSet_JustList`, `nil`, true, ), w, g.AdjCfg, g) } func (g unionKindedReprGenerator) EmitNodeMethodLength(w io.Writer) { doTemplate(kindedUnionNodeMethodTemplateMunge( `Length`, `Length() int64`, `{{- if or (.Type.RepresentationStrategy.GetMember (Kind "map")) (.Type.RepresentationStrategy.GetMember (Kind "list")) }}`, `{{- if or (eq $member.RepresentationBehavior.String "map") (eq $member.RepresentationBehavior.String "list") }}`, `.Length()`, `datamodel.KindSet_Recursive`, `-1`, true, ), w, g.AdjCfg, g) } func (g unionKindedReprGenerator) EmitNodeMethodIsAbsent(w io.Writer) { doTemplate(` func (n *_{{ .Type | TypeSymbol }}__Repr) IsAbsent() bool { return false } `, w, g.AdjCfg, g) } func (g unionKindedReprGenerator) EmitNodeMethodIsNull(w io.Writer) { doTemplate(` func (n *_{{ .Type | TypeSymbol }}__Repr) IsNull() bool { return false } `, w, g.AdjCfg, g) } func (g unionKindedReprGenerator) EmitNodeMethodAsBool(w io.Writer) { doTemplate(kindedUnionNodeMethodTemplateMunge( `AsBool`, `AsBool() (bool, error)`, `{{- if .Type.RepresentationStrategy.GetMember (Kind "bool") }}`, `{{- if eq $member.RepresentationBehavior.String "bool" }}`, `.AsBool()`, `datamodel.KindSet_JustBool`, `false`, false, ), w, g.AdjCfg, g) } func (g unionKindedReprGenerator) EmitNodeMethodAsInt(w io.Writer) { doTemplate(kindedUnionNodeMethodTemplateMunge( `AsInt`, `AsInt() (int64, error)`, `{{- if .Type.RepresentationStrategy.GetMember (Kind "int") }}`, `{{- if eq $member.RepresentationBehavior.String "int" }}`, `.AsInt()`, `datamodel.KindSet_JustInt`, `0`, false, ), w, g.AdjCfg, g) } func (g unionKindedReprGenerator) EmitNodeMethodAsFloat(w io.Writer) { doTemplate(kindedUnionNodeMethodTemplateMunge( `AsFloat`, `AsFloat() (float64, error)`, `{{- if .Type.RepresentationStrategy.GetMember (Kind "float") }}`, `{{- if eq $member.RepresentationBehavior.String "float" }}`, `.AsFloat()`, `datamodel.KindSet_JustFloat`, `0`, false, ), w, g.AdjCfg, g) } func (g unionKindedReprGenerator) EmitNodeMethodAsString(w io.Writer) { doTemplate(kindedUnionNodeMethodTemplateMunge( `AsString`, `AsString() (string, error)`, `{{- if .Type.RepresentationStrategy.GetMember (Kind "string") }}`, `{{- if eq $member.RepresentationBehavior.String "string" }}`, `.AsString()`, `datamodel.KindSet_JustString`, `""`, false, ), w, g.AdjCfg, g) } func (g unionKindedReprGenerator) EmitNodeMethodAsBytes(w io.Writer) { doTemplate(kindedUnionNodeMethodTemplateMunge( `AsBytes`, `AsBytes() ([]byte, error)`, `{{- if .Type.RepresentationStrategy.GetMember (Kind "bytes") }}`, `{{- if eq $member.RepresentationBehavior.String "bytes" }}`, `.AsBytes()`, `datamodel.KindSet_JustBytes`, `nil`, false, ), w, g.AdjCfg, g) } func (g unionKindedReprGenerator) EmitNodeMethodAsLink(w io.Writer) { doTemplate(kindedUnionNodeMethodTemplateMunge( `AsLink`, `AsLink() (datamodel.Link, error)`, `{{- if .Type.RepresentationStrategy.GetMember (Kind "link") }}`, `{{- if eq $member.RepresentationBehavior.String "link" }}`, `.AsLink()`, `datamodel.KindSet_JustLink`, `nil`, false, ), w, g.AdjCfg, g) } func (g unionKindedReprGenerator) EmitNodeMethodPrototype(w io.Writer) { emitNodeMethodPrototype_typical(w, g.AdjCfg, g) } func (g unionKindedReprGenerator) EmitNodePrototypeType(w io.Writer) { emitNodePrototypeType_typical(w, g.AdjCfg, g) } // --- NodeBuilder and NodeAssembler ---> func (g unionKindedReprGenerator) GetNodeBuilderGenerator() NodeBuilderGenerator { return unionKindedReprBuilderGenerator(g) } type unionKindedReprBuilderGenerator struct { AdjCfg *AdjunctCfg PkgName string Type *schema.TypeUnion } func (unionKindedReprBuilderGenerator) IsRepr() bool { return true } // hint used in some generalized templates. func (g unionKindedReprBuilderGenerator) EmitNodeBuilderType(w io.Writer) { emitEmitNodeBuilderType_typical(w, g.AdjCfg, g) } func (g unionKindedReprBuilderGenerator) EmitNodeBuilderMethods(w io.Writer) { emitNodeBuilderMethods_typical(w, g.AdjCfg, g) } func (g unionKindedReprBuilderGenerator) EmitNodeAssemblerType(w io.Writer) { // Much of this is familiar: the 'w', the 'm' are all as usual. // Some things may look a little odd here compared to all other assemblers: // we're kinda halfway between what's conventionally seen for a scalar and what's conventionally seen for a recursive. // There's no 'maState' or 'laState'-typed fields (which feels like a scalar) because even if we end up acting like a map or list, that state is in the relevant child assembler. // We don't even have a 'cm' field, because we can get away with something really funky: we can just copy our own 'm' _pointer_ into children; our doneness and their doneness is the same. // We never have to worry about maybeism of our children; the nullable and optional modifiers aren't possible on union members. // (We *do* still have to consider null values though, as null is still a kind, and thus can be routed to one of our members!) // 'ca' is as it is in the type-level assembler: technically, not super necessary, except that it allows minimizing the amount of work that resetting needs to do. doTemplate(` type _{{ .Type | TypeSymbol }}__ReprAssembler struct { w *_{{ .Type | TypeSymbol }} m *schema.Maybe {{- range $i, $member := .Type.Members }} ca{{ add $i 1 }} {{ if (eq (dot.AdjCfg.UnionMemlayout dot.Type) "interface") }}*{{end}}_{{ $member | TypeSymbol }}__ReprAssembler {{- end}} ca uint } `, w, g.AdjCfg, g) doTemplate(` func (na *_{{ .Type | TypeSymbol }}__ReprAssembler) reset() { switch na.ca { case 0: return {{- range $i, $member := .Type.Members }} case {{ add $i 1 }}: na.ca{{ add $i 1 }}.reset() {{- end}} default: panic("unreachable") } na.ca = 0 } `, w, g.AdjCfg, g) } func kindedUnionNodeAssemblerMethodTemplateMunge( methodName string, // for error messages methodSig string, // output literally condClause string, // template condition for the member this should match on retClause string, // clause returning the thing (how to delegate methodsig, generally) twoReturns bool, // true if a nil should be returned as well as the error ) string { // The value pointed to by `na.m` isn't modified here, because we're sharing it with the child, who should do so. // This also means that value gets checked twice -- once by us, because we need to halt if we've already been used -- // and also a second time by the child when we delegate to it, which, unbeknownst to it, is irrelevant. // I don't see a good way to remedy this shy of making more granular (unexported!) methods. (Might be worth it.) // This probably also isn't the same for all of the assembler methods: the methods we delegate to aren't doing as many check branches when they're for scalars, // because they expected to be used in contexts where many values of the 'm' enum aren't reachable -- an expectation we've suddenly subverted with this path! // // FUTURE: The error returns here are deeply questionable, and not as helpful as they could be. // ErrNotUnionStructure is about the most applicable thing so far, but it's very freetext. // ErrWrongKind wouldn't fit at all: assumes that we can say what kind of node we have, but this is the one case in the whole system where *we can't*; also, assumes there's one actual correct kind, and that too is false here! maybeNilComma := "" if twoReturns { maybeNilComma += "nil," } return ` func (na *_{{ .Type | TypeSymbol }}__ReprAssembler) ` + methodSig + ` { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") case midvalue: panic("invalid state: cannot assign into assembler that's already working on a larger structure!") } {{- $returned := false -}} {{- range $i, $member := .Type.Members }} ` + condClause + ` {{- if dot.Type | MaybeUsesPtr }} if na.w == nil { na.w = &_{{ dot.Type | TypeSymbol }}{} } {{- end}} na.ca = {{ add $i 1 }} {{- if (eq (dot.AdjCfg.UnionMemlayout dot.Type) "embedAll") }} na.w.tag = {{ add $i 1 }} na.ca{{ add $i 1 }}.w = &na.w.x{{ add $i 1 }} na.ca{{ add $i 1 }}.m = na.m return na.ca{{ add $i 1 }}` + retClause + ` {{- else if (eq (dot.AdjCfg.UnionMemlayout dot.Type) "interface") }} x := &_{{ $member | TypeSymbol }}{} na.w.x = x if na.ca{{ add $i 1 }} == nil { na.ca{{ add $i 1 }} = &_{{ $member | TypeSymbol }}__ReprAssembler{} } na.ca{{ add $i 1 }}.w = x na.ca{{ add $i 1 }}.m = na.m return na.ca{{ add $i 1 }}` + retClause + ` {{- end}} {{- $returned = true -}} {{- end }} {{- end }} {{- if not $returned }} return ` + maybeNilComma + ` schema.ErrNotUnionStructure{TypeName: "{{ .PkgName }}.{{ .Type.Name }}.Repr", Detail: "` + methodName + ` called but is not valid for any of the kinds that are valid members of this union"} {{- end }} } ` } func (g unionKindedReprBuilderGenerator) EmitNodeAssemblerMethodBeginMap(w io.Writer) { doTemplate(kindedUnionNodeAssemblerMethodTemplateMunge( `BeginMap`, `BeginMap(sizeHint int64) (datamodel.MapAssembler, error)`, `{{- if eq $member.RepresentationBehavior.String "map" }}`, `.BeginMap(sizeHint)`, true, ), w, g.AdjCfg, g) } func (g unionKindedReprBuilderGenerator) EmitNodeAssemblerMethodBeginList(w io.Writer) { doTemplate(kindedUnionNodeAssemblerMethodTemplateMunge( `BeginList`, `BeginList(sizeHint int64) (datamodel.ListAssembler, error)`, `{{- if eq $member.RepresentationBehavior.String "list" }}`, `.BeginList(sizeHint)`, true, ), w, g.AdjCfg, g) } func (g unionKindedReprBuilderGenerator) EmitNodeAssemblerMethodAssignNull(w io.Writer) { // TODO: I think this may need some special handling to account for if our union is itself used in a nullable circumstance; that should overrule this behavior. doTemplate(kindedUnionNodeAssemblerMethodTemplateMunge( `AssignNull`, `AssignNull() error `, `{{- if eq $member.RepresentationBehavior.String "null" }}`, `.AssignNull()`, false, ), w, g.AdjCfg, g) } func (g unionKindedReprBuilderGenerator) EmitNodeAssemblerMethodAssignBool(w io.Writer) { doTemplate(kindedUnionNodeAssemblerMethodTemplateMunge( `AssignBool`, `AssignBool(v bool) error `, `{{- if eq $member.RepresentationBehavior.String "bool" }}`, `.AssignBool(v)`, false, ), w, g.AdjCfg, g) } func (g unionKindedReprBuilderGenerator) EmitNodeAssemblerMethodAssignInt(w io.Writer) { doTemplate(kindedUnionNodeAssemblerMethodTemplateMunge( `AssignInt`, `AssignInt(v int64) error `, `{{- if eq $member.RepresentationBehavior.String "int" }}`, `.AssignInt(v)`, false, ), w, g.AdjCfg, g) } func (g unionKindedReprBuilderGenerator) EmitNodeAssemblerMethodAssignFloat(w io.Writer) { doTemplate(kindedUnionNodeAssemblerMethodTemplateMunge( `AssignFloat`, `AssignFloat(v float64) error `, `{{- if eq $member.RepresentationBehavior.String "float" }}`, `.AssignFloat(v)`, false, ), w, g.AdjCfg, g) } func (g unionKindedReprBuilderGenerator) EmitNodeAssemblerMethodAssignString(w io.Writer) { doTemplate(kindedUnionNodeAssemblerMethodTemplateMunge( `AssignString`, `AssignString(v string) error `, `{{- if eq $member.RepresentationBehavior.String "string" }}`, `.AssignString(v)`, false, ), w, g.AdjCfg, g) } func (g unionKindedReprBuilderGenerator) EmitNodeAssemblerMethodAssignBytes(w io.Writer) { doTemplate(kindedUnionNodeAssemblerMethodTemplateMunge( `AssignBytes`, `AssignBytes(v []byte) error `, `{{- if eq $member.RepresentationBehavior.String "bytes" }}`, `.AssignBytes(v)`, false, ), w, g.AdjCfg, g) } func (g unionKindedReprBuilderGenerator) EmitNodeAssemblerMethodAssignLink(w io.Writer) { doTemplate(kindedUnionNodeAssemblerMethodTemplateMunge( `AssignLink`, `AssignLink(v datamodel.Link) error `, `{{- if eq $member.RepresentationBehavior.String "link" }}`, `.AssignLink(v)`, false, ), w, g.AdjCfg, g) } func (g unionKindedReprBuilderGenerator) EmitNodeAssemblerMethodAssignNode(w io.Writer) { // This is a very mundane AssignNode: it just calls out to the other methods on this type. // However, even that is a little more exciting than usual: because we can't *necessarily* reject any kind of arg, // we have the whole barrage of switch cases here. We then leave any particular rejections to those methods. // Several cases could be statically replaced with errors and it would be an improvement. // // Errors are problematic again, same as is noted in kindedUnionNodeAssemblerMethodTemplateMunge. // We also end up returning errors with other method names due to how we delegate; unfortunate. doTemplate(` func (na *_{{ .Type | TypeSymbol }}__ReprAssembler) AssignNode(v datamodel.Node) error { if v.IsNull() { return na.AssignNull() } if v2, ok := v.(*_{{ .Type | TypeSymbol }}); ok { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") case midvalue: panic("invalid state: cannot assign null into an assembler that's already begun working on recursive structures!") } {{- if .Type | MaybeUsesPtr }} if na.w == nil { na.w = v2 *na.m = schema.Maybe_Value return nil } {{- end}} *na.w = *v2 *na.m = schema.Maybe_Value return nil } switch v.Kind() { case datamodel.Kind_Bool: v2, _ := v.AsBool() return na.AssignBool(v2) case datamodel.Kind_Int: v2, _ := v.AsInt() return na.AssignInt(v2) case datamodel.Kind_Float: v2, _ := v.AsFloat() return na.AssignFloat(v2) case datamodel.Kind_String: v2, _ := v.AsString() return na.AssignString(v2) case datamodel.Kind_Bytes: v2, _ := v.AsBytes() return na.AssignBytes(v2) case datamodel.Kind_Map: na, err := na.BeginMap(v.Length()) if err != nil { return err } itr := v.MapIterator() for !itr.Done() { k, v, err := itr.Next() if err != nil { return err } if err := na.AssembleKey().AssignNode(k); err != nil { return err } if err := na.AssembleValue().AssignNode(v); err != nil { return err } } return na.Finish() case datamodel.Kind_List: na, err := na.BeginList(v.Length()) if err != nil { return err } itr := v.ListIterator() for !itr.Done() { _, v, err := itr.Next() if err != nil { return err } if err := na.AssembleValue().AssignNode(v); err != nil { return err } } return na.Finish() case datamodel.Kind_Link: v2, _ := v.AsLink() return na.AssignLink(v2) default: panic("unreachable") } } `, w, g.AdjCfg, g) } func (g unionKindedReprBuilderGenerator) EmitNodeAssemblerMethodPrototype(w io.Writer) { doTemplate(` func (na *_{{ .Type | TypeSymbol }}__ReprAssembler) Prototype() datamodel.NodePrototype { return _{{ .Type | TypeSymbol }}__ReprPrototype{} } `, w, g.AdjCfg, g) } func (g unionKindedReprBuilderGenerator) EmitNodeAssemblerOtherBits(w io.Writer) { // somewhat shockingly: nothing. } ================================================ FILE: schema/gen/go/genUnionReprStringprefix.go ================================================ package gengo import ( "io" "github.com/ipld/go-ipld-prime/schema" "github.com/ipld/go-ipld-prime/schema/gen/go/mixins" ) var _ TypeGenerator = &unionReprStringprefixGenerator{} func NewUnionReprStringprefixGenerator(pkgName string, typ *schema.TypeUnion, adjCfg *AdjunctCfg) TypeGenerator { return unionReprStringprefixGenerator{ unionGenerator{ AdjCfg: adjCfg, MapTraits: mixins.MapTraits{ PkgName: pkgName, TypeName: string(typ.Name()), TypeSymbol: adjCfg.TypeSymbol(typ), }, PkgName: pkgName, Type: typ, }, } } type unionReprStringprefixGenerator struct { unionGenerator } func (g unionReprStringprefixGenerator) GetRepresentationNodeGen() NodeGenerator { return unionReprStringprefixReprGenerator{ AdjCfg: g.AdjCfg, StringTraits: mixins.StringTraits{ PkgName: g.PkgName, TypeName: string(g.Type.Name()) + ".Repr", TypeSymbol: "_" + g.AdjCfg.TypeSymbol(g.Type) + "__Repr", }, PkgName: g.PkgName, Type: g.Type, } } type unionReprStringprefixReprGenerator struct { AdjCfg *AdjunctCfg mixins.StringTraits PkgName string Type *schema.TypeUnion } func (unionReprStringprefixReprGenerator) IsRepr() bool { return true } // hint used in some generalized templates. func (g unionReprStringprefixReprGenerator) EmitNodeType(w io.Writer) { // The type is structurally the same, but will have a different set of methods. doTemplate(` type _{{ .Type | TypeSymbol }}__Repr _{{ .Type | TypeSymbol }} `, w, g.AdjCfg, g) // We do also want some constants for our discriminant values; // they'll make iterators able to work faster. doTemplate(` var ( {{- range $member := .Type.Members }} memberName__{{ dot.Type | TypeSymbol }}_{{ $member.Name }}_serial = _String{"{{ $member | dot.Type.RepresentationStrategy.GetDiscriminant }}"} {{- end }} ) `, w, g.AdjCfg, g) } func (g unionReprStringprefixReprGenerator) EmitNodeTypeAssertions(w io.Writer) { doTemplate(` var _ datamodel.Node = &_{{ .Type | TypeSymbol }}__Repr{} `, w, g.AdjCfg, g) } func (g unionReprStringprefixReprGenerator) EmitNodeMethodAsString(w io.Writer) { // See comment block in structReprStringjoinReprGenerator.EmitNodeMethodAsString for a lot of philosophizing about this. doTemplate(` func (n *_{{ .Type | TypeSymbol }}__Repr) AsString() (string, error) { return n.String(), nil } func (n *_{{ .Type | TypeSymbol }}__Repr) String() string { {{- if (eq (.AdjCfg.UnionMemlayout .Type) "embedAll") }} switch n.tag { {{- range $i, $member := .Type.Members }} case {{ add $i 1 }}: return memberName__{{ dot.Type | TypeSymbol }}_{{ $member.Name }}_serial.String() + "{{ dot.Type.RepresentationStrategy.GetDelim }}" + n.x{{ add $i 1 }}.String() {{- end}} {{- else if (eq (.AdjCfg.UnionMemlayout .Type) "interface") }} switch n2 := n.x.(type) { {{- range $member := .Type.Members }} case {{ $member | TypeSymbol }}: return memberName__{{ dot.Type | TypeSymbol }}_{{ $member.Name }}_serial.String() + "{{ dot.Type.RepresentationStrategy.GetDelim }}" + n2.String() {{- end}} {{- end}} default: panic("unreachable") } } func (n {{ .Type | TypeSymbol }}) String() string { return (*_{{ .Type | TypeSymbol }}__Repr)(n).String() } `, w, g.AdjCfg, g) } func (g unionReprStringprefixReprGenerator) EmitNodeMethodPrototype(w io.Writer) { emitNodeMethodPrototype_typical(w, g.AdjCfg, g) } func (g unionReprStringprefixReprGenerator) EmitNodePrototypeType(w io.Writer) { emitNodePrototypeType_typical(w, g.AdjCfg, g) } // --- NodeBuilder and NodeAssembler ---> func (g unionReprStringprefixReprGenerator) GetNodeBuilderGenerator() NodeBuilderGenerator { return unionReprStringprefixReprBuilderGenerator{ g.AdjCfg, mixins.StringAssemblerTraits{ PkgName: g.PkgName, TypeName: g.TypeName, AppliedPrefix: "_" + g.AdjCfg.TypeSymbol(g.Type) + "__Repr", }, g.PkgName, g.Type, } } type unionReprStringprefixReprBuilderGenerator struct { AdjCfg *AdjunctCfg mixins.StringAssemblerTraits PkgName string Type *schema.TypeUnion } func (unionReprStringprefixReprBuilderGenerator) IsRepr() bool { return true } // hint used in some generalized templates. func (g unionReprStringprefixReprBuilderGenerator) EmitNodeBuilderType(w io.Writer) { emitEmitNodeBuilderType_typical(w, g.AdjCfg, g) } func (g unionReprStringprefixReprBuilderGenerator) EmitNodeBuilderMethods(w io.Writer) { emitNodeBuilderMethods_typical(w, g.AdjCfg, g) // Generate a single-step construction function -- this is easy to do for a scalar, // and all representations of scalar kind can be expected to have a method like this. // The function is attached to the NodePrototype for convenient namespacing; // it needs no new memory, so it would be inappropriate to attach to the builder or assembler. // The function is directly used internally by anything else that might involve recursive destructuring on the same scalar kind // (for example, structs using stringjoin strategies that have one of this type as a field, etc). // Since we're a representation of scalar kind, and can recurse, // we ourselves presume this plain construction method must also exist for all our members. // REVIEW: We could make an immut-safe version of this and export it on the NodePrototype too, as `FromString(string)`. doTemplate(` func (_{{ .Type | TypeSymbol }}__ReprPrototype) fromString(w *_{{ .Type | TypeSymbol }}, v string) error { ss := mixins.SplitN(v, "{{ .Type.RepresentationStrategy.GetDelim }}", 2) if len(ss) != 2 { return schema.ErrUnmatchable{TypeName:"{{ .PkgName }}.{{ .Type.Name }}.Repr"}.Reasonf("expecting a stringprefix union but found no delimiter in the value") } switch ss[0] { {{- range $i, $member := .Type.Members }} case "{{ $member | dot.Type.RepresentationStrategy.GetDiscriminant }}": {{- if (eq (dot.AdjCfg.UnionMemlayout dot.Type) "embedAll") }} w.tag = {{ add $i 1 }} if err := (_{{ $member | TypeSymbol }}__ReprPrototype{}).fromString(&w.x{{ add $i 1 }}, ss[1]); err != nil { return schema.ErrUnmatchable{TypeName:"{{ dot.PkgName }}.{{ dot.Type.Name }}.Repr", Reason: err} } return nil {{- else if (eq (dot.AdjCfg.UnionMemlayout dot.Type) "interface") }} var n2 _{{ $member | TypeSymbol }} if err := (_{{ $member | TypeSymbol }}__ReprPrototype{}).fromString(&n2, ss[1]); err != nil { return schema.ErrUnmatchable{TypeName:"{{ dot.PkgName }}.{{ dot.Type.Name }}.Repr", Reason: err} } w.x = &n2 return nil {{- end}} {{- end}} default: return schema.ErrNoSuchField{Type: nil /*TODO*/, Field: datamodel.PathSegmentOfString(ss[0])} } } `, w, g.AdjCfg, g) } func (g unionReprStringprefixReprBuilderGenerator) EmitNodeAssemblerType(w io.Writer) { doTemplate(` type _{{ .Type | TypeSymbol }}__ReprAssembler struct { w *_{{ .Type | TypeSymbol }} m *schema.Maybe } func (na *_{{ .Type | TypeSymbol }}__ReprAssembler) reset() {} `, w, g.AdjCfg, g) } func (g unionReprStringprefixReprBuilderGenerator) EmitNodeAssemblerMethodAssignNull(w io.Writer) { emitNodeAssemblerMethodAssignNull_scalar(w, g.AdjCfg, g) } func (g unionReprStringprefixReprBuilderGenerator) EmitNodeAssemblerMethodAssignString(w io.Writer) { // This method contains a branch to support MaybeUsesPtr because new memory may need to be allocated. // This allocation only happens if the 'w' ptr is nil, which means we're being used on a Maybe; // otherwise, the 'w' ptr should already be set, and we fill that memory location without allocating, as usual. // TODO:DRY: this is identical to other string-repr-on-non-string-type. doTemplate(` func (na *_{{ .Type | TypeSymbol }}__ReprAssembler) AssignString(v string) error { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") } {{- if .Type | MaybeUsesPtr }} if na.w == nil { na.w = &_{{ .Type | TypeSymbol }}{} } {{- end}} if err := (_{{ .Type | TypeSymbol }}__ReprPrototype{}).fromString(na.w, v); err != nil { return err } *na.m = schema.Maybe_Value return nil } `, w, g.AdjCfg, g) } func (g unionReprStringprefixReprBuilderGenerator) EmitNodeAssemblerMethodAssignNode(w io.Writer) { // AssignNode goes through three phases: // 1. is it null? Jump over to AssignNull (which may or may not reject it). // 2. is it our own type? Handle specially -- we might be able to do efficient things. // 3. is it the right kind to morph into us? Do so. // TODO:DRY: this is identical to other string-repr-on-non-string-type. doTemplate(` func (na *_{{ .Type | TypeSymbol }}__ReprAssembler) AssignNode(v datamodel.Node) error { if v.IsNull() { return na.AssignNull() } if v2, ok := v.(*_{{ .Type | TypeSymbol }}); ok { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") } {{- if .Type | MaybeUsesPtr }} if na.w == nil { na.w = v2 *na.m = schema.Maybe_Value return nil } {{- end}} *na.w = *v2 *na.m = schema.Maybe_Value return nil } if v2, err := v.AsString(); err != nil { return err } else { return na.AssignString(v2) } } `, w, g.AdjCfg, g) } func (g unionReprStringprefixReprBuilderGenerator) EmitNodeAssemblerOtherBits(w io.Writer) { // None for this. } ================================================ FILE: schema/gen/go/generate.go ================================================ package gengo import ( "bytes" "fmt" "go/format" "io" "os" "path/filepath" "sort" "github.com/ipld/go-ipld-prime/schema" ) // Generate takes a typesystem and the adjunct config for codegen, // and emits generated code in the given path with the given package name. // // All of the files produced will match the pattern "ipldsch.*.gen.go". func Generate(pth string, pkgName string, ts schema.TypeSystem, adjCfg *AdjunctCfg) { // Emit fixed bits. withFile(filepath.Join(pth, "ipldsch_minima.go"), func(f io.Writer) { EmitInternalEnums(pkgName, f) }) externs, err := getExternTypes(pth) if err != nil { // Consider warning that duplication may be present due to inability to parse destination. externs = make(map[string]struct{}) } // Local helper function for applying generation logic to each type. // We will end up doing this more than once because in this layout, more than one file contains part of the story for each type. applyToEachType := func(fn func(tg TypeGenerator, w io.Writer), f io.Writer) { // Sort the type names so we have a determinisic order; this affects output consistency. // Any stable order would do, but we don't presently have one, so a sort is necessary. types := ts.GetTypes() keys := make(sortableTypeNames, 0, len(types)) for tn := range types { if _, exists := externs[tn]; !exists { keys = append(keys, tn) } } sort.Sort(keys) for _, tn := range keys { switch t2 := types[tn].(type) { case *schema.TypeBool: fn(NewBoolReprBoolGenerator(pkgName, t2, adjCfg), f) case *schema.TypeInt: fn(NewIntReprIntGenerator(pkgName, t2, adjCfg), f) case *schema.TypeFloat: fn(NewFloatReprFloatGenerator(pkgName, t2, adjCfg), f) case *schema.TypeString: fn(NewStringReprStringGenerator(pkgName, t2, adjCfg), f) case *schema.TypeBytes: fn(NewBytesReprBytesGenerator(pkgName, t2, adjCfg), f) case *schema.TypeLink: fn(NewLinkReprLinkGenerator(pkgName, t2, adjCfg), f) case *schema.TypeStruct: switch t2.RepresentationStrategy().(type) { case schema.StructRepresentation_Map: fn(NewStructReprMapGenerator(pkgName, t2, adjCfg), f) case schema.StructRepresentation_Tuple: fn(NewStructReprTupleGenerator(pkgName, t2, adjCfg), f) case schema.StructRepresentation_Stringjoin: fn(NewStructReprStringjoinGenerator(pkgName, t2, adjCfg), f) default: panic("unrecognized struct representation strategy") } case *schema.TypeMap: fn(NewMapReprMapGenerator(pkgName, t2, adjCfg), f) case *schema.TypeList: fn(NewListReprListGenerator(pkgName, t2, adjCfg), f) case *schema.TypeUnion: switch t2.RepresentationStrategy().(type) { case schema.UnionRepresentation_Keyed: fn(NewUnionReprKeyedGenerator(pkgName, t2, adjCfg), f) case schema.UnionRepresentation_Kinded: fn(NewUnionReprKindedGenerator(pkgName, t2, adjCfg), f) case schema.UnionRepresentation_Stringprefix: fn(NewUnionReprStringprefixGenerator(pkgName, t2, adjCfg), f) default: panic("unrecognized union representation strategy") } default: panic(fmt.Sprintf("add more type switches here :), failed at type %s", tn)) } } } // Emit a file with the type table, and the golang type defns for each type. withFile(filepath.Join(pth, "ipldsch_types.go"), func(f io.Writer) { // Emit headers, import statements, etc. fmt.Fprintf(f, "package %s\n\n", pkgName) fmt.Fprintf(f, doNotEditComment+"\n\n") fmt.Fprintf(f, "import (\n") fmt.Fprintf(f, "\t\"github.com/ipld/go-ipld-prime/datamodel\"\n") // referenced for links fmt.Fprintf(f, ")\n") fmt.Fprintf(f, "var _ datamodel.Node = nil // suppress errors when this dependency is not referenced\n") // Emit the type table. EmitTypeTable(pkgName, ts, adjCfg, f) // Emit the type defns matching the schema types. fmt.Fprintf(f, "\n// --- type definitions follow ---\n\n") applyToEachType(func(tg TypeGenerator, w io.Writer) { tg.EmitNativeType(w) fmt.Fprintf(f, "\n") }, f) }) // Emit a file with all the Node/NodeBuilder/NodeAssembler boilerplate. // Also includes typedefs for representation-level data. // Also includes the MaybeT boilerplate. withFile(filepath.Join(pth, "ipldsch_satisfaction.go"), func(f io.Writer) { // Emit headers, import statements, etc. fmt.Fprintf(f, "package %s\n\n", pkgName) fmt.Fprintf(f, doNotEditComment+"\n\n") fmt.Fprintf(f, "import (\n") fmt.Fprintf(f, "\t\"github.com/ipld/go-ipld-prime/datamodel\"\n") // referenced everywhere. fmt.Fprintf(f, "\t\"github.com/ipld/go-ipld-prime/node/mixins\"\n") // referenced by node implementation guts. fmt.Fprintf(f, "\t\"github.com/ipld/go-ipld-prime/schema\"\n") // referenced by maybes (and surprisingly little else). fmt.Fprintf(f, ")\n\n") // For each type, we'll emit... everything except the native type, really. applyToEachType(func(tg TypeGenerator, w io.Writer) { tg.EmitNativeAccessors(w) tg.EmitNativeBuilder(w) tg.EmitNativeMaybe(w) EmitNode(tg, w) tg.EmitTypedNodeMethodType(w) tg.EmitTypedNodeMethodRepresentation(w) nrg := tg.GetRepresentationNodeGen() EmitNode(nrg, w) fmt.Fprintf(f, "\n") }, f) }) } func withFile(filename string, fn func(io.Writer)) { // Don't write directly to the file, as that many write syscalls can be // expensive. Moreover, they can have a knock-on effect on daemons // watching for file changes. gopls can easily eat CPU for many seconds // just handling tens of thousands of file writes, for example. // // To alleviate both of those problems, write to a buffer first, and // then write the resulting bytes to disk in a single go. // A buffer is slightly better than bufio.Writer, as it gets us a bit // more atomicity via the single write. buf := new(bytes.Buffer) fn(buf) src := buf.Bytes() // Format the source before writing, just like gofmt would. // This also prevents us from writing invalid syntax to disk. src, err := format.Source(src) if err != nil { panic(err) } if err := os.WriteFile(filename, src, 0666); err != nil { panic(err) } } type sortableTypeNames []schema.TypeName func (a sortableTypeNames) Len() int { return len(a) } func (a sortableTypeNames) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a sortableTypeNames) Less(i, j int) bool { return a[i] < a[j] } ================================================ FILE: schema/gen/go/generators.go ================================================ package gengo import ( "fmt" "io" "github.com/ipld/go-ipld-prime/schema" ) // TypeGenerator gathers all the info for generating all code related to one // type in the schema. type TypeGenerator interface { // -- the natively-typed apis --> EmitNativeType(io.Writer) EmitNativeAccessors(io.Writer) // depends on the kind -- field accessors for struct, typed iterators for map, etc. EmitNativeBuilder(io.Writer) // typically emits some kind of struct that has a Build method. EmitNativeMaybe(io.Writer) // a pointer-free 'maybe' mechanism is generated for all types. // -- the schema.TypedNode.Type method and vars --> EmitTypeConst(io.Writer) // these emit dummies for now EmitTypedNodeMethodType(io.Writer) // these emit dummies for now // -- all node methods --> // (and note that the nodeBuilder for this one should be the "semantic" one, // e.g. it *always* acts like a map for structs, even if the repr is different.) NodeGenerator // -- and the representation and its node and nodebuilder --> // (these vary!) EmitTypedNodeMethodRepresentation(io.Writer) GetRepresentationNodeGen() NodeGenerator // includes transitively the matched NodeBuilderGenerator } type NodeGenerator interface { EmitNodeType(io.Writer) // usually already covered by EmitNativeType for the primary node, but has a nonzero body for the repr node EmitNodeTypeAssertions(io.Writer) // optional to include this content EmitNodeMethodKind(io.Writer) EmitNodeMethodLookupByString(io.Writer) EmitNodeMethodLookupByNode(io.Writer) EmitNodeMethodLookupByIndex(io.Writer) EmitNodeMethodLookupBySegment(io.Writer) EmitNodeMethodMapIterator(io.Writer) // also iterator itself EmitNodeMethodListIterator(io.Writer) // also iterator itself EmitNodeMethodLength(io.Writer) EmitNodeMethodIsAbsent(io.Writer) EmitNodeMethodIsNull(io.Writer) EmitNodeMethodAsBool(io.Writer) EmitNodeMethodAsInt(io.Writer) EmitNodeMethodAsFloat(io.Writer) EmitNodeMethodAsString(io.Writer) EmitNodeMethodAsBytes(io.Writer) EmitNodeMethodAsLink(io.Writer) EmitNodeMethodPrototype(io.Writer) EmitNodePrototypeType(io.Writer) GetNodeBuilderGenerator() NodeBuilderGenerator // assembler features also included inside } type NodeBuilderGenerator interface { EmitNodeBuilderType(io.Writer) EmitNodeBuilderMethods(io.Writer) // not many, so just slung them together. EmitNodeAssemblerType(io.Writer) // you can call this and not EmitNodeBuilderType in some situations. EmitNodeAssemblerMethodBeginMap(io.Writer) EmitNodeAssemblerMethodBeginList(io.Writer) EmitNodeAssemblerMethodAssignNull(io.Writer) EmitNodeAssemblerMethodAssignBool(io.Writer) EmitNodeAssemblerMethodAssignInt(io.Writer) EmitNodeAssemblerMethodAssignFloat(io.Writer) EmitNodeAssemblerMethodAssignString(io.Writer) EmitNodeAssemblerMethodAssignBytes(io.Writer) EmitNodeAssemblerMethodAssignLink(io.Writer) EmitNodeAssemblerMethodAssignNode(io.Writer) EmitNodeAssemblerMethodPrototype(io.Writer) EmitNodeAssemblerOtherBits(io.Writer) // key and value child assemblers are done here. } // EmitFileHeader emits a baseline package header that will // allow a file with a generated type to compile. // (Fortunately, there are no variations in this.) func EmitFileHeader(packageName string, w io.Writer) { fmt.Fprintf(w, "package %s\n\n", packageName) fmt.Fprintf(w, doNotEditComment+"\n\n") fmt.Fprintf(w, "import (\n") fmt.Fprintf(w, "\t\"github.com/ipld/go-ipld-prime/datamodel\"\n") fmt.Fprintf(w, "\t\"github.com/ipld/go-ipld-prime/node/mixins\"\n") fmt.Fprintf(w, "\t\"github.com/ipld/go-ipld-prime/schema\"\n") fmt.Fprintf(w, ")\n\n") } // EmitEntireType is a helper function calls all methods of TypeGenerator // and streams all results into a single writer. // (This implies two calls to EmitNode -- one for the type-level and one for the representation-level.) func EmitEntireType(tg TypeGenerator, w io.Writer) { tg.EmitNativeType(w) tg.EmitNativeAccessors(w) tg.EmitNativeBuilder(w) tg.EmitNativeMaybe(w) EmitNode(tg, w) tg.EmitTypedNodeMethodType(w) tg.EmitTypedNodeMethodRepresentation(w) rng := tg.GetRepresentationNodeGen() if rng == nil { // FIXME: hack to save me from stubbing tons right now, remove when done return } EmitNode(rng, w) } // EmitNode is a helper function that calls all methods of NodeGenerator // and streams all results into a single writer. func EmitNode(ng NodeGenerator, w io.Writer) { ng.EmitNodeType(w) ng.EmitNodeTypeAssertions(w) ng.EmitNodeMethodKind(w) ng.EmitNodeMethodLookupByString(w) ng.EmitNodeMethodLookupByNode(w) ng.EmitNodeMethodLookupByIndex(w) ng.EmitNodeMethodLookupBySegment(w) ng.EmitNodeMethodMapIterator(w) ng.EmitNodeMethodListIterator(w) ng.EmitNodeMethodLength(w) ng.EmitNodeMethodIsAbsent(w) ng.EmitNodeMethodIsNull(w) ng.EmitNodeMethodAsBool(w) ng.EmitNodeMethodAsInt(w) ng.EmitNodeMethodAsFloat(w) ng.EmitNodeMethodAsString(w) ng.EmitNodeMethodAsBytes(w) ng.EmitNodeMethodAsLink(w) ng.EmitNodeMethodPrototype(w) ng.EmitNodePrototypeType(w) nbg := ng.GetNodeBuilderGenerator() if nbg == nil { // FIXME: hack to save me from stubbing tons right now, remove when done return } nbg.EmitNodeBuilderType(w) nbg.EmitNodeBuilderMethods(w) nbg.EmitNodeAssemblerType(w) nbg.EmitNodeAssemblerMethodBeginMap(w) nbg.EmitNodeAssemblerMethodBeginList(w) nbg.EmitNodeAssemblerMethodAssignNull(w) nbg.EmitNodeAssemblerMethodAssignBool(w) nbg.EmitNodeAssemblerMethodAssignInt(w) nbg.EmitNodeAssemblerMethodAssignFloat(w) nbg.EmitNodeAssemblerMethodAssignString(w) nbg.EmitNodeAssemblerMethodAssignBytes(w) nbg.EmitNodeAssemblerMethodAssignLink(w) nbg.EmitNodeAssemblerMethodAssignNode(w) nbg.EmitNodeAssemblerMethodPrototype(w) nbg.EmitNodeAssemblerOtherBits(w) } func EmitTypeTable(pkgName string, ts schema.TypeSystem, adjCfg *AdjunctCfg, w io.Writer) { // REVIEW: if "T__Repr" is how we want to expose this. We could also put 'Repr' accessors on the type/prototype objects. // FUTURE: types and prototypes are proposed to be the same. Some of this text pretends they already are, but work is needed on this. doTemplate(` // Type is a struct embedding a NodePrototype/Type for every Node implementation in this package. // One of its major uses is to start the construction of a value. // You can use it like this: // // `+pkgName+`.Type.YourTypeName.NewBuilder().BeginMap() //... // // and: // // `+pkgName+`.Type.OtherTypeName.NewBuilder().AssignString("x") // ... // var Type typeSlab type typeSlab struct { {{- range . }} {{ .Name }} _{{ . | TypeSymbol }}__Prototype {{ .Name }}__Repr _{{ . | TypeSymbol }}__ReprPrototype {{- end}} } `, w, adjCfg, ts.GetTypes()) } ================================================ FILE: schema/gen/go/genpartsCommon.go ================================================ package gengo import ( "io" ) /* This file is full of "typical" templates. They may not be used by *every* type and representation, but if they're extracted here, they're at least used by *many*. */ // The codegen do-not-edit warning comment. // Follows the pattern in https://golang.org/s/generatedcode / https://github.com/golang/go/issues/13560#issuecomment-288457920 . // Should appear somewhere near the top of every file (though precise order doesn't matter). const doNotEditComment = `// Code generated by go-ipld-prime gengo. DO NOT EDIT.` // emitNativeMaybe turns out to be completely agnostic to pretty much everything; // it doesn't vary by kind at all, and has never yet ended up needing specialization. func emitNativeMaybe(w io.Writer, adjCfg *AdjunctCfg, data interface{}) { doTemplate(` type _{{ .Type | TypeSymbol }}__Maybe struct { m schema.Maybe v {{if not (MaybeUsesPtr .Type) }}_{{end}}{{ .Type | TypeSymbol }} } type Maybe{{ .Type | TypeSymbol }} = *_{{ .Type | TypeSymbol }}__Maybe func (m Maybe{{ .Type | TypeSymbol }}) IsNull() bool { return m.m == schema.Maybe_Null } func (m Maybe{{ .Type | TypeSymbol }}) IsAbsent() bool { return m.m == schema.Maybe_Absent } func (m Maybe{{ .Type | TypeSymbol }}) Exists() bool { return m.m == schema.Maybe_Value } func (m Maybe{{ .Type | TypeSymbol }}) AsNode() datamodel.Node { switch m.m { case schema.Maybe_Absent: return datamodel.Absent case schema.Maybe_Null: return datamodel.Null case schema.Maybe_Value: return {{if not (MaybeUsesPtr .Type) }}&{{end}}m.v default: panic("unreachable") } } func (m Maybe{{ .Type | TypeSymbol }}) Must() {{ .Type | TypeSymbol }} { if !m.Exists() { panic("unbox of a maybe rejected") } return {{if not (MaybeUsesPtr .Type) }}&{{end}}m.v } `, w, adjCfg, data) } func emitNativeType_scalar(w io.Writer, adjCfg *AdjunctCfg, data interface{}) { // Using a struct with a single member is the same size in memory as a typedef, // while also having the advantage of meaning we can block direct casting, // which is desirable because the compiler then ensures our validate methods can't be evaded. doTemplate(` {{- if Comments -}} // {{ .Type | TypeSymbol }} matches the IPLD Schema type "{{ .Type.Name }}". It has {{ .Kind }} kind. {{- end}} type {{ .Type | TypeSymbol }} = *_{{ .Type | TypeSymbol }} type _{{ .Type | TypeSymbol }} struct{ x {{ .Kind | KindPrim }} } `, w, adjCfg, data) } func emitNativeAccessors_scalar(w io.Writer, adjCfg *AdjunctCfg, data interface{}) { // The node interface's `AsFoo` method is almost sufficient... but // this method unboxes without needing to return an error that's statically impossible, // which makes it easier to use in chaining. doTemplate(` func (n {{ .Type | TypeSymbol }}) {{ .Kind.String | title }}() {{ .Kind | KindPrim }} { return n.x } `, w, adjCfg, data) } func emitNativeBuilder_scalar(w io.Writer, adjCfg *AdjunctCfg, data interface{}) { // Generate a single-step construction function -- this is easy to do for a scalar, // and all representations of scalar kind can be expected to have a method like this. // The function is attached to the NodePrototype for convenient namespacing; // it needs no new memory, so it would be inappropriate to attach to the builder or assembler. // FUTURE: should engage validation flow. doTemplate(` func (_{{ .Type | TypeSymbol }}__Prototype) From{{ .Kind.String | title }}(v {{ .Kind | KindPrim }}) ({{ .Type | TypeSymbol }}, error) { n := _{{ .Type | TypeSymbol }}{v} return &n, nil } `, w, adjCfg, data) } func emitNodeTypeAssertions_typical(w io.Writer, adjCfg *AdjunctCfg, data interface{}) { doTemplate(` var _ datamodel.Node = ({{ .Type | TypeSymbol }})(&_{{ .Type | TypeSymbol }}{}) var _ schema.TypedNode = ({{ .Type | TypeSymbol }})(&_{{ .Type | TypeSymbol }}{}) `, w, adjCfg, data) } func emitNodeMethodAsKind_scalar(w io.Writer, adjCfg *AdjunctCfg, data interface{}) { doTemplate(` func (n {{ .Type | TypeSymbol }}) As{{ .Kind.String | title }}() ({{ .Kind | KindPrim }}, error) { return n.x, nil } `, w, adjCfg, data) } func emitNodeMethodPrototype_typical(w io.Writer, adjCfg *AdjunctCfg, data interface{}) { doTemplate(` func ({{ if .IsRepr }}_{{end}}{{ .Type | TypeSymbol }}{{ if .IsRepr }}__Repr{{end}}) Prototype() datamodel.NodePrototype { return _{{ .Type | TypeSymbol }}__{{ if .IsRepr }}Repr{{end}}Prototype{} } `, w, adjCfg, data) } // nodePrototype doesn't really vary textually at all between types and kinds // because it's just builders and standard resets. func emitNodePrototypeType_typical(w io.Writer, adjCfg *AdjunctCfg, data interface{}) { doTemplate(` type _{{ .Type | TypeSymbol }}__{{ if .IsRepr }}Repr{{end}}Prototype struct{} func (_{{ .Type | TypeSymbol }}__{{ if .IsRepr }}Repr{{end}}Prototype) NewBuilder() datamodel.NodeBuilder { var nb _{{ .Type | TypeSymbol }}__{{ if .IsRepr }}Repr{{end}}Builder nb.Reset() return &nb } `, w, adjCfg, data) } // emitTypicalTypedNodeMethodRepresentation does... what it says on the tin. // // For most types, the way to get the representation node pointer doesn't // textually depend on either the node implementation details nor what the representation strategy is, // or really much at all for that matter. // It only depends on that they have the same structure, so this cast works. // // Most (all?) types can use this. However, it's here rather in the mixins, for two reasons: // one, it still seems possible to imagine we'll have a type someday for which this pattern won't hold; // and two, mixins are also used in the repr generators, and it wouldn't be all sane for this method to end up also on reprs. func emitTypicalTypedNodeMethodRepresentation(w io.Writer, adjCfg *AdjunctCfg, data interface{}) { doTemplate(` func (n {{ .Type | TypeSymbol }}) Representation() datamodel.Node { return (*_{{ .Type | TypeSymbol }}__Repr)(n) } `, w, adjCfg, data) } // Turns out basically all builders are just an embed of the corresponding assembler. func emitEmitNodeBuilderType_typical(w io.Writer, adjCfg *AdjunctCfg, data interface{}) { doTemplate(` type _{{ .Type | TypeSymbol }}__{{ if .IsRepr }}Repr{{end}}Builder struct { _{{ .Type | TypeSymbol }}__{{ if .IsRepr }}Repr{{end}}Assembler } `, w, adjCfg, data) } // Builder build and reset methods are common even when some parts of the assembler vary. // We count on the zero value of any addntl non-common fields of the assembler being correct. func emitNodeBuilderMethods_typical(w io.Writer, adjCfg *AdjunctCfg, data interface{}) { doTemplate(` func (nb *_{{ .Type | TypeSymbol }}__{{ if .IsRepr }}Repr{{end}}Builder) Build() datamodel.Node { if *nb.m != schema.Maybe_Value { panic("invalid state: cannot call Build on an assembler that's not finished") } return nb.w } func (nb *_{{ .Type | TypeSymbol }}__{{ if .IsRepr }}Repr{{end}}Builder) Reset() { var w _{{ .Type | TypeSymbol }} var m schema.Maybe *nb = _{{ .Type | TypeSymbol }}__{{ if .IsRepr }}Repr{{end}}Builder{_{{ .Type | TypeSymbol }}__{{ if .IsRepr }}Repr{{end}}Assembler{w: &w, m: &m}} } `, w, adjCfg, data) } // emitNodeAssemblerType_scalar emits a NodeAssembler that's typical for a scalar. // Types that are recursive tend to have more state and custom stuff, so won't use this // (although the 'm' and 'w' variable names may still be presumed universally). func emitNodeAssemblerType_scalar(w io.Writer, adjCfg *AdjunctCfg, data interface{}) { doTemplate(` type _{{ .Type | TypeSymbol }}__Assembler struct { w *_{{ .Type | TypeSymbol }} m *schema.Maybe } func (na *_{{ .Type | TypeSymbol }}__Assembler) reset() {} `, w, adjCfg, data) } func emitNodeAssemblerMethodAssignNull_scalar(w io.Writer, adjCfg *AdjunctCfg, data interface{}) { doTemplate(` func (na *_{{ .Type | TypeSymbol }}__{{ if .IsRepr }}Repr{{end}}Assembler) AssignNull() error { switch *na.m { case allowNull: *na.m = schema.Maybe_Null return nil case schema.Maybe_Absent: return mixins.{{ .Kind.String | title }}Assembler{TypeName: "{{ .PkgName }}.{{ .TypeName }}{{ if .IsRepr }}.Repr{{end}}"}.AssignNull() case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") } panic("unreachable") } `, w, adjCfg, data) } // almost the same as the variant for scalars, but also has to check for midvalue state. func emitNodeAssemblerMethodAssignNull_recursive(w io.Writer, adjCfg *AdjunctCfg, data interface{}) { doTemplate(` func (na *_{{ .Type | TypeSymbol }}__{{ if .IsRepr }}Repr{{end}}Assembler) AssignNull() error { switch *na.m { case allowNull: *na.m = schema.Maybe_Null return nil case schema.Maybe_Absent: return mixins.{{ .Kind.String | title }}Assembler{TypeName: "{{ .PkgName }}.{{ .TypeName }}{{ if .IsRepr }}.Repr{{end}}"}.AssignNull() case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") case midvalue: panic("invalid state: cannot assign null into an assembler that's already begun working on recursive structures!") } panic("unreachable") } `, w, adjCfg, data) } // works for the AssignFoo methods for scalar kinds that are just boxing a thing. // There's no equivalent of this at all for recursives -- they're too diverse. func emitNodeAssemblerMethodAssignKind_scalar(w io.Writer, adjCfg *AdjunctCfg, data interface{}) { // This method contains a branch to support MaybeUsesPtr because new memory may need to be allocated. // This allocation only happens if the 'w' ptr is nil, which means we're being used on a Maybe; // otherwise, the 'w' ptr should already be set, and we fill that memory location without allocating, as usual. doTemplate(` func (na *_{{ .Type | TypeSymbol }}__Assembler) Assign{{ .Kind.String | title }}(v {{ .Kind | KindPrim }}) error { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") } {{- if .Type | MaybeUsesPtr }} if na.w == nil { na.w = &_{{ .Type | TypeSymbol }}{} } {{- end}} na.w.x = v *na.m = schema.Maybe_Value return nil } `, w, adjCfg, data) } // leans heavily on the fact all the AsFoo and AssignFoo methods follow a very consistent textual pattern. // FUTURE: may be able to get this to work for recursives, too -- but maps and lists each have very unique bottom thirds of this function. func emitNodeAssemblerMethodAssignNode_scalar(w io.Writer, adjCfg *AdjunctCfg, data interface{}) { // AssignNode goes through three phases: // 1. is it null? Jump over to AssignNull (which may or may not reject it). // 2. is it our own type? Handle specially -- we might be able to do efficient things. // 3. is it the right kind to morph into us? Do so. doTemplate(` func (na *_{{ .Type | TypeSymbol }}__Assembler) AssignNode(v datamodel.Node) error { if v.IsNull() { return na.AssignNull() } if v2, ok := v.(*_{{ .Type | TypeSymbol }}); ok { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") } {{- if .Type | MaybeUsesPtr }} if na.w == nil { na.w = v2 *na.m = schema.Maybe_Value return nil } {{- end}} *na.w = *v2 *na.m = schema.Maybe_Value return nil } if v2, err := v.As{{ .Kind.String | title }}(); err != nil { return err } else { return na.Assign{{ .Kind.String | title }}(v2) } } `, w, adjCfg, data) } ================================================ FILE: schema/gen/go/genpartsList.go ================================================ package gengo import ( "io" ) // FIXME docs: these methods all say "-oid" but I think that was overoptimistic and not actually that applicable, really. // AssignNode? Okay, that one's fine. // The rest? They're all *very* emphatic about knowing either: // - that na.w.x is a slice; or, // - that there's only one 'va' (one type; and that it's reused). // The reuse level for those two traits is pretty minimal. func emitNodeAssemblerMethodBeginList_listoid(w io.Writer, adjCfg *AdjunctCfg, data interface{}) { // This method contains a branch to support MaybeUsesPtr because new memory may need to be allocated. // This allocation only happens if the 'w' ptr is nil, which means we're being used on a Maybe; // otherwise, the 'w' ptr should already be set, and we fill that memory location without allocating, as usual. // // There's surprisingly little variation for IsRepr on this one: // - the child types we *store* are the same either way, so that doesn't vary; // - the only thing that we return that's different is... ourself. // // DRY: even further, to an extracted function in the final output? Maybe. // This could be plausible, iff... the top half of the struct (na.m, na.w) was independently addressable. (na.va has a varying concrete type and blocks extractions.) // Would also want to examine if that makes desirable trades in gsloc/asmsize/speed/debuggability. // Only seems to apply to case of list-repr-list, so unclear if worth the effort. doTemplate(` func (na *_{{ .Type | TypeSymbol }}__{{ if .IsRepr }}Repr{{end}}Assembler) BeginList(sizeHint int64) (datamodel.ListAssembler, error) { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") case midvalue: panic("invalid state: it makes no sense to 'begin' twice on the same assembler!") } *na.m = midvalue if sizeHint < 0 { sizeHint = 0 } {{- if .Type | MaybeUsesPtr }} if na.w == nil { na.w = &_{{ .Type | TypeSymbol }}{} } {{- end}} if sizeHint > 0 { na.w.x = make([]_{{ .Type.ValueType | TypeSymbol }}{{if .Type.ValueIsNullable }}__Maybe{{end}}, 0, sizeHint) } return na, nil } `, w, adjCfg, data) } func emitNodeAssemblerMethodAssignNode_listoid(w io.Writer, adjCfg *AdjunctCfg, data interface{}) { // AssignNode goes through three phases: // 1. is it null? Jump over to AssignNull (which may or may not reject it). // 2. is it our own type? Handle specially -- we might be able to do efficient things. // 3. is it the right kind to morph into us? Do so. // // We do not set m=midvalue in phase 3 -- it shouldn't matter unless you're trying to pull off concurrent access, which is wrong and unsafe regardless. // // This works easily for both type-level and representational nodes because // any divergences that have to do with the child value are nicely hidden behind `AssembleValue`. doTemplate(` func (na *_{{ .Type | TypeSymbol }}__{{ if .IsRepr }}Repr{{end}}Assembler) AssignNode(v datamodel.Node) error { if v.IsNull() { return na.AssignNull() } if v2, ok := v.(*_{{ .Type | TypeSymbol }}); ok { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") case midvalue: panic("invalid state: cannot assign null into an assembler that's already begun working on recursive structures!") } {{- if .Type | MaybeUsesPtr }} if na.w == nil { na.w = v2 *na.m = schema.Maybe_Value return nil } {{- end}} *na.w = *v2 *na.m = schema.Maybe_Value return nil } if v.Kind() != datamodel.Kind_List { return datamodel.ErrWrongKind{TypeName: "{{ .PkgName }}.{{ .Type.Name }}{{ if .IsRepr }}.Repr{{end}}", MethodName: "AssignNode", AppropriateKind: datamodel.KindSet_JustList, ActualKind: v.Kind()} } itr := v.ListIterator() for !itr.Done() { _, v, err := itr.Next() if err != nil { return err } if err := na.AssembleValue().AssignNode(v); err != nil { return err } } return na.Finish() } `, w, adjCfg, data) } func emitNodeAssemblerHelper_listoid_tidyHelper(w io.Writer, adjCfg *AdjunctCfg, data interface{}) { // This function attempts to clean up the state machine to acknolwedge child value assembly finish. // If the child was finished and we just collected it, return true and update state to laState_initial. // Otherwise, if it wasn't done, return false; // and the caller is almost certain to emit an error momentarily. // The function will only be called when the current state is laState_midValue. // (In general, the idea is that if the user is doing things correctly, // this function will only be called when the child is in fact finished.) // If 'cm' is used, we reset it to its initial condition of Maybe_Absent here. // At the same time, we nil the 'w' pointer for the child assembler; otherwise its own state machine would probably let it modify 'w' again! // // DRY(nope): Can this be extracted to be a shared function between repr and type level nodes? // It is textually identical, so... yeah, that'd be nice. But... // Nope. It touches `la.va` directly. // Attempting to extract that or hide it behind an interface would create virtual function calls in a very tight spot, and we don't want the execution time cost. doTemplate(` func (la *_{{ .Type | TypeSymbol }}__{{ if .IsRepr }}Repr{{end}}Assembler) valueFinishTidy() bool { {{- if .Type.ValueIsNullable }} row := &la.w.x[len(la.w.x)-1] switch row.m { case schema.Maybe_Value: {{- if (MaybeUsesPtr .Type.ValueType) }} row.v = la.va.w {{- end}} la.va.w = nil fallthrough case schema.Maybe_Null: la.state = laState_initial la.va.reset() return true {{- else}} switch la.cm { case schema.Maybe_Value: la.va.w = nil la.cm = schema.Maybe_Absent la.state = laState_initial la.va.reset() return true {{- end}} default: return false } } `, w, adjCfg, data) } func emitNodeAssemblerHelper_listoid_listAssemblerMethods(w io.Writer, adjCfg *AdjunctCfg, data interface{}) { // DRY: Might want to split this up a bit further so it can be used by more kinds. // Some parts of this could be reused by struct-repr-tuple, potentially, but would require being able to insert some more checks relating to length. // This would also require excluding *all* 'va' references; those are radicaly different for structs, in that there's not even one (singular) of them. // // DRY(nope): Can this be extracted to a shared function in the output? // Same story as the tidy helper -- it touches `la.va` concretely in several places, and that blocks extraction. doTemplate(` func (la *_{{ .Type | TypeSymbol }}__{{ if .IsRepr }}Repr{{end}}Assembler) AssembleValue() datamodel.NodeAssembler { switch la.state { case laState_initial: // carry on case laState_midValue: if !la.valueFinishTidy() { panic("invalid state: AssembleValue cannot be called when still in the middle of assembling the previous value") } // if tidy success: carry on case laState_finished: panic("invalid state: AssembleValue cannot be called on an assembler that's already finished") } la.w.x = append(la.w.x, _{{ .Type.ValueType | TypeSymbol }}{{if .Type.ValueIsNullable }}__Maybe{{end}}{}) la.state = laState_midValue row := &la.w.x[len(la.w.x)-1] {{- if .Type.ValueIsNullable }} {{- if not (MaybeUsesPtr .Type.ValueType) }} la.va.w = &row.v {{- end}} la.va.m = &row.m row.m = allowNull {{- else}} la.va.w = row la.va.m = &la.cm {{- end}} return &la.va } `, w, adjCfg, data) doTemplate(` func (la *_{{ .Type | TypeSymbol }}__{{ if .IsRepr }}Repr{{end}}Assembler) Finish() error { switch la.state { case laState_initial: // carry on case laState_midValue: if !la.valueFinishTidy() { panic("invalid state: Finish cannot be called when in the middle of assembling a value") } // if tidy success: carry on case laState_finished: panic("invalid state: Finish cannot be called on an assembler that's already finished") } la.state = laState_finished *la.m = schema.Maybe_Value return nil } `, w, adjCfg, data) doTemplate(` func (la *_{{ .Type | TypeSymbol }}__{{ if .IsRepr }}Repr{{end}}Assembler) ValuePrototype(_ int64) datamodel.NodePrototype { return _{{ .Type.ValueType | TypeSymbol }}__{{ if .IsRepr }}Repr{{end}}Prototype{} } `, w, adjCfg, data) } ================================================ FILE: schema/gen/go/genpartsMap.go ================================================ package gengo import ( "io" ) // FIXME docs: these methods all say "-oid" but I think that was overoptimistic and not actually that applicable, really. // AssignNode? Okay, that one's fine. // The rest? They're all *very* emphatic about knowing either: // - that na.w.t and na.w.m are fields; or, // - that there's only one 'ka' and 'va' (one type each; and that it's reused). // The reuse level for those two traits is pretty minimal. func emitNodeAssemblerMethodBeginMap_mapoid(w io.Writer, adjCfg *AdjunctCfg, data interface{}) { // This method contains a branch to support MaybeUsesPtr because new memory may need to be allocated. // This allocation only happens if the 'w' ptr is nil, which means we're being used on a Maybe; // otherwise, the 'w' ptr should already be set, and we fill that memory location without allocating, as usual. doTemplate(` func (na *_{{ .Type | TypeSymbol }}__{{ if .IsRepr }}Repr{{end}}Assembler) BeginMap(sizeHint int64) (datamodel.MapAssembler, error) { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") case midvalue: panic("invalid state: it makes no sense to 'begin' twice on the same assembler!") } *na.m = midvalue if sizeHint < 0 { sizeHint = 0 } {{- if .Type | MaybeUsesPtr }} if na.w == nil { na.w = &_{{ .Type | TypeSymbol }}{} } {{- end}} na.w.m = make(map[_{{ .Type.KeyType | TypeSymbol }}]{{if .Type.ValueIsNullable }}Maybe{{else}}*_{{end}}{{ .Type.ValueType | TypeSymbol }}, sizeHint) na.w.t = make([]_{{ .Type | TypeSymbol }}__entry, 0, sizeHint) return na, nil } `, w, adjCfg, data) } func emitNodeAssemblerMethodAssignNode_mapoid(w io.Writer, adjCfg *AdjunctCfg, data interface{}) { // AssignNode goes through three phases: // 1. is it null? Jump over to AssignNull (which may or may not reject it). // 2. is it our own type? Handle specially -- we might be able to do efficient things. // 3. is it the right kind to morph into us? Do so. // // We do not set m=midvalue in phase 3 -- it shouldn't matter unless you're trying to pull off concurrent access, which is wrong and unsafe regardless. // // This works easily for both type-level and representational nodes because // any divergences that have to do with the child value are nicely hidden behind `AssembleKey` and `AssembleValue`. doTemplate(` func (na *_{{ .Type | TypeSymbol }}__{{ if .IsRepr }}Repr{{end}}Assembler) AssignNode(v datamodel.Node) error { if v.IsNull() { return na.AssignNull() } if v2, ok := v.(*_{{ .Type | TypeSymbol }}); ok { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") case midvalue: panic("invalid state: cannot assign null into an assembler that's already begun working on recursive structures!") } {{- if .Type | MaybeUsesPtr }} if na.w == nil { na.w = v2 *na.m = schema.Maybe_Value return nil } {{- end}} *na.w = *v2 *na.m = schema.Maybe_Value return nil } if v.Kind() != datamodel.Kind_Map { return datamodel.ErrWrongKind{TypeName: "{{ .PkgName }}.{{ .Type.Name }}{{ if .IsRepr }}.Repr{{end}}", MethodName: "AssignNode", AppropriateKind: datamodel.KindSet_JustMap, ActualKind: v.Kind()} } itr := v.MapIterator() for !itr.Done() { k, v, err := itr.Next() if err != nil { return err } if err := na.AssembleKey().AssignNode(k); err != nil { return err } if err := na.AssembleValue().AssignNode(v); err != nil { return err } } return na.Finish() } `, w, adjCfg, data) } func emitNodeAssemblerHelper_mapoid_keyTidyHelper(w io.Writer, adjCfg *AdjunctCfg, data interface{}) { // This function attempts to clean up the state machine to acknolwedge key assembly finish. // If the child was finished and we just collected it, return true and update state to maState_expectValue. // Collecting the child includes updating the 'ma.w.m' to point into the relevant row of 'ma.w.t', since that couldn't be done earlier, // AND initializing the 'ma.va' (since we're already holding relevant offsets into 'ma.w.t'). // Otherwise, if it wasn't done, return false; // and the caller is almost certain to emit an error momentarily. // The function will only be called when the current state is maState_midKey. // (In general, the idea is that if the user is doing things correctly, // this function will only be called when the child is in fact finished.) // Completion info always comes via 'cm', and we reset it to its initial condition of Maybe_Absent here. // At the same time, we nil the 'w' pointer for the child assembler; otherwise its own state machine would probably let it modify 'w' again! // // DRY(nope): Can this be extracted to be a shared function between repr and type level nodes? // It is textually identical, so... yeah, that'd be nice. But... // Nope. It touches `ma.ka` and `ma.va` directly. // Attempting to extract or hide those behind an interface would create virtual function calls in a very tight spot, and we don't want the execution time cost. doTemplate(` func (ma *_{{ .Type | TypeSymbol }}__{{ if .IsRepr }}Repr{{end}}Assembler) keyFinishTidy() bool { switch ma.cm { case schema.Maybe_Value: ma.ka.w = nil tz := &ma.w.t[len(ma.w.t)-1] ma.cm = schema.Maybe_Absent ma.state = maState_expectValue ma.w.m[tz.k] = &tz.v {{- if .Type.ValueIsNullable }} {{- if not (MaybeUsesPtr .Type.ValueType) }} ma.va.w = &tz.v.v {{- end}} ma.va.m = &tz.v.m tz.v.m = allowNull {{- else}} ma.va.w = &tz.v ma.va.m = &ma.cm {{- end}} ma.ka.reset() return true default: return false } } `, w, adjCfg, data) } func emitNodeAssemblerHelper_mapoid_valueTidyHelper(w io.Writer, adjCfg *AdjunctCfg, data interface{}) { // This function attempts to clean up the state machine to acknolwedge child value assembly finish. // If the child was finished and we just collected it, return true and update state to maState_initial. // Otherwise, if it wasn't done, return false; // and the caller is almost certain to emit an error momentarily. // The function will only be called when the current state is maState_midValue. // (In general, the idea is that if the user is doing things correctly, // this function will only be called when the child is in fact finished.) // If 'cm' is used, we reset it to its initial condition of Maybe_Absent here. // At the same time, we nil the 'w' pointer for the child assembler; otherwise its own state machine would probably let it modify 'w' again! // // DRY(nope): Can this be extracted to be a shared function between repr and type level nodes? // Exact same story as the key tidy helper -- touches child assemblers concretely, and that blocks extraction. doTemplate(` func (ma *_{{ .Type | TypeSymbol }}__{{ if .IsRepr }}Repr{{end}}Assembler) valueFinishTidy() bool { {{- if .Type.ValueIsNullable }} tz := &ma.w.t[len(ma.w.t)-1] switch tz.v.m { case schema.Maybe_Null: ma.state = maState_initial ma.va.reset() return true case schema.Maybe_Value: {{- if (MaybeUsesPtr .Type.ValueType) }} tz.v.v = ma.va.w {{- end}} ma.va.w = nil ma.state = maState_initial ma.va.reset() return true {{- else}} switch ma.cm { case schema.Maybe_Value: ma.va.w = nil ma.cm = schema.Maybe_Absent ma.state = maState_initial ma.va.reset() return true {{- end}} default: return false } } `, w, adjCfg, data) } func emitNodeAssemblerHelper_mapoid_mapAssemblerMethods(w io.Writer, adjCfg *AdjunctCfg, data interface{}) { // FUTURE: some of the setup of the child assemblers could probably be DRY'd up. // // REVIEW: there's a copy-by-value of k2 that's avoidable. But it simplifies the error path. Worth working on? // // REVIEW: processing the key via the reprPrototype of the key even when we're at the type level if it's type kind isn't string is currently supported, but should it be? or is that more confusing than valuable? // Very possible that it shouldn't be supported: the full-on keyAssembler route won't accept this, so consistency with that might be best. // On the other hand, lookups by string *do* support this kind of processing (and it must, or PathSegment utility becomes unacceptably damaged), so either way, something feels surprising. // // DRY(nope): Can this be extracted to a shared function in the output? // Same story as the tidy helpers -- it touches `va` and `ka` concretely in several places, and that blocks extraction. // // DRY: a lot of the state transition fences again are common for all mapoids, and could probably even be a function over '*state'... // except for the fact they need to call the valueFinishTidy function, which is another one of those points that blocks extraction because we strongly don't want virtual functions calls there. // Maybe the templates can be textually dedup'd more, though, at least. doTemplate(` func (ma *_{{ .Type | TypeSymbol }}__{{ if .IsRepr }}Repr{{end}}Assembler) AssembleEntry(k string) (datamodel.NodeAssembler, error) { switch ma.state { case maState_initial: // carry on case maState_midKey: panic("invalid state: AssembleEntry cannot be called when in the middle of assembling another key") case maState_expectValue: panic("invalid state: AssembleEntry cannot be called when expecting start of value assembly") case maState_midValue: if !ma.valueFinishTidy() { panic("invalid state: AssembleEntry cannot be called when in the middle of assembling a value") } // if tidy success: carry on case maState_finished: panic("invalid state: AssembleEntry cannot be called on an assembler that's already finished") } var k2 _{{ .Type.KeyType | TypeSymbol }} {{- if or (not (eq .Type.KeyType.TypeKind.String "String")) .IsRepr }} if err := (_{{ .Type.KeyType | TypeSymbol }}__ReprPrototype{}).fromString(&k2, k); err != nil { return nil, err // TODO wrap in some kind of ErrInvalidKey } {{- else}} if err := (_{{ .Type.KeyType | TypeSymbol }}__Prototype{}).fromString(&k2, k); err != nil { return nil, err // TODO wrap in some kind of ErrInvalidKey } {{- end}} if _, exists := ma.w.m[k2]; exists { return nil, datamodel.ErrRepeatedMapKey{Key: &k2} } ma.w.t = append(ma.w.t, _{{ .Type | TypeSymbol }}__entry{k: k2}) tz := &ma.w.t[len(ma.w.t)-1] ma.state = maState_midValue ma.w.m[k2] = &tz.v {{- if .Type.ValueIsNullable }} {{- if not (MaybeUsesPtr .Type.ValueType) }} ma.va.w = &tz.v.v {{- end}} ma.va.m = &tz.v.m tz.v.m = allowNull {{- else}} ma.va.w = &tz.v ma.va.m = &ma.cm {{- end}} return &ma.va, nil } `, w, adjCfg, data) doTemplate(` func (ma *_{{ .Type | TypeSymbol }}__{{ if .IsRepr }}Repr{{end}}Assembler) AssembleKey() datamodel.NodeAssembler { switch ma.state { case maState_initial: // carry on case maState_midKey: panic("invalid state: AssembleKey cannot be called when in the middle of assembling another key") case maState_expectValue: panic("invalid state: AssembleKey cannot be called when expecting start of value assembly") case maState_midValue: if !ma.valueFinishTidy() { panic("invalid state: AssembleKey cannot be called when in the middle of assembling a value") } // if tidy success: carry on case maState_finished: panic("invalid state: AssembleKey cannot be called on an assembler that's already finished") } ma.w.t = append(ma.w.t, _{{ .Type | TypeSymbol }}__entry{}) ma.state = maState_midKey ma.ka.m = &ma.cm ma.ka.w = &ma.w.t[len(ma.w.t)-1].k return &ma.ka } `, w, adjCfg, data) doTemplate(` func (ma *_{{ .Type | TypeSymbol }}__{{ if .IsRepr }}Repr{{end}}Assembler) AssembleValue() datamodel.NodeAssembler { switch ma.state { case maState_initial: panic("invalid state: AssembleValue cannot be called when no key is primed") case maState_midKey: if !ma.keyFinishTidy() { panic("invalid state: AssembleValue cannot be called when in the middle of assembling a key") } // if tidy success: carry on case maState_expectValue: // carry on case maState_midValue: panic("invalid state: AssembleValue cannot be called when in the middle of assembling another value") case maState_finished: panic("invalid state: AssembleValue cannot be called on an assembler that's already finished") } ma.state = maState_midValue return &ma.va } `, w, adjCfg, data) doTemplate(` func (ma *_{{ .Type | TypeSymbol }}__{{ if .IsRepr }}Repr{{end}}Assembler) Finish() error { switch ma.state { case maState_initial: // carry on case maState_midKey: panic("invalid state: Finish cannot be called when in the middle of assembling a key") case maState_expectValue: panic("invalid state: Finish cannot be called when expecting start of value assembly") case maState_midValue: if !ma.valueFinishTidy() { panic("invalid state: Finish cannot be called when in the middle of assembling a value") } // if tidy success: carry on case maState_finished: panic("invalid state: Finish cannot be called on an assembler that's already finished") } ma.state = maState_finished *ma.m = schema.Maybe_Value return nil } `, w, adjCfg, data) doTemplate(` func (ma *_{{ .Type | TypeSymbol }}__{{ if .IsRepr }}Repr{{end}}Assembler) KeyPrototype() datamodel.NodePrototype { return _{{ .Type.KeyType | TypeSymbol }}__{{ if .IsRepr }}Repr{{end}}Prototype{} } func (ma *_{{ .Type | TypeSymbol }}__{{ if .IsRepr }}Repr{{end}}Assembler) ValuePrototype(_ string) datamodel.NodePrototype { return _{{ .Type.ValueType | TypeSymbol }}__{{ if .IsRepr }}Repr{{end}}Prototype{} } `, w, adjCfg, data) } ================================================ FILE: schema/gen/go/genpartsMinima.go ================================================ package gengo import ( "fmt" "io" "github.com/ipld/go-ipld-prime/testutil" ) // EmitInternalEnums creates a file with enum types used internally. // For example, the state machine values used in map and list builders. // These always need to exist exactly once in each package created by codegen. // // The file header and import statements are included in the output of this function. // (The imports in this file are different than most others in codegen output; // we gather up any references to other packages in this file in order to simplify the rest of codegen's awareness of imports.) func EmitInternalEnums(packageName string, w io.Writer) { fmt.Fprint(w, testutil.Dedent(` package `+packageName+` `+doNotEditComment+` import ( "fmt" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/schema" ) `)) // The 'Maybe' enum does double-duty in this package as a state machine for assembler completion. // // The 'Maybe_Absent' value gains the additional semantic of "clear to assign (but not null)" // (which works because if you're *in* a value assembler, "absent" as a final result is already off the table). // Additionally, we get a few extra states that we cram into the same area of bits: // - `midvalue` is used by assemblers of recursives to block AssignNull after BeginX. // - `allowNull` is used by parent assemblers when initializing a child assembler to tell the child a transition to Maybe_Null is allowed in this context. fmt.Fprint(w, testutil.Dedent(` const ( midvalue = schema.Maybe(4) allowNull = schema.Maybe(5) ) `)) fmt.Fprint(w, testutil.Dedent(` type maState uint8 const ( maState_initial maState = iota maState_midKey maState_expectValue maState_midValue maState_finished ) type laState uint8 const ( laState_initial laState = iota laState_midValue laState_finished ) `)) // We occasionally need this erroring thunk to be able to snake an error out from some assembly processes. // It implements all of datamodel.NodeAssembler, but all of its methods return errors when used. fmt.Fprint(w, testutil.Dedent(` type _ErrorThunkAssembler struct { e error } func (ea _ErrorThunkAssembler) BeginMap(_ int64) (datamodel.MapAssembler, error) { return nil, ea.e } func (ea _ErrorThunkAssembler) BeginList(_ int64) (datamodel.ListAssembler, error) { return nil, ea.e } func (ea _ErrorThunkAssembler) AssignNull() error { return ea.e } func (ea _ErrorThunkAssembler) AssignBool(bool) error { return ea.e } func (ea _ErrorThunkAssembler) AssignInt(int64) error { return ea.e } func (ea _ErrorThunkAssembler) AssignFloat(float64) error { return ea.e } func (ea _ErrorThunkAssembler) AssignString(string) error { return ea.e } func (ea _ErrorThunkAssembler) AssignBytes([]byte) error { return ea.e } func (ea _ErrorThunkAssembler) AssignLink(datamodel.Link) error { return ea.e } func (ea _ErrorThunkAssembler) AssignNode(datamodel.Node) error { return ea.e } func (ea _ErrorThunkAssembler) Prototype() datamodel.NodePrototype { panic(fmt.Errorf("cannot get prototype from error-carrying assembler: already derailed with error: %w", ea.e)) } `)) } ================================================ FILE: schema/gen/go/genpartsStrictoid.go ================================================ package gengo import ( "io" ) // What's "strictoid" mean? Anything that's recursive but not infinite -- so structs, unions. // This form of BeginMap method is reusable for anything that has fixed size and thus ignores sizehint. // That means: structs, unions, and their relevant representations. func emitNodeAssemblerMethodBeginMap_strictoid(w io.Writer, adjCfg *AdjunctCfg, data interface{}) { // We currently disregard sizeHint. It's not relevant to us. // We could check it strictly and emit errors; presently, we don't. // This method contains a branch to support MaybeUsesPtr because new memory may need to be allocated. // This allocation only happens if the 'w' ptr is nil, which means we're being used on a Maybe; // otherwise, the 'w' ptr should already be set, and we fill that memory location without allocating, as usual. // (Note that the Maybe we're talking about here is for us, not our children: so it applies on unions too.) doTemplate(` func (na *_{{ .Type | TypeSymbol }}__{{ if .IsRepr }}Repr{{end}}Assembler) BeginMap(int64) (datamodel.MapAssembler, error) { switch *na.m { case schema.Maybe_Value, schema.Maybe_Null: panic("invalid state: cannot assign into assembler that's already finished") case midvalue: panic("invalid state: it makes no sense to 'begin' twice on the same assembler!") } *na.m = midvalue {{- if .Type | MaybeUsesPtr }} if na.w == nil { na.w = &_{{ .Type | TypeSymbol }}{} } {{- end}} return na, nil } `, w, adjCfg, data) } ================================================ FILE: schema/gen/go/mixins/boolGenMixin.go ================================================ package mixins import ( "io" "github.com/ipld/go-ipld-prime/datamodel" ) type BoolTraits struct { PkgName string TypeName string // see doc in kindTraitsGenerator TypeSymbol string // see doc in kindTraitsGenerator } func (BoolTraits) Kind() datamodel.Kind { return datamodel.Kind_Bool } func (g BoolTraits) EmitNodeMethodKind(w io.Writer) { doTemplate(` func ({{ .TypeSymbol }}) Kind() datamodel.Kind { return datamodel.Kind_Bool } `, w, g) } func (g BoolTraits) EmitNodeMethodLookupByString(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Bool}.emitNodeMethodLookupByString(w) } func (g BoolTraits) EmitNodeMethodLookupByNode(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Bool}.emitNodeMethodLookupByNode(w) } func (g BoolTraits) EmitNodeMethodLookupByIndex(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Bool}.emitNodeMethodLookupByIndex(w) } func (g BoolTraits) EmitNodeMethodLookupBySegment(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Bool}.emitNodeMethodLookupBySegment(w) } func (g BoolTraits) EmitNodeMethodMapIterator(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Bool}.emitNodeMethodMapIterator(w) } func (g BoolTraits) EmitNodeMethodListIterator(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Bool}.emitNodeMethodListIterator(w) } func (g BoolTraits) EmitNodeMethodLength(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Bool}.emitNodeMethodLength(w) } func (g BoolTraits) EmitNodeMethodIsAbsent(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Bool}.emitNodeMethodIsAbsent(w) } func (g BoolTraits) EmitNodeMethodIsNull(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Bool}.emitNodeMethodIsNull(w) } func (g BoolTraits) EmitNodeMethodAsInt(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Bool}.emitNodeMethodAsInt(w) } func (g BoolTraits) EmitNodeMethodAsFloat(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Bool}.emitNodeMethodAsFloat(w) } func (g BoolTraits) EmitNodeMethodAsString(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Bool}.emitNodeMethodAsString(w) } func (g BoolTraits) EmitNodeMethodAsBytes(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Bool}.emitNodeMethodAsBytes(w) } func (g BoolTraits) EmitNodeMethodAsLink(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Bool}.emitNodeMethodAsLink(w) } type BoolAssemblerTraits struct { PkgName string TypeName string // see doc in kindAssemblerTraitsGenerator AppliedPrefix string // see doc in kindAssemblerTraitsGenerator } func (BoolAssemblerTraits) Kind() datamodel.Kind { return datamodel.Kind_Bool } func (g BoolAssemblerTraits) EmitNodeAssemblerMethodBeginMap(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Bool}.emitNodeAssemblerMethodBeginMap(w) } func (g BoolAssemblerTraits) EmitNodeAssemblerMethodBeginList(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Bool}.emitNodeAssemblerMethodBeginList(w) } func (g BoolAssemblerTraits) EmitNodeAssemblerMethodAssignNull(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Bool}.emitNodeAssemblerMethodAssignNull(w) } func (g BoolAssemblerTraits) EmitNodeAssemblerMethodAssignInt(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Bool}.emitNodeAssemblerMethodAssignInt(w) } func (g BoolAssemblerTraits) EmitNodeAssemblerMethodAssignFloat(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Bool}.emitNodeAssemblerMethodAssignFloat(w) } func (g BoolAssemblerTraits) EmitNodeAssemblerMethodAssignString(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Bool}.emitNodeAssemblerMethodAssignString(w) } func (g BoolAssemblerTraits) EmitNodeAssemblerMethodAssignBytes(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Bool}.emitNodeAssemblerMethodAssignBytes(w) } func (g BoolAssemblerTraits) EmitNodeAssemblerMethodAssignLink(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Bool}.emitNodeAssemblerMethodAssignLink(w) } func (g BoolAssemblerTraits) EmitNodeAssemblerMethodPrototype(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Bool}.emitNodeAssemblerMethodPrototype(w) } ================================================ FILE: schema/gen/go/mixins/bytesGenMixin.go ================================================ package mixins import ( "io" "github.com/ipld/go-ipld-prime/datamodel" ) type BytesTraits struct { PkgName string TypeName string // see doc in kindTraitsGenerator TypeSymbol string // see doc in kindTraitsGenerator } func (BytesTraits) Kind() datamodel.Kind { return datamodel.Kind_Bytes } func (g BytesTraits) EmitNodeMethodKind(w io.Writer) { doTemplate(` func ({{ .TypeSymbol }}) Kind() datamodel.Kind { return datamodel.Kind_Bytes } `, w, g) } func (g BytesTraits) EmitNodeMethodLookupByString(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Bytes}.emitNodeMethodLookupByString(w) } func (g BytesTraits) EmitNodeMethodLookupByNode(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Bytes}.emitNodeMethodLookupByNode(w) } func (g BytesTraits) EmitNodeMethodLookupByIndex(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Bytes}.emitNodeMethodLookupByIndex(w) } func (g BytesTraits) EmitNodeMethodLookupBySegment(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Bytes}.emitNodeMethodLookupBySegment(w) } func (g BytesTraits) EmitNodeMethodMapIterator(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Bytes}.emitNodeMethodMapIterator(w) } func (g BytesTraits) EmitNodeMethodListIterator(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Bytes}.emitNodeMethodListIterator(w) } func (g BytesTraits) EmitNodeMethodLength(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Bytes}.emitNodeMethodLength(w) } func (g BytesTraits) EmitNodeMethodIsAbsent(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Bytes}.emitNodeMethodIsAbsent(w) } func (g BytesTraits) EmitNodeMethodIsNull(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Bytes}.emitNodeMethodIsNull(w) } func (g BytesTraits) EmitNodeMethodAsBool(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Bytes}.emitNodeMethodAsBool(w) } func (g BytesTraits) EmitNodeMethodAsInt(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Bytes}.emitNodeMethodAsInt(w) } func (g BytesTraits) EmitNodeMethodAsFloat(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Bytes}.emitNodeMethodAsFloat(w) } func (g BytesTraits) EmitNodeMethodAsString(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Bytes}.emitNodeMethodAsString(w) } func (g BytesTraits) EmitNodeMethodAsLink(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Bytes}.emitNodeMethodAsLink(w) } type BytesAssemblerTraits struct { PkgName string TypeName string // see doc in kindAssemblerTraitsGenerator AppliedPrefix string // see doc in kindAssemblerTraitsGenerator } func (BytesAssemblerTraits) Kind() datamodel.Kind { return datamodel.Kind_Bytes } func (g BytesAssemblerTraits) EmitNodeAssemblerMethodBeginMap(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Bytes}.emitNodeAssemblerMethodBeginMap(w) } func (g BytesAssemblerTraits) EmitNodeAssemblerMethodBeginList(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Bytes}.emitNodeAssemblerMethodBeginList(w) } func (g BytesAssemblerTraits) EmitNodeAssemblerMethodAssignNull(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Bytes}.emitNodeAssemblerMethodAssignNull(w) } func (g BytesAssemblerTraits) EmitNodeAssemblerMethodAssignBool(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Bytes}.emitNodeAssemblerMethodAssignBool(w) } func (g BytesAssemblerTraits) EmitNodeAssemblerMethodAssignInt(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Bytes}.emitNodeAssemblerMethodAssignInt(w) } func (g BytesAssemblerTraits) EmitNodeAssemblerMethodAssignFloat(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Bytes}.emitNodeAssemblerMethodAssignFloat(w) } func (g BytesAssemblerTraits) EmitNodeAssemblerMethodAssignString(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Bytes}.emitNodeAssemblerMethodAssignString(w) } func (g BytesAssemblerTraits) EmitNodeAssemblerMethodAssignLink(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Bytes}.emitNodeAssemblerMethodAssignLink(w) } func (g BytesAssemblerTraits) EmitNodeAssemblerMethodPrototype(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Bytes}.emitNodeAssemblerMethodPrototype(w) } ================================================ FILE: schema/gen/go/mixins/floatGenMixin.go ================================================ package mixins import ( "io" "github.com/ipld/go-ipld-prime/datamodel" ) type FloatTraits struct { PkgName string TypeName string // see doc in kindTraitsGenerator TypeSymbol string // see doc in kindTraitsGenerator } func (FloatTraits) Kind() datamodel.Kind { return datamodel.Kind_Float } func (g FloatTraits) EmitNodeMethodKind(w io.Writer) { doTemplate(` func ({{ .TypeSymbol }}) Kind() datamodel.Kind { return datamodel.Kind_Float } `, w, g) } func (g FloatTraits) EmitNodeMethodLookupByString(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Float}.emitNodeMethodLookupByString(w) } func (g FloatTraits) EmitNodeMethodLookupByNode(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Float}.emitNodeMethodLookupByNode(w) } func (g FloatTraits) EmitNodeMethodLookupByIndex(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Float}.emitNodeMethodLookupByIndex(w) } func (g FloatTraits) EmitNodeMethodLookupBySegment(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Float}.emitNodeMethodLookupBySegment(w) } func (g FloatTraits) EmitNodeMethodMapIterator(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Float}.emitNodeMethodMapIterator(w) } func (g FloatTraits) EmitNodeMethodListIterator(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Float}.emitNodeMethodListIterator(w) } func (g FloatTraits) EmitNodeMethodLength(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Float}.emitNodeMethodLength(w) } func (g FloatTraits) EmitNodeMethodIsAbsent(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Float}.emitNodeMethodIsAbsent(w) } func (g FloatTraits) EmitNodeMethodIsNull(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Float}.emitNodeMethodIsNull(w) } func (g FloatTraits) EmitNodeMethodAsBool(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Float}.emitNodeMethodAsBool(w) } func (g FloatTraits) EmitNodeMethodAsInt(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Float}.emitNodeMethodAsInt(w) } func (g FloatTraits) EmitNodeMethodAsString(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Float}.emitNodeMethodAsString(w) } func (g FloatTraits) EmitNodeMethodAsBytes(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Float}.emitNodeMethodAsBytes(w) } func (g FloatTraits) EmitNodeMethodAsLink(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Float}.emitNodeMethodAsLink(w) } type FloatAssemblerTraits struct { PkgName string TypeName string // see doc in kindAssemblerTraitsGenerator AppliedPrefix string // see doc in kindAssemblerTraitsGenerator } func (FloatAssemblerTraits) Kind() datamodel.Kind { return datamodel.Kind_Float } func (g FloatAssemblerTraits) EmitNodeAssemblerMethodBeginMap(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Float}.emitNodeAssemblerMethodBeginMap(w) } func (g FloatAssemblerTraits) EmitNodeAssemblerMethodBeginList(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Float}.emitNodeAssemblerMethodBeginList(w) } func (g FloatAssemblerTraits) EmitNodeAssemblerMethodAssignNull(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Float}.emitNodeAssemblerMethodAssignNull(w) } func (g FloatAssemblerTraits) EmitNodeAssemblerMethodAssignBool(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Float}.emitNodeAssemblerMethodAssignBool(w) } func (g FloatAssemblerTraits) EmitNodeAssemblerMethodAssignInt(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Float}.emitNodeAssemblerMethodAssignInt(w) } func (g FloatAssemblerTraits) EmitNodeAssemblerMethodAssignString(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Float}.emitNodeAssemblerMethodAssignString(w) } func (g FloatAssemblerTraits) EmitNodeAssemblerMethodAssignBytes(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Float}.emitNodeAssemblerMethodAssignBytes(w) } func (g FloatAssemblerTraits) EmitNodeAssemblerMethodAssignLink(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Float}.emitNodeAssemblerMethodAssignLink(w) } func (g FloatAssemblerTraits) EmitNodeAssemblerMethodPrototype(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Float}.emitNodeAssemblerMethodPrototype(w) } ================================================ FILE: schema/gen/go/mixins/intGenMixin.go ================================================ package mixins import ( "io" "github.com/ipld/go-ipld-prime/datamodel" ) type IntTraits struct { PkgName string TypeName string // see doc in kindTraitsGenerator TypeSymbol string // see doc in kindTraitsGenerator } func (IntTraits) Kind() datamodel.Kind { return datamodel.Kind_Int } func (g IntTraits) EmitNodeMethodKind(w io.Writer) { doTemplate(` func ({{ .TypeSymbol }}) Kind() datamodel.Kind { return datamodel.Kind_Int } `, w, g) } func (g IntTraits) EmitNodeMethodLookupByString(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Int}.emitNodeMethodLookupByString(w) } func (g IntTraits) EmitNodeMethodLookupByNode(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Int}.emitNodeMethodLookupByNode(w) } func (g IntTraits) EmitNodeMethodLookupByIndex(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Int}.emitNodeMethodLookupByIndex(w) } func (g IntTraits) EmitNodeMethodLookupBySegment(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Int}.emitNodeMethodLookupBySegment(w) } func (g IntTraits) EmitNodeMethodMapIterator(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Int}.emitNodeMethodMapIterator(w) } func (g IntTraits) EmitNodeMethodListIterator(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Int}.emitNodeMethodListIterator(w) } func (g IntTraits) EmitNodeMethodLength(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Int}.emitNodeMethodLength(w) } func (g IntTraits) EmitNodeMethodIsAbsent(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Int}.emitNodeMethodIsAbsent(w) } func (g IntTraits) EmitNodeMethodIsNull(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Int}.emitNodeMethodIsNull(w) } func (g IntTraits) EmitNodeMethodAsBool(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Int}.emitNodeMethodAsBool(w) } func (g IntTraits) EmitNodeMethodAsFloat(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Int}.emitNodeMethodAsFloat(w) } func (g IntTraits) EmitNodeMethodAsString(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Int}.emitNodeMethodAsString(w) } func (g IntTraits) EmitNodeMethodAsBytes(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Int}.emitNodeMethodAsBytes(w) } func (g IntTraits) EmitNodeMethodAsLink(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Int}.emitNodeMethodAsLink(w) } type IntAssemblerTraits struct { PkgName string TypeName string // see doc in kindAssemblerTraitsGenerator AppliedPrefix string // see doc in kindAssemblerTraitsGenerator } func (IntAssemblerTraits) Kind() datamodel.Kind { return datamodel.Kind_Int } func (g IntAssemblerTraits) EmitNodeAssemblerMethodBeginMap(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Int}.emitNodeAssemblerMethodBeginMap(w) } func (g IntAssemblerTraits) EmitNodeAssemblerMethodBeginList(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Int}.emitNodeAssemblerMethodBeginList(w) } func (g IntAssemblerTraits) EmitNodeAssemblerMethodAssignNull(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Int}.emitNodeAssemblerMethodAssignNull(w) } func (g IntAssemblerTraits) EmitNodeAssemblerMethodAssignBool(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Int}.emitNodeAssemblerMethodAssignBool(w) } func (g IntAssemblerTraits) EmitNodeAssemblerMethodAssignFloat(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Int}.emitNodeAssemblerMethodAssignFloat(w) } func (g IntAssemblerTraits) EmitNodeAssemblerMethodAssignString(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Int}.emitNodeAssemblerMethodAssignString(w) } func (g IntAssemblerTraits) EmitNodeAssemblerMethodAssignBytes(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Int}.emitNodeAssemblerMethodAssignBytes(w) } func (g IntAssemblerTraits) EmitNodeAssemblerMethodAssignLink(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Int}.emitNodeAssemblerMethodAssignLink(w) } func (g IntAssemblerTraits) EmitNodeAssemblerMethodPrototype(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Int}.emitNodeAssemblerMethodPrototype(w) } ================================================ FILE: schema/gen/go/mixins/kindTraits.go ================================================ package mixins import ( "io" "github.com/ipld/go-ipld-prime/datamodel" ) // kindTraitsGenerator is the center of all the other mixins, // and handles all the method generation which is a pure function of the kind. // // OVERRIDE THE METHODS THAT DO APPLY TO YOUR KIND; // the default method bodies produced by this mixin are those that return errors, // and that is not what you want for the methods that *are* interesting for your kind. // The kindTraitsGenerator methods will panic if called for a kind that should've overridden them. // // If you're implementing something that can hold "any" kind, // probably none of these methods apply to you at all. // // The other types in this package use kindTraitsGenerator with a fixed Kind, // and only forward the methods to it that don't apply for their kind; // this means when they're used as an anonymous embed, they grant // all the appropriate dummy methods to their container, // while leaving the ones that are still needed entirely absent, // so the compiler helpfully tells you to finish rather than waiting until // runtime to panic if a should-have-been-overriden method slips through. type kindTraitsGenerator struct { PkgName string TypeName string // as will be printed in messages (e.g. can be goosed up a bit, like "Thing.Repr" instead of "_Thing__Repr"). TypeSymbol string // the identifier in code (sometimes is munged internals like "_Thing__Repr" corresponding to no publicly admitted schema.Type.Name). Kind datamodel.Kind } func (g kindTraitsGenerator) emitNodeMethodLookupByString(w io.Writer) { if datamodel.KindSet_JustMap.Contains(g.Kind) { panic("gen internals error: you should've overridden this") } doTemplate(` func ({{ .TypeSymbol }}) LookupByString(string) (datamodel.Node, error) { return mixins.{{ .Kind.String | title }}{TypeName: "{{ .PkgName }}.{{ .TypeName }}"}.LookupByString("") } `, w, g) } func (g kindTraitsGenerator) emitNodeMethodLookupByNode(w io.Writer) { if datamodel.KindSet_JustMap.Contains(g.Kind) { panic("gen internals error: you should've overridden this") } doTemplate(` func ({{ .TypeSymbol }}) LookupByNode(datamodel.Node) (datamodel.Node, error) { return mixins.{{ .Kind.String | title }}{TypeName: "{{ .PkgName }}.{{ .TypeName }}"}.LookupByNode(nil) } `, w, g) } func (g kindTraitsGenerator) emitNodeMethodLookupByIndex(w io.Writer) { if datamodel.KindSet_JustList.Contains(g.Kind) { panic("gen internals error: you should've overridden this") } doTemplate(` func ({{ .TypeSymbol }}) LookupByIndex(idx int64) (datamodel.Node, error) { return mixins.{{ .Kind.String | title }}{TypeName: "{{ .PkgName }}.{{ .TypeName }}"}.LookupByIndex(0) } `, w, g) } func (g kindTraitsGenerator) emitNodeMethodLookupBySegment(w io.Writer) { if datamodel.KindSet_Recursive.Contains(g.Kind) { panic("gen internals error: you should've overridden this") } doTemplate(` func ({{ .TypeSymbol }}) LookupBySegment(seg datamodel.PathSegment) (datamodel.Node, error) { return mixins.{{ .Kind.String | title }}{TypeName: "{{ .PkgName }}.{{ .TypeName }}"}.LookupBySegment(seg) } `, w, g) } func (g kindTraitsGenerator) emitNodeMethodMapIterator(w io.Writer) { if datamodel.KindSet_JustMap.Contains(g.Kind) { panic("gen internals error: you should've overridden this") } doTemplate(` func ({{ .TypeSymbol }}) MapIterator() datamodel.MapIterator { return nil } `, w, g) } func (g kindTraitsGenerator) emitNodeMethodListIterator(w io.Writer) { if datamodel.KindSet_JustList.Contains(g.Kind) { panic("gen internals error: you should've overridden this") } doTemplate(` func ({{ .TypeSymbol }}) ListIterator() datamodel.ListIterator { return nil } `, w, g) } func (g kindTraitsGenerator) emitNodeMethodLength(w io.Writer) { if datamodel.KindSet_Recursive.Contains(g.Kind) { panic("gen internals error: you should've overridden this") } doTemplate(` func ({{ .TypeSymbol }}) Length() int64 { return -1 } `, w, g) } func (g kindTraitsGenerator) emitNodeMethodIsAbsent(w io.Writer) { doTemplate(` func ({{ .TypeSymbol }}) IsAbsent() bool { return false } `, w, g) } func (g kindTraitsGenerator) emitNodeMethodIsNull(w io.Writer) { doTemplate(` func ({{ .TypeSymbol }}) IsNull() bool { return false } `, w, g) } func (g kindTraitsGenerator) emitNodeMethodAsBool(w io.Writer) { if datamodel.KindSet_JustBool.Contains(g.Kind) { panic("gen internals error: you should've overridden this") } doTemplate(` func ({{ .TypeSymbol }}) AsBool() (bool, error) { return mixins.{{ .Kind.String | title }}{TypeName: "{{ .PkgName }}.{{ .TypeName }}"}.AsBool() } `, w, g) } func (g kindTraitsGenerator) emitNodeMethodAsInt(w io.Writer) { if datamodel.KindSet_JustInt.Contains(g.Kind) { panic("gen internals error: you should've overridden this") } doTemplate(` func ({{ .TypeSymbol }}) AsInt() (int64, error) { return mixins.{{ .Kind.String | title }}{TypeName: "{{ .PkgName }}.{{ .TypeName }}"}.AsInt() } `, w, g) } func (g kindTraitsGenerator) emitNodeMethodAsFloat(w io.Writer) { if datamodel.KindSet_JustFloat.Contains(g.Kind) { panic("gen internals error: you should've overridden this") } doTemplate(` func ({{ .TypeSymbol }}) AsFloat() (float64, error) { return mixins.{{ .Kind.String | title }}{TypeName: "{{ .PkgName }}.{{ .TypeName }}"}.AsFloat() } `, w, g) } func (g kindTraitsGenerator) emitNodeMethodAsString(w io.Writer) { if datamodel.KindSet_JustString.Contains(g.Kind) { panic("gen internals error: you should've overridden this") } doTemplate(` func ({{ .TypeSymbol }}) AsString() (string, error) { return mixins.{{ .Kind.String | title }}{TypeName: "{{ .PkgName }}.{{ .TypeName }}"}.AsString() } `, w, g) } func (g kindTraitsGenerator) emitNodeMethodAsBytes(w io.Writer) { if datamodel.KindSet_JustBytes.Contains(g.Kind) { panic("gen internals error: you should've overridden this") } doTemplate(` func ({{ .TypeSymbol }}) AsBytes() ([]byte, error) { return mixins.{{ .Kind.String | title }}{TypeName: "{{ .PkgName }}.{{ .TypeName }}"}.AsBytes() } `, w, g) } func (g kindTraitsGenerator) emitNodeMethodAsLink(w io.Writer) { if datamodel.KindSet_JustLink.Contains(g.Kind) { panic("gen internals error: you should've overridden this") } doTemplate(` func ({{ .TypeSymbol }}) AsLink() (datamodel.Link, error) { return mixins.{{ .Kind.String | title }}{TypeName: "{{ .PkgName }}.{{ .TypeName }}"}.AsLink() } `, w, g) } // kindAssemblerTraitsGenerator is an awfully lot like kindTraitsGenerator, // except applying to methods for builders and assemblers. type kindAssemblerTraitsGenerator struct { PkgName string TypeName string // as will be printed in messages (e.g. can be goosed up a bit, like "Thing.Repr" instead of "_Thing__Repr"). AppliedPrefix string // the prefix of what to attach methods to... this one is a little wild: should probably be either "_{{ .Type | TypeSymbol }}__" or "_{{ .Type | TypeSymbol }}__Repr", and we'll just add the words "Builder" and "Assembler". Kind datamodel.Kind } // bailed on extracting a common emitNodeBuilderType: too many variations in content and pointer placement to be worth it. // bailed on extracting a common emitNodeBuilderMethods: same. // bailed on extracting a common emitNodeAssemblerType: same. // // If you try to do these, you'll probably need: // - an explicit understanding of if generating representations or not // - to still be ready for boatloads of exceptions if the representation isn't directly castable to and from the type-level node. func (g kindAssemblerTraitsGenerator) emitNodeAssemblerMethodBeginMap(w io.Writer) { if datamodel.KindSet_JustMap.Contains(g.Kind) { panic("gen internals error: you should've overridden this") } doTemplate(` func ({{ .AppliedPrefix }}Assembler) BeginMap(sizeHint int64) (datamodel.MapAssembler, error) { return mixins.{{ .Kind.String | title }}Assembler{TypeName: "{{ .PkgName }}.{{ .TypeName }}"}.BeginMap(0) } `, w, g) } func (g kindAssemblerTraitsGenerator) emitNodeAssemblerMethodBeginList(w io.Writer) { if datamodel.KindSet_JustList.Contains(g.Kind) { panic("gen internals error: you should've overridden this") } doTemplate(` func ({{ .AppliedPrefix }}Assembler) BeginList(sizeHint int64) (datamodel.ListAssembler, error) { return mixins.{{ .Kind.String | title }}Assembler{TypeName: "{{ .PkgName }}.{{ .TypeName }}"}.BeginList(0) } `, w, g) } func (g kindAssemblerTraitsGenerator) emitNodeAssemblerMethodAssignNull(w io.Writer) { if datamodel.KindSet_JustNull.Contains(g.Kind) { panic("gen internals error: you should've overridden this") } doTemplate(` func (na *{{ .AppliedPrefix }}Assembler) AssignNull() error { return mixins.{{ .Kind.String | title }}Assembler{TypeName: "{{ .PkgName }}.{{ .TypeName }}"}.AssignNull() } `, w, g) } func (g kindAssemblerTraitsGenerator) emitNodeAssemblerMethodAssignBool(w io.Writer) { if datamodel.KindSet_JustBool.Contains(g.Kind) { panic("gen internals error: you should've overridden this") } doTemplate(` func ({{ .AppliedPrefix }}Assembler) AssignBool(bool) error { return mixins.{{ .Kind.String | title }}Assembler{TypeName: "{{ .PkgName }}.{{ .TypeName }}"}.AssignBool(false) } `, w, g) } func (g kindAssemblerTraitsGenerator) emitNodeAssemblerMethodAssignInt(w io.Writer) { if datamodel.KindSet_JustInt.Contains(g.Kind) { panic("gen internals error: you should've overridden this") } doTemplate(` func ({{ .AppliedPrefix }}Assembler) AssignInt(int64) error { return mixins.{{ .Kind.String | title }}Assembler{TypeName: "{{ .PkgName }}.{{ .TypeName }}"}.AssignInt(0) } `, w, g) } func (g kindAssemblerTraitsGenerator) emitNodeAssemblerMethodAssignFloat(w io.Writer) { if datamodel.KindSet_JustFloat.Contains(g.Kind) { panic("gen internals error: you should've overridden this") } doTemplate(` func ({{ .AppliedPrefix }}Assembler) AssignFloat(float64) error { return mixins.{{ .Kind.String | title }}Assembler{TypeName: "{{ .PkgName }}.{{ .TypeName }}"}.AssignFloat(0) } `, w, g) } func (g kindAssemblerTraitsGenerator) emitNodeAssemblerMethodAssignString(w io.Writer) { if datamodel.KindSet_JustString.Contains(g.Kind) { panic("gen internals error: you should've overridden this") } doTemplate(` func ({{ .AppliedPrefix }}Assembler) AssignString(string) error { return mixins.{{ .Kind.String | title }}Assembler{TypeName: "{{ .PkgName }}.{{ .TypeName }}"}.AssignString("") } `, w, g) } func (g kindAssemblerTraitsGenerator) emitNodeAssemblerMethodAssignBytes(w io.Writer) { if datamodel.KindSet_JustBytes.Contains(g.Kind) { panic("gen internals error: you should've overridden this") } doTemplate(` func ({{ .AppliedPrefix }}Assembler) AssignBytes([]byte) error { return mixins.{{ .Kind.String | title }}Assembler{TypeName: "{{ .PkgName }}.{{ .TypeName }}"}.AssignBytes(nil) } `, w, g) } func (g kindAssemblerTraitsGenerator) emitNodeAssemblerMethodAssignLink(w io.Writer) { if datamodel.KindSet_JustLink.Contains(g.Kind) { panic("gen internals error: you should've overridden this") } doTemplate(` func ({{ .AppliedPrefix }}Assembler) AssignLink(datamodel.Link) error { return mixins.{{ .Kind.String | title }}Assembler{TypeName: "{{ .PkgName }}.{{ .TypeName }}"}.AssignLink(nil) } `, w, g) } // bailed on extracting a common emitNodeAssemblerMethodAssignNode: way too many variations. func (g kindAssemblerTraitsGenerator) emitNodeAssemblerMethodPrototype(w io.Writer) { doTemplate(` func ({{ .AppliedPrefix }}Assembler) Prototype() datamodel.NodePrototype { return {{ .AppliedPrefix }}Prototype{} } `, w, g) } // bailed on extracting a common emitNodeAssemblerOtherBits: it's just self-evident there's nothing common there. ================================================ FILE: schema/gen/go/mixins/linkGenMixin.go ================================================ package mixins import ( "io" "github.com/ipld/go-ipld-prime/datamodel" ) type LinkTraits struct { PkgName string TypeName string // see doc in kindTraitsGenerator TypeSymbol string // see doc in kindTraitsGenerator } func (LinkTraits) Kind() datamodel.Kind { return datamodel.Kind_Link } func (g LinkTraits) EmitNodeMethodKind(w io.Writer) { doTemplate(` func ({{ .TypeSymbol }}) Kind() datamodel.Kind { return datamodel.Kind_Link } `, w, g) } func (g LinkTraits) EmitNodeMethodLookupByString(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Link}.emitNodeMethodLookupByString(w) } func (g LinkTraits) EmitNodeMethodLookupByNode(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Link}.emitNodeMethodLookupByNode(w) } func (g LinkTraits) EmitNodeMethodLookupByIndex(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Link}.emitNodeMethodLookupByIndex(w) } func (g LinkTraits) EmitNodeMethodLookupBySegment(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Link}.emitNodeMethodLookupBySegment(w) } func (g LinkTraits) EmitNodeMethodMapIterator(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Link}.emitNodeMethodMapIterator(w) } func (g LinkTraits) EmitNodeMethodListIterator(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Link}.emitNodeMethodListIterator(w) } func (g LinkTraits) EmitNodeMethodLength(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Link}.emitNodeMethodLength(w) } func (g LinkTraits) EmitNodeMethodIsAbsent(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Link}.emitNodeMethodIsAbsent(w) } func (g LinkTraits) EmitNodeMethodIsNull(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Link}.emitNodeMethodIsNull(w) } func (g LinkTraits) EmitNodeMethodAsBool(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Link}.emitNodeMethodAsBool(w) } func (g LinkTraits) EmitNodeMethodAsInt(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Link}.emitNodeMethodAsInt(w) } func (g LinkTraits) EmitNodeMethodAsFloat(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Link}.emitNodeMethodAsFloat(w) } func (g LinkTraits) EmitNodeMethodAsString(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Link}.emitNodeMethodAsString(w) } func (g LinkTraits) EmitNodeMethodAsBytes(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Link}.emitNodeMethodAsBytes(w) } type LinkAssemblerTraits struct { PkgName string TypeName string // see doc in kindAssemblerTraitsGenerator AppliedPrefix string // see doc in kindAssemblerTraitsGenerator } func (LinkAssemblerTraits) Kind() datamodel.Kind { return datamodel.Kind_Link } func (g LinkAssemblerTraits) EmitNodeAssemblerMethodBeginMap(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Link}.emitNodeAssemblerMethodBeginMap(w) } func (g LinkAssemblerTraits) EmitNodeAssemblerMethodBeginList(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Link}.emitNodeAssemblerMethodBeginList(w) } func (g LinkAssemblerTraits) EmitNodeAssemblerMethodAssignNull(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Link}.emitNodeAssemblerMethodAssignNull(w) } func (g LinkAssemblerTraits) EmitNodeAssemblerMethodAssignBool(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Link}.emitNodeAssemblerMethodAssignBool(w) } func (g LinkAssemblerTraits) EmitNodeAssemblerMethodAssignInt(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Link}.emitNodeAssemblerMethodAssignInt(w) } func (g LinkAssemblerTraits) EmitNodeAssemblerMethodAssignFloat(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Link}.emitNodeAssemblerMethodAssignFloat(w) } func (g LinkAssemblerTraits) EmitNodeAssemblerMethodAssignString(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Link}.emitNodeAssemblerMethodAssignString(w) } func (g LinkAssemblerTraits) EmitNodeAssemblerMethodAssignBytes(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Link}.emitNodeAssemblerMethodAssignBytes(w) } func (g LinkAssemblerTraits) EmitNodeAssemblerMethodPrototype(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Link}.emitNodeAssemblerMethodPrototype(w) } ================================================ FILE: schema/gen/go/mixins/listGenMixin.go ================================================ package mixins import ( "io" "github.com/ipld/go-ipld-prime/datamodel" ) type ListTraits struct { PkgName string TypeName string // see doc in kindTraitsGenerator TypeSymbol string // see doc in kindTraitsGenerator } func (ListTraits) Kind() datamodel.Kind { return datamodel.Kind_List } func (g ListTraits) EmitNodeMethodKind(w io.Writer) { doTemplate(` func ({{ .TypeSymbol }}) Kind() datamodel.Kind { return datamodel.Kind_List } `, w, g) } func (g ListTraits) EmitNodeMethodLookupByString(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_List}.emitNodeMethodLookupByString(w) } func (g ListTraits) EmitNodeMethodLookupBySegment(w io.Writer) { doTemplate(` func (n {{ .TypeSymbol }}) LookupBySegment(seg datamodel.PathSegment) (datamodel.Node, error) { i, err := seg.Index() if err != nil { return nil, datamodel.ErrInvalidSegmentForList{TypeName: "{{ .PkgName }}.{{ .TypeName }}", TroubleSegment: seg, Reason: err} } return n.LookupByIndex(i) } `, w, g) } func (g ListTraits) EmitNodeMethodMapIterator(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_List}.emitNodeMethodMapIterator(w) } func (g ListTraits) EmitNodeMethodIsAbsent(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_List}.emitNodeMethodIsAbsent(w) } func (g ListTraits) EmitNodeMethodIsNull(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_List}.emitNodeMethodIsNull(w) } func (g ListTraits) EmitNodeMethodAsBool(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_List}.emitNodeMethodAsBool(w) } func (g ListTraits) EmitNodeMethodAsInt(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_List}.emitNodeMethodAsInt(w) } func (g ListTraits) EmitNodeMethodAsFloat(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_List}.emitNodeMethodAsFloat(w) } func (g ListTraits) EmitNodeMethodAsString(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_List}.emitNodeMethodAsString(w) } func (g ListTraits) EmitNodeMethodAsBytes(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_List}.emitNodeMethodAsBytes(w) } func (g ListTraits) EmitNodeMethodAsLink(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_List}.emitNodeMethodAsLink(w) } type ListAssemblerTraits struct { PkgName string TypeName string // see doc in kindAssemblerTraitsGenerator AppliedPrefix string // see doc in kindAssemblerTraitsGenerator } func (ListAssemblerTraits) Kind() datamodel.Kind { return datamodel.Kind_List } func (g ListAssemblerTraits) EmitNodeAssemblerMethodBeginMap(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_List}.emitNodeAssemblerMethodBeginMap(w) } func (g ListAssemblerTraits) EmitNodeAssemblerMethodAssignNull(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_List}.emitNodeAssemblerMethodAssignNull(w) } func (g ListAssemblerTraits) EmitNodeAssemblerMethodAssignBool(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_List}.emitNodeAssemblerMethodAssignBool(w) } func (g ListAssemblerTraits) EmitNodeAssemblerMethodAssignInt(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_List}.emitNodeAssemblerMethodAssignInt(w) } func (g ListAssemblerTraits) EmitNodeAssemblerMethodAssignFloat(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_List}.emitNodeAssemblerMethodAssignFloat(w) } func (g ListAssemblerTraits) EmitNodeAssemblerMethodAssignString(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_List}.emitNodeAssemblerMethodAssignString(w) } func (g ListAssemblerTraits) EmitNodeAssemblerMethodAssignBytes(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_List}.emitNodeAssemblerMethodAssignBytes(w) } func (g ListAssemblerTraits) EmitNodeAssemblerMethodAssignLink(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_List}.emitNodeAssemblerMethodAssignLink(w) } func (g ListAssemblerTraits) EmitNodeAssemblerMethodPrototype(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_List}.emitNodeAssemblerMethodPrototype(w) } ================================================ FILE: schema/gen/go/mixins/mapGenMixin.go ================================================ package mixins import ( "io" "github.com/ipld/go-ipld-prime/datamodel" ) type MapTraits struct { PkgName string TypeName string // see doc in kindTraitsGenerator TypeSymbol string // see doc in kindTraitsGenerator } func (MapTraits) Kind() datamodel.Kind { return datamodel.Kind_Map } func (g MapTraits) EmitNodeMethodKind(w io.Writer) { doTemplate(` func ({{ .TypeSymbol }}) Kind() datamodel.Kind { return datamodel.Kind_Map } `, w, g) } func (g MapTraits) EmitNodeMethodLookupByIndex(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Map}.emitNodeMethodLookupByIndex(w) } func (g MapTraits) EmitNodeMethodLookupBySegment(w io.Writer) { doTemplate(` func (n {{ .TypeSymbol }}) LookupBySegment(seg datamodel.PathSegment) (datamodel.Node, error) { return n.LookupByString(seg.String()) } `, w, g) } func (g MapTraits) EmitNodeMethodListIterator(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Map}.emitNodeMethodListIterator(w) } func (g MapTraits) EmitNodeMethodIsAbsent(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Map}.emitNodeMethodIsAbsent(w) } func (g MapTraits) EmitNodeMethodIsNull(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Map}.emitNodeMethodIsNull(w) } func (g MapTraits) EmitNodeMethodAsBool(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Map}.emitNodeMethodAsBool(w) } func (g MapTraits) EmitNodeMethodAsInt(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Map}.emitNodeMethodAsInt(w) } func (g MapTraits) EmitNodeMethodAsFloat(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Map}.emitNodeMethodAsFloat(w) } func (g MapTraits) EmitNodeMethodAsString(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Map}.emitNodeMethodAsString(w) } func (g MapTraits) EmitNodeMethodAsBytes(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Map}.emitNodeMethodAsBytes(w) } func (g MapTraits) EmitNodeMethodAsLink(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_Map}.emitNodeMethodAsLink(w) } type MapAssemblerTraits struct { PkgName string TypeName string // see doc in kindAssemblerTraitsGenerator AppliedPrefix string // see doc in kindAssemblerTraitsGenerator } func (MapAssemblerTraits) Kind() datamodel.Kind { return datamodel.Kind_Map } func (g MapAssemblerTraits) EmitNodeAssemblerMethodBeginList(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Map}.emitNodeAssemblerMethodBeginList(w) } func (g MapAssemblerTraits) EmitNodeAssemblerMethodAssignNull(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Map}.emitNodeAssemblerMethodAssignNull(w) } func (g MapAssemblerTraits) EmitNodeAssemblerMethodAssignBool(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Map}.emitNodeAssemblerMethodAssignBool(w) } func (g MapAssemblerTraits) EmitNodeAssemblerMethodAssignInt(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Map}.emitNodeAssemblerMethodAssignInt(w) } func (g MapAssemblerTraits) EmitNodeAssemblerMethodAssignFloat(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Map}.emitNodeAssemblerMethodAssignFloat(w) } func (g MapAssemblerTraits) EmitNodeAssemblerMethodAssignString(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Map}.emitNodeAssemblerMethodAssignString(w) } func (g MapAssemblerTraits) EmitNodeAssemblerMethodAssignBytes(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Map}.emitNodeAssemblerMethodAssignBytes(w) } func (g MapAssemblerTraits) EmitNodeAssemblerMethodAssignLink(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Map}.emitNodeAssemblerMethodAssignLink(w) } func (g MapAssemblerTraits) EmitNodeAssemblerMethodPrototype(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_Map}.emitNodeAssemblerMethodPrototype(w) } ================================================ FILE: schema/gen/go/mixins/stringGenMixin.go ================================================ package mixins import ( "io" "github.com/ipld/go-ipld-prime/datamodel" ) type StringTraits struct { PkgName string TypeName string // see doc in kindTraitsGenerator TypeSymbol string // see doc in kindTraitsGenerator } func (StringTraits) Kind() datamodel.Kind { return datamodel.Kind_String } func (g StringTraits) EmitNodeMethodKind(w io.Writer) { doTemplate(` func ({{ .TypeSymbol }}) Kind() datamodel.Kind { return datamodel.Kind_String } `, w, g) } func (g StringTraits) EmitNodeMethodLookupByString(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_String}.emitNodeMethodLookupByString(w) } func (g StringTraits) EmitNodeMethodLookupByNode(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_String}.emitNodeMethodLookupByNode(w) } func (g StringTraits) EmitNodeMethodLookupByIndex(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_String}.emitNodeMethodLookupByIndex(w) } func (g StringTraits) EmitNodeMethodLookupBySegment(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_String}.emitNodeMethodLookupBySegment(w) } func (g StringTraits) EmitNodeMethodMapIterator(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_String}.emitNodeMethodMapIterator(w) } func (g StringTraits) EmitNodeMethodListIterator(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_String}.emitNodeMethodListIterator(w) } func (g StringTraits) EmitNodeMethodLength(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_String}.emitNodeMethodLength(w) } func (g StringTraits) EmitNodeMethodIsAbsent(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_String}.emitNodeMethodIsAbsent(w) } func (g StringTraits) EmitNodeMethodIsNull(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_String}.emitNodeMethodIsNull(w) } func (g StringTraits) EmitNodeMethodAsBool(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_String}.emitNodeMethodAsBool(w) } func (g StringTraits) EmitNodeMethodAsInt(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_String}.emitNodeMethodAsInt(w) } func (g StringTraits) EmitNodeMethodAsFloat(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_String}.emitNodeMethodAsFloat(w) } func (g StringTraits) EmitNodeMethodAsBytes(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_String}.emitNodeMethodAsBytes(w) } func (g StringTraits) EmitNodeMethodAsLink(w io.Writer) { kindTraitsGenerator{g.PkgName, g.TypeName, g.TypeSymbol, datamodel.Kind_String}.emitNodeMethodAsLink(w) } type StringAssemblerTraits struct { PkgName string TypeName string // see doc in kindAssemblerTraitsGenerator AppliedPrefix string // see doc in kindAssemblerTraitsGenerator } func (StringAssemblerTraits) Kind() datamodel.Kind { return datamodel.Kind_String } func (g StringAssemblerTraits) EmitNodeAssemblerMethodBeginMap(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_String}.emitNodeAssemblerMethodBeginMap(w) } func (g StringAssemblerTraits) EmitNodeAssemblerMethodBeginList(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_String}.emitNodeAssemblerMethodBeginList(w) } func (g StringAssemblerTraits) EmitNodeAssemblerMethodAssignNull(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_String}.emitNodeAssemblerMethodAssignNull(w) } func (g StringAssemblerTraits) EmitNodeAssemblerMethodAssignBool(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_String}.emitNodeAssemblerMethodAssignBool(w) } func (g StringAssemblerTraits) EmitNodeAssemblerMethodAssignInt(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_String}.emitNodeAssemblerMethodAssignInt(w) } func (g StringAssemblerTraits) EmitNodeAssemblerMethodAssignFloat(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_String}.emitNodeAssemblerMethodAssignFloat(w) } func (g StringAssemblerTraits) EmitNodeAssemblerMethodAssignBytes(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_String}.emitNodeAssemblerMethodAssignBytes(w) } func (g StringAssemblerTraits) EmitNodeAssemblerMethodAssignLink(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_String}.emitNodeAssemblerMethodAssignLink(w) } func (g StringAssemblerTraits) EmitNodeAssemblerMethodPrototype(w io.Writer) { kindAssemblerTraitsGenerator{g.PkgName, g.TypeName, g.AppliedPrefix, datamodel.Kind_String}.emitNodeAssemblerMethodPrototype(w) } ================================================ FILE: schema/gen/go/mixins/templateUtil.go ================================================ package mixins import ( "io" "strings" "text/template" "github.com/ipld/go-ipld-prime/testutil" ) func doTemplate(tmplstr string, w io.Writer, data interface{}) { tmpl := template.Must(template.New(""). Funcs(template.FuncMap{ "title": func(s string) string { return strings.Title(s) }, //lint:ignore SA1019 cases.Title doesn't work for this }). Parse(testutil.Dedent(tmplstr))) if err := tmpl.Execute(w, data); err != nil { panic(err) } } ================================================ FILE: schema/gen/go/templateUtil.go ================================================ package gengo import ( "io" "strings" "text/template" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/testutil" ) func doTemplate(tmplstr string, w io.Writer, adjCfg *AdjunctCfg, data interface{}) { tmpl := template.Must(template.New(""). Funcs(template.FuncMap{ // These methods are used for symbol munging and appear constantly, so they need to be short. // (You could also get at them through `.AdjCfg`, but going direct saves some screen real estate.) "TypeSymbol": adjCfg.TypeSymbol, "FieldSymbolLower": adjCfg.FieldSymbolLower, "FieldSymbolUpper": adjCfg.FieldSymbolUpper, "MaybeUsesPtr": adjCfg.MaybeUsesPtr, "Comments": adjCfg.Comments, // The whole AdjunctConfig can be accessed. // Access methods like UnionMemlayout through this, as e.g. `.AdjCfg.UnionMemlayout`. "AdjCfg": func() *AdjunctCfg { return adjCfg }, // "dot" is a dummy value that's equal to the original `.` expression, but stays there. // Use this if you're inside a range or other feature that shifted the dot and you want the original. // (This may seem silly, but empirically, I found myself writing a dummy line to store the value of dot before endering a range clause >20 times; that's plenty.) "dot": func() interface{} { return data }, "KindPrim": func(k datamodel.Kind) string { switch k { case datamodel.Kind_Map: panic("this isn't useful for non-scalars") case datamodel.Kind_List: panic("this isn't useful for non-scalars") case datamodel.Kind_Null: panic("this isn't useful for null") case datamodel.Kind_Bool: return "bool" case datamodel.Kind_Int: return "int64" case datamodel.Kind_Float: return "float64" case datamodel.Kind_String: return "string" case datamodel.Kind_Bytes: return "[]byte" case datamodel.Kind_Link: return "datamodel.Link" default: panic("invalid enumeration value!") } }, "Kind": func(s string) datamodel.Kind { switch s { case "map": return datamodel.Kind_Map case "list": return datamodel.Kind_List case "null": return datamodel.Kind_Null case "bool": return datamodel.Kind_Bool case "int": return datamodel.Kind_Int case "float": return datamodel.Kind_Float case "string": return datamodel.Kind_String case "bytes": return datamodel.Kind_Bytes case "link": return datamodel.Kind_Link default: panic("invalid enumeration value!") } }, "KindSymbol": func(k datamodel.Kind) string { switch k { case datamodel.Kind_Map: return "datamodel.Kind_Map" case datamodel.Kind_List: return "datamodel.Kind_List" case datamodel.Kind_Null: return "datamodel.Kind_Null" case datamodel.Kind_Bool: return "datamodel.Kind_Bool" case datamodel.Kind_Int: return "datamodel.Kind_Int" case datamodel.Kind_Float: return "datamodel.Kind_Float" case datamodel.Kind_String: return "datamodel.Kind_String" case datamodel.Kind_Bytes: return "datamodel.Kind_Bytes" case datamodel.Kind_Link: return "datamodel.Kind_Link" default: panic("invalid enumeration value!") } }, "add": func(a, b int) int { return a + b }, "title": func(s string) string { return strings.Title(s) }, //lint:ignore SA1019 cases.Title doesn't work for this }). Parse(testutil.Dedent(tmplstr))) if err := tmpl.Execute(w, data); err != nil { panic(err) } } // We really need to do some more composable stuff around here. // Generators should probably be carrying down their own doTemplate methods that curry customizations. // E.g., map generators would benefit hugely from being able to make a clause for "entTypeStrung", "mTypeStrung", etc. // // Open question: how exactly? Should some of this stuff should be composed by: // - composing template fragments; // - amending the funcmap; // - computing the whole result and injecting it as a string; // - ... combinations of the above? // Adding to the complexity of the question is that sometimes we want to be // doing composition inside the output (e.g. DRY by functions in the result, // rather than by DRY'ing the templates). // Best practice to make this evolve nicely is not at all obvious to this author. // ================================================ FILE: schema/gen/go/testEngine_disabled_test.go ================================================ //go:build skipgenbehavtests || windows package gengo import ( "fmt" "io" "os" "os/exec" "path/filepath" "runtime" "testing" "github.com/ipld/go-ipld-prime/datamodel" ) func buildGennedCode(t *testing.T, prefix string, pkgName string) { // Emit a small file with a 'main' method. // 'go build' doesn't like it we're in a package called "main" and there isn't one // (and at the same time, plugins demand that they be in a package called 'main', // so 'pkgName' in practice is almost always "main"). // I dunno, friend. I didn't write the rules. if pkgName == "main" { withFile(filepath.Join(tmpGenBuildDir, prefix, "main.go"), func(w io.Writer) { fmt.Fprintf(w, "package %s\n\n", pkgName) fmt.Fprintf(w, "func main() {}\n") }) } // If windows, remove all files in tmpGenBuildDir with the .exe extension so we don't get a "already exists" error // https://github.com/golang/go/issues/57039 if runtime.GOOS == "windows" { files, err := filepath.Glob(filepath.Join(tmpGenBuildDir, prefix, "*.exe")) if err != nil { t.Fatal(err) } for _, file := range files { if err := os.Remove(file); err != nil { t.Fatal(err) } } } // Invoke 'go build' -- nothing fancy. files, err := filepath.Glob(filepath.Join(tmpGenBuildDir, prefix, "*.go")) if err != nil { t.Fatal(err) } args := []string{"build"} args = append(args, files...) cmd := exec.Command("go", args...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr err = cmd.Run() if err != nil { t.Fatalf("genned code failed to compile: %s", err) } t.Skip("behavioral tests for generated code skipped: you used the 'skipgenbehavtests' build tag.") } func fnPrototypeByName(prefix string) func(string) datamodel.NodePrototype { return nil // unused } ================================================ FILE: schema/gen/go/testEngine_nocgo_test.go ================================================ //go:build !cgo && !skipgenbehavtests && !windows // Confession: // This build tag specification is NOT sufficient nor necessarily correct -- // it's a vague approximation of what's present in the stdlib 'plugin' package. // It's also not at all a sure thing that cgo will actually *work* just // because a build tag hasn't explicitly stated that it *mayn't* -- cgo can // and will fail for environmental reasons at the point the compiler uses it. // // Ideally, there'd be a way to *ask* the plugin package if it's going to // work or not before we try to use it; unfortunately, at the time of writing, // it does not appear there is such an ability. // // If you run afoul of these build tags somehow (e.g., building plugins isn't // possible in your environment for some reason), use the 'skipgenbehavtests' // build tag to right yourself. That's what it's there for. package gengo import ( "fmt" "io" "os" "os/exec" "path/filepath" "testing" "github.com/ipld/go-ipld-prime/datamodel" ) func buildGennedCode(t *testing.T, prefix string, pkgName string) { // Emit a small file with a 'main' method. // 'go build' doesn't like it we're in a package called "main" and there isn't one // (and at the same time, plugins demand that they be in a package called 'main', // so 'pkgName' in practice is almost always "main"). // I dunno, friend. I didn't write the rules. if pkgName == "main" { withFile(filepath.Join(tmpGenBuildDir, prefix, "main.go"), func(w io.Writer) { fmt.Fprintf(w, "package %s\n\n", pkgName) fmt.Fprintf(w, "func main() {}\n") }) } // Invoke 'go build' -- nothing fancy. files, err := filepath.Glob(filepath.Join(tmpGenBuildDir, prefix, "*.go")) if err != nil { t.Fatal(err) } args := []string{"build"} args = append(args, files...) cmd := exec.Command("go", args...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr err = cmd.Run() if err != nil { t.Fatalf("genned code failed to compile: %s", err) } t.Skip("behavioral tests for generated code skipped: cgo is required for these tests") } func fnPrototypeByName(prefix string) func(string) datamodel.NodePrototype { return nil // unused } ================================================ FILE: schema/gen/go/testEngine_plugin_test.go ================================================ //go:build cgo && !skipgenbehavtests && !windows package gengo import ( "os" "os/exec" "path/filepath" "plugin" "testing" "github.com/ipld/go-ipld-prime/datamodel" ) func objPath(prefix string) string { return filepath.Join(tmpGenBuildDir, prefix, "obj.so") } func buildGennedCode(t *testing.T, prefix string, _ string) { // Invoke `go build` with flags to create a plugin -- we'll be able to // load into this plugin into this selfsame process momentarily. // Use globbing, because these are files outside our module. files, err := filepath.Glob(filepath.Join(tmpGenBuildDir, prefix, "*.go")) if err != nil { t.Fatal(err) } args := []string{"build", "-o=" + objPath(prefix), "-buildmode=plugin"} args = append(args, files...) cmd := exec.Command("go", args...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { t.Fatalf("genned code failed to compile: %s", err) } } func fnPrototypeByName(prefix string) func(string) datamodel.NodePrototype { plg, err := plugin.Open(objPath(prefix)) if err != nil { panic(err) // Panic because if this was going to flunk, we expected it to flunk earlier when we ran 'go build'. } sym, err := plg.Lookup("GetPrototypeByName") if err != nil { panic(err) } return sym.(func(string) datamodel.NodePrototype) } ================================================ FILE: schema/gen/go/testEngine_test.go ================================================ package gengo import ( "io" "os" "path/filepath" "testing" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/node/tests" "github.com/ipld/go-ipld-prime/schema" ) var _ tests.Engine = (*genAndCompileEngine)(nil) type genAndCompileEngine struct { subtestName string prefix string adjCfg AdjunctCfg prototypeByName func(string) datamodel.NodePrototype } var tmpGenBuildDir = filepath.Join(os.TempDir(), "test-go-ipld-prime-gengo") func (e *genAndCompileEngine) Init(t *testing.T, ts schema.TypeSystem) { // Make directories for the package we're about to generate. // They will live in a temporary directory, usually // /tmp/test-go-ipld-prime-gengo on Linux. It can be removed at any time. // We don't by default because it's nicer to let go's builds of things cache. // If you change the names of types, though, you'll have garbage files leftover, // and that's currently a manual cleanup problem. Sorry. dir := filepath.Join(tmpGenBuildDir, e.prefix) if err := os.MkdirAll(dir, 0755); err != nil { t.Fatal(err) } pkgName := "main" // Generate... everything, really. Generate(dir, pkgName, ts, &e.adjCfg) // Emit an exported top level function for getting NodePrototype. // This part isn't necessary except for a special need we have with this plugin trick; // normally, user code uses the `{pkgname}.Prototype.{TypeName}` constant (so-to-speak, anyway) to get a hold of NodePrototypes... // but for plugins, we need a top-level exported symbol to grab ahold of, and we can't easily look through the `Prototype` value // without an interface... so we generate this function to fit the bill instead. withFile(filepath.Join(dir, "prototypeGetter.go"), func(w io.Writer) { doTemplate(` package `+pkgName+` import "github.com/ipld/go-ipld-prime/datamodel" func GetPrototypeByName(name string) datamodel.NodePrototype { switch name { {{- range . }} case "{{ .Name }}": return _{{ . | TypeSymbol }}__Prototype{} case "{{ .Name }}.Repr": return _{{ . | TypeSymbol }}__ReprPrototype{} {{- end}} default: return nil } } `, w, &e.adjCfg, ts.GetTypes()) }) // Build the genned code. // This will either make a plugin (which we can run behavioral tests on next!), // or just build it quietly just to see if there are compile-time errors, // depending on your build tags. // See 'HACKME_testing.md' for discussion. buildGennedCode(t, e.prefix, pkgName) e.prototypeByName = fnPrototypeByName(e.prefix) } func (e *genAndCompileEngine) PrototypeByName(name string) datamodel.NodePrototype { return e.prototypeByName(name) } ================================================ FILE: schema/gen/go/testLinks_test.go ================================================ package gengo import ( "runtime" "testing" "github.com/ipld/go-ipld-prime/node/tests" ) func TestLinks(t *testing.T) { if runtime.GOOS != "darwin" { // TODO: enable parallelism on macos t.Parallel() } engine := &genAndCompileEngine{prefix: "links"} tests.SchemaTestLinks(t, engine) } ================================================ FILE: schema/gen/go/testLists_test.go ================================================ package gengo import ( "runtime" "testing" "github.com/ipld/go-ipld-prime/node/tests" "github.com/ipld/go-ipld-prime/schema" ) func TestListsContainingMaybe(t *testing.T) { if runtime.GOOS != "darwin" { // TODO: enable parallelism on macos t.Parallel() } for _, engine := range []*genAndCompileEngine{ { subtestName: "maybe-using-embed", prefix: "lists-embed", adjCfg: AdjunctCfg{ maybeUsesPtr: map[schema.TypeName]bool{"String": false}, }, }, { subtestName: "maybe-using-ptr", prefix: "lists-mptr", adjCfg: AdjunctCfg{ maybeUsesPtr: map[schema.TypeName]bool{"String": false}, }, }, } { t.Run(engine.subtestName, func(t *testing.T) { tests.SchemaTestListsContainingMaybe(t, engine) }) } } func TestListsContainingLists(t *testing.T) { if runtime.GOOS != "darwin" { // TODO: enable parallelism on macos t.Parallel() } engine := &genAndCompileEngine{prefix: "lists-of-lists"} tests.SchemaTestListsContainingLists(t, engine) } ================================================ FILE: schema/gen/go/testMaps_test.go ================================================ package gengo import ( "runtime" "testing" "github.com/ipld/go-ipld-prime/node/tests" "github.com/ipld/go-ipld-prime/schema" ) func TestMapsContainingMaybe(t *testing.T) { if runtime.GOOS != "darwin" { // TODO: enable parallelism on macos t.Parallel() } for _, engine := range []*genAndCompileEngine{ { subtestName: "maybe-using-embed", prefix: "maps-embed", adjCfg: AdjunctCfg{ maybeUsesPtr: map[schema.TypeName]bool{"String": false}, }, }, { subtestName: "maybe-using-ptr", prefix: "maps-mptr", adjCfg: AdjunctCfg{ maybeUsesPtr: map[schema.TypeName]bool{"String": false}, }, }, } { t.Run(engine.subtestName, func(t *testing.T) { tests.SchemaTestMapsContainingMaybe(t, engine) }) } } func TestMapsContainingMaps(t *testing.T) { if runtime.GOOS != "darwin" { // TODO: enable parallelism on macos t.Parallel() } engine := &genAndCompileEngine{prefix: "maps-recursive"} tests.SchemaTestMapsContainingMaps(t, engine) } func TestMapsWithComplexKeys(t *testing.T) { if runtime.GOOS != "darwin" { // TODO: enable parallelism on macos t.Parallel() } engine := &genAndCompileEngine{prefix: "maps-cmplx-keys"} tests.SchemaTestMapsWithComplexKeys(t, engine) } ================================================ FILE: schema/gen/go/testScalars_test.go ================================================ package gengo import ( "runtime" "testing" "github.com/ipld/go-ipld-prime/node/tests" ) func TestScalars(t *testing.T) { if runtime.GOOS != "darwin" { // TODO: enable parallelism on macos t.Parallel() } engine := &genAndCompileEngine{prefix: "scalars"} tests.SchemaTestScalars(t, engine) } ================================================ FILE: schema/gen/go/testStructReprStringjoin_test.go ================================================ package gengo import ( "runtime" "testing" "github.com/ipld/go-ipld-prime/node/tests" ) func TestStructReprStringjoin(t *testing.T) { if runtime.GOOS != "darwin" { // TODO: enable parallelism on macos t.Parallel() } engine := &genAndCompileEngine{prefix: "struct-str-join"} tests.SchemaTestStructReprStringjoin(t, engine) } ================================================ FILE: schema/gen/go/testStructReprTuple_test.go ================================================ package gengo import ( "runtime" "testing" "github.com/ipld/go-ipld-prime/node/tests" ) func TestStructReprTuple(t *testing.T) { if runtime.GOOS != "darwin" { // TODO: enable parallelism on macos t.Parallel() } engine := &genAndCompileEngine{prefix: "struct-tuple"} tests.SchemaTestStructReprTuple(t, engine) } ================================================ FILE: schema/gen/go/testStruct_test.go ================================================ package gengo import ( "runtime" "testing" "github.com/ipld/go-ipld-prime/node/tests" ) func TestRequiredFields(t *testing.T) { if runtime.GOOS != "darwin" { // TODO: enable parallelism on macos t.Parallel() } engine := &genAndCompileEngine{prefix: "struct-required-fields"} tests.SchemaTestRequiredFields(t, engine) } ================================================ FILE: schema/gen/go/testStructsContainingMaybe_test.go ================================================ package gengo import ( "runtime" "testing" "github.com/ipld/go-ipld-prime/node/tests" "github.com/ipld/go-ipld-prime/schema" ) func TestStructsContainingMaybe(t *testing.T) { if runtime.GOOS != "darwin" { // TODO: enable parallelism on macos t.Parallel() } for _, engine := range []*genAndCompileEngine{ { subtestName: "maybe-using-embed", prefix: "stroct", adjCfg: AdjunctCfg{ maybeUsesPtr: map[schema.TypeName]bool{"String": false}, }, }, { subtestName: "maybe-using-ptr", prefix: "stroct2", adjCfg: AdjunctCfg{ maybeUsesPtr: map[schema.TypeName]bool{"String": false}, }, }, } { t.Run(engine.subtestName, func(t *testing.T) { tests.SchemaTestStructsContainingMaybe(t, engine) }) } } ================================================ FILE: schema/gen/go/testUnionsKinded_test.go ================================================ package gengo import ( "runtime" "testing" "github.com/ipld/go-ipld-prime/node/tests" "github.com/ipld/go-ipld-prime/schema" ) func TestUnionKinded(t *testing.T) { if runtime.GOOS != "darwin" { // TODO: enable parallelism on macos t.Parallel() } for _, engine := range []*genAndCompileEngine{ { subtestName: "union-using-embed", prefix: "union-kinded-using-embed", adjCfg: AdjunctCfg{ CfgUnionMemlayout: map[schema.TypeName]string{"WheeUnion": "embedAll"}, }, }, { subtestName: "union-using-interface", prefix: "union-kinded-using-interface", adjCfg: AdjunctCfg{ CfgUnionMemlayout: map[schema.TypeName]string{"WheeUnion": "interface"}, }, }, } { t.Run(engine.subtestName, func(t *testing.T) { tests.SchemaTestUnionKinded(t, engine) }) } } ================================================ FILE: schema/gen/go/testUnionsStringprefix_test.go ================================================ package gengo import ( "runtime" "testing" "github.com/ipld/go-ipld-prime/node/tests" "github.com/ipld/go-ipld-prime/schema" ) func TestUnionStringprefix(t *testing.T) { if runtime.GOOS != "darwin" { // TODO: enable parallelism on macos t.Parallel() } for _, engine := range []*genAndCompileEngine{ { subtestName: "union-using-embed", prefix: "union-stringprefix-using-embed", adjCfg: AdjunctCfg{ CfgUnionMemlayout: map[schema.TypeName]string{"WheeUnion": "embedAll"}, }, }, { subtestName: "union-using-interface", prefix: "union-stringprefix-using-interface", adjCfg: AdjunctCfg{ CfgUnionMemlayout: map[schema.TypeName]string{"WheeUnion": "interface"}, }, }, } { t.Run(engine.subtestName, func(t *testing.T) { tests.SchemaTestUnionStringprefix(t, engine) }) } } ================================================ FILE: schema/gen/go/testUnions_test.go ================================================ package gengo import ( "runtime" "testing" "github.com/ipld/go-ipld-prime/node/tests" "github.com/ipld/go-ipld-prime/schema" ) func TestUnionKeyed(t *testing.T) { if runtime.GOOS != "darwin" { // TODO: enable parallelism on macos t.Parallel() } for _, engine := range []*genAndCompileEngine{ { subtestName: "union-using-embed", prefix: "union-keyed-using-embed", adjCfg: AdjunctCfg{ CfgUnionMemlayout: map[schema.TypeName]string{"StrStr": "embedAll"}, }, }, { subtestName: "union-using-ptr", prefix: "union-keyed-using-interface", adjCfg: AdjunctCfg{ CfgUnionMemlayout: map[schema.TypeName]string{"StrStr": "interface"}, }, }, } { t.Run(engine.subtestName, func(t *testing.T) { tests.SchemaTestUnionKeyed(t, engine) }) } } func TestUnionKeyedComplexChildren(t *testing.T) { if runtime.GOOS != "darwin" { // TODO: enable parallelism on macos t.Parallel() } for _, engine := range []*genAndCompileEngine{ { subtestName: "union-using-embed", prefix: "union-keyed-complex-child-using-embed", adjCfg: AdjunctCfg{ CfgUnionMemlayout: map[schema.TypeName]string{"WheeUnion": "embedAll"}, }, }, { subtestName: "union-using-interface", prefix: "union-keyed-complex-child-using-interface", adjCfg: AdjunctCfg{ CfgUnionMemlayout: map[schema.TypeName]string{"WheeUnion": "interface"}, }, }, } { t.Run(engine.subtestName, func(t *testing.T) { tests.SchemaTestUnionKeyedComplexChildren(t, engine) }) } } func TestUnionKeyedReset(t *testing.T) { if runtime.GOOS != "darwin" { // TODO: enable parallelism on macos t.Parallel() } for _, engine := range []*genAndCompileEngine{ { subtestName: "union-using-embed", prefix: "union-keyed-reset-using-embed", adjCfg: AdjunctCfg{ CfgUnionMemlayout: map[schema.TypeName]string{"WheeUnion": "embedAll"}, }, }, { subtestName: "union-using-interface", prefix: "union-keyed-reset-using-interface", adjCfg: AdjunctCfg{ CfgUnionMemlayout: map[schema.TypeName]string{"WheeUnion": "interface"}, }, }, } { t.Run(engine.subtestName, func(t *testing.T) { tests.SchemaTestUnionKeyedReset(t, engine) }) } } ================================================ FILE: schema/kind.go ================================================ package schema import ( "github.com/ipld/go-ipld-prime/datamodel" ) // TypeKind is an enum of kind in the IPLD Schema system. // // Note that schema.TypeKind is distinct from datamodel.Kind! // Schema kinds include concepts such as "struct" and "enum", which are // concepts only introduced by the Schema layer, and not present in the // Data Model layer. type TypeKind uint8 const ( TypeKind_Invalid TypeKind = 0 TypeKind_Map TypeKind = '{' TypeKind_List TypeKind = '[' TypeKind_Unit TypeKind = '1' TypeKind_Bool TypeKind = 'b' TypeKind_Int TypeKind = 'i' TypeKind_Float TypeKind = 'f' TypeKind_String TypeKind = 's' TypeKind_Bytes TypeKind = 'x' TypeKind_Link TypeKind = '/' TypeKind_Struct TypeKind = '$' TypeKind_Union TypeKind = '^' TypeKind_Enum TypeKind = '%' TypeKind_Any TypeKind = '?' ) func (k TypeKind) String() string { switch k { case TypeKind_Invalid: return "invalid" case TypeKind_Map: return "map" case TypeKind_Any: return "any" case TypeKind_List: return "list" case TypeKind_Unit: return "unit" case TypeKind_Bool: return "bool" case TypeKind_Int: return "int" case TypeKind_Float: return "float" case TypeKind_String: return "string" case TypeKind_Bytes: return "bytes" case TypeKind_Link: return "link" case TypeKind_Struct: return "struct" case TypeKind_Union: return "union" case TypeKind_Enum: return "enum" default: panic("invalid enumeration value!") } } // ActsLike returns a constant from the datamodel.Kind enum describing what // this schema.TypeKind acts like at the Data Model layer. // // Things with similar names are generally conserved // (e.g. "map" acts like "map"); // concepts added by the schema layer have to be mapped onto something // (e.g. "struct" acts like "map"). // // Note that this mapping describes how a typed Node will *act*, programmatically; // it does not necessarily describe how it will be *serialized* // (for example, a struct will always act like a map, even if it has a tuple // representation strategy and thus becomes a list when serialized). func (k TypeKind) ActsLike() datamodel.Kind { switch k { case TypeKind_Invalid: return datamodel.Kind_Invalid case TypeKind_Map: return datamodel.Kind_Map case TypeKind_List: return datamodel.Kind_List case TypeKind_Unit: return datamodel.Kind_Bool // maps to 'true'. // REVIEW: odd that this doesn't map to 'null'? // TODO this should be standardized in the specs, in a table. case TypeKind_Bool: return datamodel.Kind_Bool case TypeKind_Int: return datamodel.Kind_Int case TypeKind_Float: return datamodel.Kind_Float case TypeKind_String: return datamodel.Kind_String case TypeKind_Bytes: return datamodel.Kind_Bytes case TypeKind_Link: return datamodel.Kind_Link case TypeKind_Struct: return datamodel.Kind_Map // clear enough: fields are keys. case TypeKind_Union: return datamodel.Kind_Map case TypeKind_Enum: return datamodel.Kind_String // 'AsString' is the one clear thing to define. case TypeKind_Any: return datamodel.Kind_Invalid // TODO: maybe ActsLike should return (Kind, bool) default: panic("invalid enumeration value!") } } ================================================ FILE: schema/maybe.go ================================================ package schema type Maybe uint8 const ( Maybe_Absent = Maybe(0) Maybe_Null = Maybe(1) Maybe_Value = Maybe(2) ) ================================================ FILE: schema/tmpBuilders.go ================================================ package schema import ( "fmt" "github.com/ipld/go-ipld-prime/datamodel" ) // Everything in this file is __a temporary hack__ and will be __removed__. // // These methods will only hang around until more of the "ast" packages are finished; // thereafter, building schema.Type and schema.TypeSystem values will only be // possible through first constructing a schema AST, and *then* using Reify(), // which will validate things correctly, cycle-check, cross-link, etc. // // (Meanwhile, we're using these methods in the codegen prototypes.) // These methods use Type objects as parameters when pointing to other things, // but this is... turning out consistently problematic. // Even when we're doing this hacky direct-call doesn't-need-to-be-serializable temp stuff, // as written, this doesn't actually let us express cyclic things viably! // The same initialization questions are also going to come up again when we try to make // concrete values in the output of codegen. // Maybe it's actually just a bad idea to have our reified Type types use Type pointers at all. // (I will never get tired of the tongue twisters, evidently.) // I'm not actually using that much, and it's always avoidable (it's trivial to replace with a map lookup bouncing through a 'ts' variable somewhere). // And having the AST gen'd types be... just... the thing... sounds nice. It could save a lot of work. // (It would mean the golang types don't tell you whether the values have been checked for global properties or not, but, eh.) // (It's not really compatible with "Prototype and Type are the same thing for codegen'd stuff", either (or, we need more interfaces, and to *really* lean into them), but maybe that's okay.) func SpawnTypeSystem(types ...Type) (*TypeSystem, []error) { ts := TypeSystem{} ts.Init() for _, typ := range types { ts.Accumulate(typ) } if errs := ts.ValidateGraph(); errs != nil { return nil, errs } return &ts, nil } func MustTypeSystem(types ...Type) *TypeSystem { if ts, err := SpawnTypeSystem(types...); err != nil { panic(err) } else { return ts } } func SpawnString(name TypeName) *TypeString { return &TypeString{typeBase{name, nil}} } func SpawnBool(name TypeName) *TypeBool { return &TypeBool{typeBase{name, nil}} } func SpawnInt(name TypeName) *TypeInt { return &TypeInt{typeBase{name, nil}} } func SpawnFloat(name TypeName) *TypeFloat { return &TypeFloat{typeBase{name, nil}} } func SpawnBytes(name TypeName) *TypeBytes { return &TypeBytes{typeBase{name, nil}} } func SpawnLink(name TypeName) *TypeLink { return &TypeLink{typeBase{name, nil}, "", false} } func SpawnLinkReference(name TypeName, pointsTo TypeName) *TypeLink { return &TypeLink{typeBase{name, nil}, pointsTo, true} } func SpawnList(name TypeName, valueType TypeName, nullable bool) *TypeList { return &TypeList{typeBase{name, nil}, false, valueType, nullable} } func SpawnMap(name TypeName, keyType TypeName, valueType TypeName, nullable bool) *TypeMap { return &TypeMap{typeBase{name, nil}, false, keyType, valueType, nullable} } func SpawnAny(name TypeName) *TypeAny { return &TypeAny{typeBase{name, nil}} } func SpawnStruct(name TypeName, fields []StructField, repr StructRepresentation) *TypeStruct { v := &TypeStruct{ typeBase{name, nil}, fields, make(map[string]StructField, len(fields)), repr, } for i := range fields { fields[i].parent = v v.fieldsMap[fields[i].name] = fields[i] } switch repr.(type) { case StructRepresentation_Stringjoin: for _, f := range fields { if f.IsMaybe() { panic("neither nullable nor optional is supported on struct stringjoin representation") } } case nil: v.representation = SpawnStructRepresentationMap(nil) } return v } func SpawnStructField(name string, typ TypeName, optional bool, nullable bool) StructField { return StructField{nil /*populated later*/, name, typ, optional, nullable} } func SpawnStructRepresentationMap(renames map[string]string) StructRepresentation_Map { return StructRepresentation_Map{renames, nil} } func SpawnStructRepresentationMap2(renames map[string]string, implicits map[string]ImplicitValue) StructRepresentation_Map { return StructRepresentation_Map{renames, implicits} } func SpawnStructRepresentationTuple() StructRepresentation_Tuple { return StructRepresentation_Tuple{} } func SpawnStructRepresentationListPairs() StructRepresentation_ListPairs { return StructRepresentation_ListPairs{} } func SpawnStructRepresentationStringjoin(delim string) StructRepresentation_Stringjoin { return StructRepresentation_Stringjoin{delim} } func SpawnUnion(name TypeName, members []TypeName, repr UnionRepresentation) *TypeUnion { return &TypeUnion{typeBase{name, nil}, members, repr} } func SpawnUnionRepresentationKeyed(table map[string]TypeName) UnionRepresentation_Keyed { return UnionRepresentation_Keyed{table} } func SpawnUnionRepresentationKinded(table map[datamodel.Kind]TypeName) UnionRepresentation_Kinded { return UnionRepresentation_Kinded{table} } func SpawnUnionRepresentationStringprefix(delim string, table map[string]TypeName) UnionRepresentation_Stringprefix { return UnionRepresentation_Stringprefix{delim, table} } func SpawnUnionRepresentationInline(discriminantKey string, table map[string]TypeName) UnionRepresentation_Inline { return UnionRepresentation_Inline{discriminantKey, table} } func SpawnEnum(name TypeName, members []string, repr EnumRepresentation) *TypeEnum { return &TypeEnum{typeBase{name, nil}, members, repr} } // Utility function adding default basic types to schema type system func SpawnDefaultBasicTypes(ts *TypeSystem) { ts.Accumulate(SpawnBool("Bool")) ts.Accumulate(SpawnInt("Int")) ts.Accumulate(SpawnFloat("Float")) ts.Accumulate(SpawnString("String")) ts.Accumulate(SpawnBytes("Bytes")) ts.Accumulate(SpawnAny("Any")) ts.Accumulate(SpawnMap("Map", "String", "Any", false)) ts.Accumulate(SpawnList("List", "Any", false)) // Should be &Any, really. ts.Accumulate(SpawnLink("Link")) // TODO: schema package lacks support? // ts.Accumulate(schema.SpawnUnit("Null", NullRepr)) } // Clone creates a copy of a type that is not in the original's universe (so it can be used elsewhere safely) func Clone(typ Type) Type { switch kindedType := typ.(type) { case *TypeBool: return SpawnBool(kindedType.Name()) case *TypeString: return SpawnString(kindedType.Name()) case *TypeBytes: return SpawnBytes(kindedType.Name()) case *TypeInt: return SpawnInt(kindedType.Name()) case *TypeFloat: return SpawnFloat(kindedType.Name()) case *TypeAny: return SpawnAny(kindedType.Name()) case *TypeMap: return SpawnMap(kindedType.Name(), kindedType.KeyType().Name(), kindedType.ValueType().Name(), kindedType.ValueIsNullable()) case *TypeList: return SpawnList(kindedType.Name(), kindedType.ValueType().Name(), kindedType.ValueIsNullable()) case *TypeLink: if kindedType.HasReferencedType() { return SpawnLinkReference(kindedType.Name(), kindedType.ReferencedType().Name()) } else { return SpawnLink(kindedType.Name()) } case *TypeUnion: members := kindedType.Members() memberNames := make([]TypeName, 0, len(members)) for _, member := range members { memberNames = append(memberNames, member.Name()) } return SpawnUnion(kindedType.Name(), memberNames, kindedType.RepresentationStrategy()) case *TypeStruct: oldFields := kindedType.Fields() newFields := make([]StructField, 0, len(oldFields)) for _, oldField := range oldFields { newFields = append(newFields, SpawnStructField(oldField.Name(), oldField.Type().Name(), oldField.IsOptional(), oldField.IsNullable())) } return SpawnStruct(kindedType.Name(), newFields, kindedType.RepresentationStrategy()) case *TypeEnum: return SpawnEnum(kindedType.Name(), kindedType.Members(), kindedType.RepresentationStrategy()) default: panic("unexpected type, don't know how to clone") } } func MergeTypeSystem(target *TypeSystem, source *TypeSystem, ignoreDups bool) { for _, name := range source.Names() { typ := Clone(source.TypeByName(name)) if ignoreDups { accumulateWithRecovery(target, typ) } else { target.Accumulate(typ) } } } func accumulateWithRecovery(ts *TypeSystem, typ Type) { defer func() { _ = recover() }() ts.Accumulate(typ) } // The methods relating to TypeSystem are also mutation-heavy and placeholdery. func (ts *TypeSystem) Init() { ts.namedTypes = make(map[TypeName]Type) } func (ts *TypeSystem) Accumulate(typ Type) { typ._Type(ts) name := typ.Name() if _, ok := ts.namedTypes[name]; ok { panic(fmt.Sprintf("duplicate type name: %s", name)) } ts.namedTypes[name] = typ ts.names = append(ts.names, name) } func (ts TypeSystem) GetTypes() map[TypeName]Type { return ts.namedTypes } func (ts TypeSystem) TypeByName(n string) Type { return ts.namedTypes[n] } func (ts TypeSystem) Names() []TypeName { return ts.names } // ValidateGraph checks that all type names referenced are defined. // // It does not do any other validations of individual type's sensibleness // (that should've happened when they were created // (although also note many of those validates are NYI, // and are roadmapped for after we research self-hosting)). func (ts TypeSystem) ValidateGraph() []error { var ee []error for tn, t := range ts.namedTypes { switch t2 := t.(type) { case *TypeBool, *TypeInt, *TypeFloat, *TypeString, *TypeBytes, *TypeEnum: continue // nothing to check: these are leaf nodes and refer to no other types. case *TypeLink: if !t2.hasReferencedType { continue } if _, ok := ts.namedTypes[t2.referencedType]; !ok { ee = append(ee, fmt.Errorf("type %s refers to missing type %s (as link reference type)", tn, t2.referencedType)) } case *TypeStruct: for _, f := range t2.fields { if _, ok := ts.namedTypes[f.typ]; !ok { ee = append(ee, fmt.Errorf("type %s refers to missing type %s (in field %q)", tn, f.typ, f.name)) } } case *TypeMap: if _, ok := ts.namedTypes[t2.keyType]; !ok { ee = append(ee, fmt.Errorf("type %s refers to missing type %s (as key type)", tn, t2.keyType)) } if _, ok := ts.namedTypes[t2.valueType]; !ok { ee = append(ee, fmt.Errorf("type %s refers to missing type %s (as value type)", tn, t2.valueType)) } case *TypeList: if _, ok := ts.namedTypes[t2.valueType]; !ok { ee = append(ee, fmt.Errorf("type %s refers to missing type %s (as value type)", tn, t2.valueType)) } case *TypeUnion: for _, mn := range t2.members { if _, ok := ts.namedTypes[mn]; !ok { ee = append(ee, fmt.Errorf("type %s refers to missing type %s (as a member)", tn, mn)) } } } } return ee } ================================================ FILE: schema/type.go ================================================ package schema import ( "github.com/ipld/go-ipld-prime/datamodel" ) type TypeName = string // typesystem.Type is an union interface; each of the `Type*` concrete types // in this package are one of its members. // // Specifically, // // TypeBool // TypeString // TypeBytes // TypeInt // TypeFloat // TypeMap // TypeList // TypeLink // TypeUnion // TypeStruct // TypeEnum // // are all of the kinds of Type. // // This is a closed union; you can switch upon the above members without // including a default case. The membership is closed by the unexported // '_Type' method; you may use the BurntSushi/go-sumtype tool to check // your switches for completeness. // // Many interesting properties of each Type are only defined for that specific // type, so it's typical to use a type switch to handle each type of Type. // (Your humble author is truly sorry for the word-mash that results from // attempting to describe the types that describe the typesystem.Type.) // // For example, to inspect the kind of fields in a struct: you might // cast a `Type` interface into `TypeStruct`, and then the `Fields()` on // that `TypeStruct` can be inspected. (`Fields()` isn't defined for any // other kind of Type.) type Type interface { // Unexported marker method to force the union closed. // Also used to set the internal pointer back to the universe its part of. _Type(*TypeSystem) // Returns a pointer to the TypeSystem this Type is a member of. TypeSystem() *TypeSystem // Returns the string name of the Type. This name is unique within the // universe this type is a member of, *unless* this type is Anonymous, // in which case a string describing the type will still be returned, but // that string will not be required to be unique. Name() TypeName // Returns the TypeKind of this Type. // // The returned value is a 1:1 association with which of the concrete // "schema.Type*" structs this interface can be cast to. // // Note that a schema.TypeKind is a different enum than datamodel.Kind; // and furthermore, there's no strict relationship between them. // schema.TypedNode values can be described by *two* distinct Kinds: // one which describes how the Node itself will act, // and another which describes how the Node presents for serialization. // For some combinations of Type and representation strategy, one or both // of the Kinds can be determined statically; but not always: // it can sometimes be necessary to inspect the value quite concretely // (e.g., `schema.TypedNode{}.Representation().Kind()`) in order to find // out exactly how a node will be serialized! This is because some types // can vary in representation kind based on their value (specifically, // kinded-representation unions have this property). TypeKind() TypeKind // RepresentationBehavior returns a description of how the representation // of this type will behave in terms of the IPLD Data Model. // This property varies based on the representation strategy of a type. // // In one case, the representation behavior cannot be known statically, // and varies based on the data: kinded unions have this trait. // // This property is used by kinded unions, which require that their members // all have distinct representation behavior. // (It follows that a kinded union cannot have another kinded union as a member.) // // You may also be interested in a related property that might have been called "TypeBehavior". // However, this method doesn't exist, because it's a deterministic property of `TypeKind()`! // You can use `TypeKind.ActsLike()` to get type-level behavioral information. RepresentationBehavior() datamodel.Kind } var ( _ Type = &TypeBool{} _ Type = &TypeString{} _ Type = &TypeBytes{} _ Type = &TypeInt{} _ Type = &TypeFloat{} _ Type = &TypeAny{} _ Type = &TypeMap{} _ Type = &TypeList{} _ Type = &TypeLink{} _ Type = &TypeUnion{} _ Type = &TypeStruct{} _ Type = &TypeEnum{} ) type typeBase struct { name TypeName universe *TypeSystem } type TypeBool struct { typeBase } type TypeString struct { typeBase } type TypeBytes struct { typeBase } type TypeInt struct { typeBase } type TypeFloat struct { typeBase } type TypeAny struct { typeBase } type TypeMap struct { typeBase anonymous bool keyType TypeName // must be Kind==string (e.g. Type==String|Enum). valueType TypeName valueNullable bool } type TypeList struct { typeBase anonymous bool valueType TypeName valueNullable bool } type TypeLink struct { typeBase referencedType TypeName hasReferencedType bool // ...? } type TypeUnion struct { typeBase // Members are listed in the order they appear in the schema. // To find the discriminant info, you must look inside the representation; they all contain a 'table' of some kind in which the member types are the values. // Note that multiple appearances of the same type as distinct members of the union is not possible. // While we could do this... A: that's... odd, and nearly never called for; B: not possible with kinded mode; C: imagine the golang-native type switch! it's impossible. // We rely on this clarity in many ways: most visibly, the type-level Node implementation for a union always uses the type names as if they were map keys! This behavior is consistent for all union representations. members []TypeName representation UnionRepresentation } type UnionRepresentation interface{ _UnionRepresentation() } func (UnionRepresentation_Keyed) _UnionRepresentation() {} func (UnionRepresentation_Kinded) _UnionRepresentation() {} func (UnionRepresentation_Envelope) _UnionRepresentation() {} func (UnionRepresentation_Inline) _UnionRepresentation() {} func (UnionRepresentation_Stringprefix) _UnionRepresentation() {} // A bunch of these tables in union representation might be easier to use if flipped; // we almost always index into them by type (since that's what we have an ordered list of); // and they're unique in both directions, so it's equally valid either way. // The order they're currently written in matches the serial form in the schema AST. type UnionRepresentation_Keyed struct { table map[string]TypeName // key is user-defined freetext } type UnionRepresentation_Kinded struct { table map[datamodel.Kind]TypeName } //lint:ignore U1000 implementation TODO type UnionRepresentation_Envelope struct { discriminantKey string contentKey string table map[string]TypeName // key is user-defined freetext } //lint:ignore U1000 implementation TODO type UnionRepresentation_Inline struct { discriminantKey string table map[string]TypeName // key is user-defined freetext } type UnionRepresentation_Stringprefix struct { delim string table map[string]TypeName // key is user-defined freetext } type TypeStruct struct { typeBase // n.b. `Fields` is an (order-preserving!) map in the schema-schema; // but it's a list here, with the keys denormalized into the value, // because that's typically how we use it. fields []StructField fieldsMap map[string]StructField // same content, indexed for lookup. representation StructRepresentation } type StructField struct { parent *TypeStruct name string typ TypeName optional bool nullable bool } type StructRepresentation interface{ _StructRepresentation() } func (StructRepresentation_Map) _StructRepresentation() {} func (StructRepresentation_Tuple) _StructRepresentation() {} func (StructRepresentation_ListPairs) _StructRepresentation() {} func (StructRepresentation_StringPairs) _StructRepresentation() {} func (StructRepresentation_Stringjoin) _StructRepresentation() {} type StructRepresentation_Map struct { renames map[string]string implicits map[string]ImplicitValue } type StructRepresentation_Tuple struct{} type StructRepresentation_ListPairs struct{} //lint:ignore U1000 implementation TODO type StructRepresentation_StringPairs struct{ sep1, sep2 string } type StructRepresentation_Stringjoin struct{ sep string } type TypeEnum struct { typeBase members []string representation EnumRepresentation } type EnumRepresentation interface{ _EnumRepresentation() } func (EnumRepresentation_String) _EnumRepresentation() {} func (EnumRepresentation_Int) _EnumRepresentation() {} type EnumRepresentation_String map[string]string type EnumRepresentation_Int map[string]int // ImplicitValue is an sum type holding values that are implicits. // It's not an 'Any' value because it can't be recursive // (or to be slightly more specific, it can be one of the recursive kinds, // but if so, only its empty value is valid here). type ImplicitValue interface{ _ImplicitValue() } func (ImplicitValue_EmptyList) _ImplicitValue() {} func (ImplicitValue_EmptyMap) _ImplicitValue() {} func (ImplicitValue_String) _ImplicitValue() {} func (ImplicitValue_Int) _ImplicitValue() {} func (ImplicitValue_Bool) _ImplicitValue() {} type ImplicitValue_EmptyList struct{} type ImplicitValue_EmptyMap struct{} type ImplicitValue_String string type ImplicitValue_Int int type ImplicitValue_Bool bool ================================================ FILE: schema/typeMethods.go ================================================ package schema import ( "github.com/ipld/go-ipld-prime/datamodel" ) /* cookie-cutter standard interface stuff */ func (t *typeBase) _Type(ts *TypeSystem) { t.universe = ts } func (t typeBase) TypeSystem() *TypeSystem { return t.universe } func (t typeBase) Name() TypeName { return t.name } func (TypeBool) TypeKind() TypeKind { return TypeKind_Bool } func (TypeString) TypeKind() TypeKind { return TypeKind_String } func (TypeBytes) TypeKind() TypeKind { return TypeKind_Bytes } func (TypeInt) TypeKind() TypeKind { return TypeKind_Int } func (TypeFloat) TypeKind() TypeKind { return TypeKind_Float } func (TypeAny) TypeKind() TypeKind { return TypeKind_Any } func (TypeMap) TypeKind() TypeKind { return TypeKind_Map } func (TypeList) TypeKind() TypeKind { return TypeKind_List } func (TypeLink) TypeKind() TypeKind { return TypeKind_Link } func (TypeUnion) TypeKind() TypeKind { return TypeKind_Union } func (TypeStruct) TypeKind() TypeKind { return TypeKind_Struct } func (TypeEnum) TypeKind() TypeKind { return TypeKind_Enum } func (TypeBool) RepresentationBehavior() datamodel.Kind { return datamodel.Kind_Bool } func (TypeString) RepresentationBehavior() datamodel.Kind { return datamodel.Kind_String } func (TypeBytes) RepresentationBehavior() datamodel.Kind { return datamodel.Kind_Bytes } func (TypeInt) RepresentationBehavior() datamodel.Kind { return datamodel.Kind_Int } func (TypeFloat) RepresentationBehavior() datamodel.Kind { return datamodel.Kind_Float } func (TypeMap) RepresentationBehavior() datamodel.Kind { return datamodel.Kind_Map } func (TypeList) RepresentationBehavior() datamodel.Kind { return datamodel.Kind_List } func (TypeLink) RepresentationBehavior() datamodel.Kind { return datamodel.Kind_Link } func (t TypeUnion) RepresentationBehavior() datamodel.Kind { switch t.representation.(type) { case UnionRepresentation_Keyed: return datamodel.Kind_Map case UnionRepresentation_Kinded: return datamodel.Kind_Invalid // you can't know with this one, until you see the value (and thus can its inhabitant's behavior)! case UnionRepresentation_Envelope: return datamodel.Kind_Map case UnionRepresentation_Inline: return datamodel.Kind_Map case UnionRepresentation_Stringprefix: return datamodel.Kind_String default: panic("unreachable") } } func (t TypeStruct) RepresentationBehavior() datamodel.Kind { switch t.representation.(type) { case StructRepresentation_Map: return datamodel.Kind_Map case StructRepresentation_Tuple: return datamodel.Kind_List case StructRepresentation_ListPairs: return datamodel.Kind_List case StructRepresentation_StringPairs: return datamodel.Kind_String case StructRepresentation_Stringjoin: return datamodel.Kind_String default: panic("unreachable") } } func (t TypeEnum) RepresentationBehavior() datamodel.Kind { // TODO: this should have a representation strategy switch too; sometimes that will indicate int representation behavior. return datamodel.Kind_String } func (t TypeAny) RepresentationBehavior() datamodel.Kind { return datamodel.Kind_Invalid // TODO: what can we possibly do here? } /* interesting methods per Type type */ // beware: many of these methods will change when we successfully bootstrap self-hosting. // // The current methods return reified Type objects; in the future, there might be less of that. // Returning reified Type objects requires bouncing lookups through the typesystem map; // this is unavoidable because we need to handle cycles in definitions. // However, the extra (and cyclic) pointers that requires won't necessarily jive well if // we remake the Type types to have close resemblances to the Data Model tree data.) // // It's also unfortunate that some of the current methods collide in name with // the names of the Data Model fields. We might reshuffling things to reduce this. // // At any rate, all of these changes will come as a sweep once we // get a self-hosting gen of the schema-schema, not before // (the effort of updating template references is substantial). // IsAnonymous is returns true if the type was unnamed. Unnamed types will // claim to have a Name property like `{Foo:Bar}`, and this is not guaranteed // to be a unique string for all types in the universe. func (t TypeMap) IsAnonymous() bool { return t.anonymous } // KeyType returns the Type of the map keys. // // Note that map keys will must always be some type which is representable as a // string in the IPLD Data Model (e.g. either TypeString or TypeEnum). func (t TypeMap) KeyType() Type { return t.universe.namedTypes[t.keyType] } // ValueType returns the Type of the map values. func (t TypeMap) ValueType() Type { return t.universe.namedTypes[t.valueType] } // ValueIsNullable returns a bool describing if the map values are permitted // to be null. func (t TypeMap) ValueIsNullable() bool { return t.valueNullable } // IsAnonymous is returns true if the type was unnamed. Unnamed types will // claim to have a Name property like `[Foo]`, and this is not guaranteed // to be a unique string for all types in the universe. func (t TypeList) IsAnonymous() bool { return t.anonymous } // ValueType returns to the Type of the list values. func (t TypeList) ValueType() Type { return t.universe.namedTypes[t.valueType] } // ValueIsNullable returns a bool describing if the list values are permitted // to be null. func (t TypeList) ValueIsNullable() bool { return t.valueNullable } // Members returns the list of all types that are possible inhabitants of this union. func (t TypeUnion) Members() []Type { a := make([]Type, len(t.members)) for i := range t.members { a[i] = t.universe.namedTypes[t.members[i]] } return a } func (t TypeUnion) RepresentationStrategy() UnionRepresentation { return t.representation } func (r UnionRepresentation_Keyed) GetDiscriminant(t Type) string { for d, t2 := range r.table { if t2 == t.Name() { return d } } panic("that type isn't a member of this union") } func (r UnionRepresentation_Stringprefix) GetDelim() string { return r.delim } func (r UnionRepresentation_Stringprefix) GetDiscriminant(t Type) string { for d, t2 := range r.table { if t2 == t.Name() { return d } } panic("that type isn't a member of this union") } // GetMember returns type info for the member matching the kind argument, // or may return nil if that kind is not mapped to a member of this union. func (r UnionRepresentation_Kinded) GetMember(k datamodel.Kind) TypeName { return r.table[k] } // Fields returns a slice of descriptions of the object's fields. func (t TypeStruct) Fields() []StructField { return t.fields } // Field looks up a StructField by name, or returns nil if no such field. func (t TypeStruct) Field(name string) *StructField { if v, ok := t.fieldsMap[name]; ok { return &v } return nil } // Parent returns the type information that this field describes a part of. // // While in many cases, you may know the parent already from context, // there may still be situations where want to pass around a field and // not need to continue passing down the parent type with it; this method // helps your code be less redundant in such a situation. // (You'll find this useful for looking up any rename directives, for example, // when holding onto a field, since that requires looking up information from // the representation strategy, which is a property of the type as a whole.) func (f StructField) Parent() *TypeStruct { return f.parent } // Name returns the string name of this field. The name is the string that // will be used as a map key if the structure this field is a member of is // serialized as a map representation. func (f StructField) Name() string { return f.name } // Type returns the Type of this field's value. Note the field may // also be unset if it is either Optional or Nullable. func (f StructField) Type() Type { return f.parent.universe.namedTypes[f.typ] } // IsOptional returns true if the field is allowed to be absent from the object. // If IsOptional is false, the field may be absent from the serial representation // of the object entirely. // // Note being optional is different than saying the value is permitted to be null! // A field may be both nullable and optional simultaneously, or either, or neither. func (f StructField) IsOptional() bool { return f.optional } // IsNullable returns true if the field value is allowed to be null. // // If is Nullable is false, note that it's still possible that the field value // will be absent if the field is Optional! Being nullable is unrelated to // whether the field's presence is optional as a whole. // // Note that a field may be both nullable and optional simultaneously, // or either, or neither. func (f StructField) IsNullable() bool { return f.nullable } // IsMaybe returns true if the field value is allowed to be either null or absent. // // This is a simple "or" of the two properties, // but this method is a shorthand that turns out useful often. func (f StructField) IsMaybe() bool { return f.nullable || f.optional } func (t TypeStruct) RepresentationStrategy() StructRepresentation { return t.representation } func (r StructRepresentation_Map) GetFieldKey(field StructField) string { if n, ok := r.renames[field.name]; ok { return n } return field.name } func (r StructRepresentation_Map) FieldHasRename(field StructField) bool { _, ok := r.renames[field.name] return ok } // FieldImplicit returns the 'implicit' value for a field, or nil, if there isn't one. // // Because this returns the golang ImplicitValue type, which is an interface, // golang type switching is needed to distinguish what it holds. // (In other words, be warned that this function is not very friendly to use from templating engines.) func (r StructRepresentation_Map) FieldImplicit(field StructField) ImplicitValue { if r.implicits == nil { return nil } return r.implicits[field.name] } func (r StructRepresentation_Stringjoin) GetDelim() string { return r.sep } // Members returns a slice the strings which are valid inhabitants of this enum. func (t TypeEnum) Members() []string { return t.members } func (t TypeEnum) RepresentationStrategy() EnumRepresentation { return t.representation } // Links can keep a referenced type, which is a hint only about the data on the // other side of the link, no something that can be explicitly validated without // loading the link // HasReferencedType returns true if the link has a hint about the type it references // false if it's generic func (t TypeLink) HasReferencedType() bool { return t.hasReferencedType } // ReferencedType returns the type hint for the node on the other side of the link func (t TypeLink) ReferencedType() Type { return t.universe.namedTypes[t.referencedType] } ================================================ FILE: schema/typedNode.go ================================================ package schema import ( "github.com/ipld/go-ipld-prime/datamodel" ) // schema.TypedNode is a superset of the datamodel.Node interface, and has additional behaviors. // // A schema.TypedNode can be inspected for its schema.Type and schema.TypeKind, // which conveys much more and richer information than the Data Model layer // datamodel.Kind. // // There are many different implementations of schema.TypedNode. // One implementation can wrap any other existing datamodel.Node (i.e., it's zero-copy) // and promises that it has *already* been validated to match the typesystem.Type; // another implementation similarly wraps any other existing datamodel.Node, but // defers to the typesystem validation checking to fields that are accessed; // and when using code generation tools, all of the generated native Golang // types produced by the codegen will each individually implement schema.TypedNode. // // Typed nodes sometimes have slightly different behaviors than plain nodes: // For example, when looking up fields on a typed node that's a struct, // the error returned for a lookup with a key that's not a field name will // be ErrNoSuchField (instead of ErrNotExists). // These behaviors apply to the schema.TypedNode only and not their representations; // continuing the example, the .Representation().LookupByString() method on // that same node for the same key as plain `.LookupByString()` will still // return ErrNotExists, because the representation isn't a schema.TypedNode! type TypedNode interface { // schema.TypedNode acts just like a regular Node for almost all purposes; // which datamodel.Kind it acts as is determined by the TypeKind. // (Note that the representation strategy of the type does *not* affect // the Kind of schema.TypedNode -- rather, the representation strategy // affects the `.Representation().Kind()`.) // // For example: if the `.Type().TypeKind()` of this node is "struct", // it will act like Kind() == "map" // (even if Type().(Struct).ReprStrategy() is "tuple"). datamodel.Node // Type returns a reference to the reified schema.Type value. Type() Type // Representation returns a datamodel.Node which sees the data in this node // in its representation form. // // For example: if the `.Type().TypeKind()` of this node is "struct", // `.Representation().TypeKind()` may vary based on its representation strategy: // if the representation strategy is "map", then it will be Kind=="map"; // if the streatgy is "tuple", then it will be Kind=="list". Representation() datamodel.Node } // schema.TypedLinkNode is a superset of the schema.TypedNode interface, and has one additional behavior. // // A schema.TypedLinkNode contains a hint for the appropriate node builder to use for loading data // on the other side of the link contained within the node, so that it can be assembled // into a node representation and validated against the schema as quickly as possible // // So, for example, if you wanted to support loading the other side of a link // with a code-gen'd node builder while utilizing the automatic loading facilities // of the traversal package, you could write a LinkNodeBuilderChooser as follows: // // func LinkNodeBuilderChooser(lnk datamodel.Link, lnkCtx linking.LinkContext) datamodel.NodePrototype { // if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { // return tlnkNd.LinkTargetNodePrototype() // } // return basicnode.Prototype.Any // } type TypedLinkNode interface { LinkTargetNodePrototype() datamodel.NodePrototype } // TypedPrototype is a superset of the datamodel.Nodeprototype interface, and has // additional behaviors, much like TypedNode for datamodel.Node. type TypedPrototype interface { datamodel.NodePrototype // Type returns a reference to the reified schema.Type value. Type() Type // Representation returns a datamodel.NodePrototype for the representation // form of the prototype. Representation() datamodel.NodePrototype } ================================================ FILE: schema/typesystem.go ================================================ package schema type TypeSystem struct { // namedTypes is the set of all named types in this universe. // The map's key is the value's Name() property and must be unique. // // The IsAnonymous property is false for all values in this map that // support the IsAnonymous property. // // Each Type in the universe may only refer to other types in their // definition if those type are either A) in this namedTypes map, // or B) are IsAnonymous==true. namedTypes map[TypeName]Type // names are the same set of names stored in namedTypes, // but in insertion order. names []TypeName } ================================================ FILE: schema/validate.go ================================================ package schema /* Okay, so. There are several fun considerations for a "validate" method. --- There's two radically different approaches to "validate"/"reify": - Option 1: Look at the schema.Type info and check if a data node seems to match it -- recursing on the type info. - Option 2: Use the schema.Type{}.RepresentationNodeBuilder() to feed data into it -- recursing on what the nodebuilder already expresses. (Option 2 also need to take a `memStorage ipld.NodeBuilder` param, btw, for handling all the cases where we *aren't* doing codegen.) Option 1 provides a little more opportunity for returning multiple errors. Option 2 will generally have a hard time with that (nodebuilers are not necessarily in a valid state after their first error encounter). As a result of having these two options at all, we may indeed end up with at least two very different functions -- despite seeming to do similar things, their interior will radically diverge. --- We may also need to consider distinct reification paths: we may want one that returns a new node tree which is eagerly converted to schema.TypedNode recursively; and another that returns a lazyNode which wraps things with their typed node constraints only as they're requested. (Note that the latter would have interesting implications for any code which has expectations about pointer equality consistency.) --- A further fun issue which needs consideration: well, I'll just save a snip of prospective docs I wrote while trying to iterate on these functions: // Note that using Validate on a node that's already a schema.TypedNode is likely // to be nonsensical. In many schemas, the schema.TypedNode tree is actually a // different depth than its representational tree (e.g. unions can cause this), ... and that's ... that's a fairly sizable issue that needs resolving. There's a couple of different ways to handle some of the behaviors around unions, and some of them make the tradeoff described above, and I'm really unsure if all the implications have been sussed out yet. We should defer writing code that depends on this issue until gathering some more info. --- One more note: about returning multiple errors from a Validate function: there's an upper bound of the utility of the thing. Going farther than the first parse error is nice, but it will still hit limits: for example, upon encountering a union and failing to match it, we can't generally produce further errors from anywhere deeper in the tree without them being combinatorial "if previous juncture X was type Y, then..." nonsense. (This applies to all recursive kinds to some degree, but it's especially rough with unions. For most of the others, it's flatly a missing field, or an excessive field, or a leaf error; with unions it can be hard to tell.) --- And finally: both "Validate" and "Reify" methods might actually belong in the schema.TypedNode package -- if they make *any* reference to `schema.TypedNode`, then they have no choice (otherwise, cyclic imports would occur). If we make a "Validate" that works purely on the schema.Type info, and returns *only* errors: only then we can have it in the schema package. */ ================================================ FILE: schema.go ================================================ package ipld import ( "bytes" "io" "os" "github.com/ipld/go-ipld-prime/schema" schemadmt "github.com/ipld/go-ipld-prime/schema/dmt" schemadsl "github.com/ipld/go-ipld-prime/schema/dsl" ) // LoadSchemaBytes is a shortcut for LoadSchema for the common case where // the schema is available as a buffer or a string, such as via go:embed. func LoadSchemaBytes(src []byte) (*schema.TypeSystem, error) { return LoadSchema("", bytes.NewReader(src)) } // LoadSchemaBytes is a shortcut for LoadSchema for the common case where // the schema is a file on disk. func LoadSchemaFile(path string) (*schema.TypeSystem, error) { f, err := os.Open(path) if err != nil { return nil, err } defer f.Close() return LoadSchema(path, f) } // LoadSchema parses an IPLD Schema in its DSL form // and compiles its types into a standalone TypeSystem. func LoadSchema(name string, r io.Reader) (*schema.TypeSystem, error) { sch, err := schemadsl.Parse(name, r) if err != nil { return nil, err } ts := new(schema.TypeSystem) ts.Init() if err := schemadmt.Compile(ts, sch); err != nil { return nil, err } return ts, nil } ================================================ FILE: storage/README_adapters.md ================================================ Storage Adapters ================ The go-ipld-prime storage APIs were introduced in the v0.14.x ranges of go-ipld-prime, which happened in fall 2021. There are many other pieces of code in the IPLD (and even more so, the IPFS) ecosystem which predate this, and have interfaces that are very _similar_, but not quite exactly the same. In order to keep using that code, we've built a series of adapters. You can see these in packages beneath this one: - `go-ipld-prime/storage/bsadapter` is an adapter to `github.com/ipfs/go-ipfs-blockstore`. - `go-ipld-prime/storage/dsadapter` is an adapter to `github.com/ipfs/go-datastore`. - `go-ipld-prime/storage/bsrvadapter` is an adapter to `github.com/ipfs/go-blockservice`. Note that there are also other packages which implement the go-ipld-prime storage APIs, but are not considered "adapters" -- these just implement the storage APIs directly: - `go-ipld-prime/storage/memstore` is a simple in-memory storage system. - `go-ipld-prime/storage/fsstore` is a simple filesystem-backed storage system (comparable to, and compatible with [flatfs](https://pkg.go.dev/github.com/ipfs/go-ds-flatfs), if you're familiar with that -- but higher efficiency). Finally, note that there are some shared benchmarks across all this: - check out `go-ipld-prime/storage/benchmarks`! Why structured like this? ------------------------- ### Why is there adapter code at all? The `go-ipld-prime/storage` interfaces are a newer generation. A new generation of APIs was desirable because it unifies the old APIs, and also because we were able to improves and update several things in the process. (You can see some of the list of improvements in https://github.com/ipld/go-ipld-prime/pull/265, where these APIs were first introduced.) The new generation of APIs avoids several types present in the old APIs which forced otherwise-avoidable allocations. (See notes later in this document about "which adapter should I use" for more on that.) Finally, the new generation of APIs is carefully designed to support minimal implementations, by carefully avoiding use of non-standard-library types in key API definitions, and by keeping most advanced features behind a standardized convention of feature detection. Because the newer generation of APIs are not exactly the same as the multiple older APIs we're unifying and updating, some amount of adapter code is necessary. (Fortunately, it's not much! But it's not "none", either.) ### Why have this code in a shared place? The glue code to connect `go-datastore` and the other older APIs to the new `go-ipld-prime/storage` APIs is fairly minimal... but there's also no reason for anyone to write it twice, so we want to put it somewhere easy to share. ### Why do the adapters have their own go modules? A separate module is used because it's important that go-ipld-prime can be used without forming a dependency on `go-datastore` (or the other relevant modules, per adapter). We want this so that there's a reasonable deprecation pathway -- it must be possible to write new code that doesn't take on transitive dependencies to old code. (As a bonus, looking at the module dependency graphs makes an interestingly clear statement about why minimal APIs that don't force transitive dependencies are a good idea!) ### Why is this code all together in this repo? We put these separate modules in the same git repo as `go-ipld-prime`... because we can. Technically, neither the storage adapter modules nor the `go-ipld-prime` module depend on each other -- they just have interfaces that are aligned with each other -- so it's very easy to hold them as separate go modules in the same repo, even though that can otherwise sometimes be tricky. You may want to make a point of pulling updated versions of the storage adapters that you use when pulling updates to go-ipld-prime, though. ### Could we put these adapters upstream into the other relevant repos? Certainly! We started with them here because it seemed developmentally lower-friction. That may change; these APIs could move. This code is just interface satisfaction, so even having multiple copies of it is utterly harmless. Which of `dsadapter` vs `bsadapter` vs `bsrvadapter` should I use? ------------------------------------------------------------------ None of them, ideally. A direct implementation of the storage APIs will almost certainly be able to perform better than any of these adapters. (Check out the `fsstore` package, for example.) Failing that: use the adapter matching whatever you've got on hand in your code. There is no correct choice. `dsadapter` suffers avoidable excessive allocs in processing its key type, due to choices in the interior of `github.com/ipfs/go-datastore`. It is also unable to support streaming operation, should you desire it. `bsadapter` and `bsrvadapter` both also suffer overhead due to their key type, because they require a transformation back from the plain binary strings used in the storage API to the concrete go-cid type, which spends some avoidable CPU time (and also, at present, causes avoidable allocs because of some interesting absenses in `go-cid`). Additionally, they suffer avoidable allocs because they wrap the raw binary data in a "block" type, which is an interface, and thus heap-escapes; and we need none of that in the storage APIs, and just return the raw data. They are also unable to support streaming operation, should you desire it. It's best to choose the shortest path and use the adapter to whatever layer you need to get to -- for example, if you really want to use a `go-datastore` implementation, *don't* use `bsadapter` and have it wrap a `go-blockstore` that wraps a `go-datastore` if you can help it: instead, use `dsadapter` and wrap the `go-datastore` without any extra layers of indirection. You should prefer this because most of the notes above about avoidable allocs are true when the legacy interfaces are communicating with each other, as well... so the less you use the internal layering of the legacy interfaces, the better off you'll be. Using a direct implementation of the storage APIs will suffer none of these overheads, and so will always be your best bet if possible. If you have to use one of these adapters, hopefully the performance overheads fall within an acceptable margin. If not: we'll be overjoyed to accept help porting things. ================================================ FILE: storage/api.go ================================================ package storage import ( "context" "io" ) // --- basics ---> // Storage is one of the base interfaces in the storage APIs. // This type is rarely seen by itself alone (and never useful to implement alone), // but is included in both ReadableStorage and WritableStorage. // Because it's included in both the of the other two useful base interfaces, // you can define functions that work on either one of them // by using this type to describe your function's parameters. // // Library functions that work with storage systems should take either // ReadableStorage, or WritableStorage, or Storage, as a parameter, // depending on whether the function deals with the reading of data, // or the writing of data, or may be found on either, respectively. // // An implementation of Storage may also support many other methods. // At the very least, it should also support one of either ReadableStorage or WritableStorage. // It may support even more interfaces beyond that for additional feature detection. // See the package-wide docs for more discussion of this design. // // The Storage interface does not include much of use in itself alone, // because ReadableStorage and WritableStorage are meant to be the most used types in declarations. // However, it does include the Has function, because that function is reasonable to require ubiquitously from all implementations, // and it serves as a reasonable marker to make sure the Storage interface is not trivially satisfied. type Storage interface { Has(ctx context.Context, key string) (bool, error) } // ReadableStorage is one of the base interfaces in the storage APIs; // a storage system should implement at minimum either this, or WritableStorage, // depending on whether it supports reading or writing. // (One type may also implement both.) // // ReadableStorage implementations must at minimum provide // a way to ask the store whether it contains a key, // and a way to ask it to return the value. // // Library functions that work with storage systems should take either // ReadableStorage, or WritableStorage, or Storage, as a parameter, // depending on whether the function deals with the reading of data, // or the writing of data, or may be found on either, respectively. // // An implementation of ReadableStorage may also support many other methods -- // for example, it may additionally match StreamingReadableStorage, or yet more interfaces. // Usually, you should not need to check for this yourself; instead, // you should use the storage package's functions to ask for the desired mode of interaction. // Those functions will will accept any ReadableStorage as an argument, // detect the additional interfaces automatically and use them if present, // or, fall back to synthesizing equivalent behaviors from the basics. // See the package-wide docs for more discussion of this design. type ReadableStorage interface { Storage Get(ctx context.Context, key string) ([]byte, error) } // WritableStorage is one of the base interfaces in the storage APIs; // a storage system should implement at minimum either this, or ReadableStorage, // depending on whether it supports reading or writing. // (One type may also implement both.) // // WritableStorage implementations must at minimum provide // a way to ask the store whether it contains a key, // and a way to put a value into storage indexed by some key. // // Library functions that work with storage systems should take either // ReadableStorage, or WritableStorage, or Storage, as a parameter, // depending on whether the function deals with the reading of data, // or the writing of data, or may be found on either, respectively. // // An implementation of WritableStorage may also support many other methods -- // for example, it may additionally match StreamingWritableStorage, or yet more interfaces. // Usually, you should not need to check for this yourself; instead, // you should use the storage package's functions to ask for the desired mode of interaction. // Those functions will will accept any WritableStorage as an argument, // detect the additional interfaces automatically and use them if present, // or, fall back to synthesizing equivalent behaviors from the basics. // See the package-wide docs for more discussion of this design. type WritableStorage interface { Storage Put(ctx context.Context, key string, content []byte) error } // --- streaming ---> type StreamingReadableStorage interface { GetStream(ctx context.Context, key string) (io.ReadCloser, error) } // StreamingWritableStorage is a feature-detection interface that advertises support for streaming writes. // It is normal for APIs to use WritableStorage in their exported API surface, // and then internally check if that value implements StreamingWritableStorage if they wish to use streaming operations. // // Streaming writes can be preferable to the all-in-one style of writing of WritableStorage.Put, // because with streaming writes, the high water mark for memory usage can be kept lower. // On the other hand, streaming writes can incur slightly higher allocation counts, // which may cause some performance overhead when handling many small writes in sequence. // // The PutStream function returns three parameters: an io.Writer (as you'd expect), another function, and an error. // The function returned is called a "WriteCommitter". // The final error value is as usual: it will contain an error value if the write could not be begun. // ("WriteCommitter" will be referred to as such throughout the docs, but we don't give it a named type -- // unfortunately, this is important, because we don't want to force implementers of storage systems to import this package just for a type name.) // // The WriteCommitter function should be called when you're done writing, // at which time you give it the key you want to commit the data as. // It will close and flush any streams, and commit the data to its final location under this key. // (If the io.Writer is also an io.WriteCloser, it is not necessary to call Close on it, // because using the WriteCommiter will do this for you.) // // Because these storage APIs are meant to work well for content-addressed systems, // the key argument is not provided at the start of the write -- it's provided at the end. // (This gives the opportunity to be computing a hash of the contents as they're written to the stream.) // // As a special case, giving a key of the zero string to the WriteCommiter will // instead close and remove any temp files, and store nothing. // An error may still be returned from the WriteCommitter if there is an error cleaning up // any temporary storage buffers that were created. // // Continuing to write to the io.Writer after calling the WriteCommitter function will result in errors. // Calling the WriteCommitter function more than once will result in errors. type StreamingWritableStorage interface { PutStream(ctx context.Context) (io.Writer, func(key string) error, error) } // --- other specializations ---> // VectorWritableStorage is an API for writing several slices of bytes at once into storage. // It's meant a feature-detection interface; not all storage implementations need to provide this feature. // This kind of API can be useful for maximizing performance in scenarios where // data is already loaded completely into memory, but scattered across several non-contiguous regions. type VectorWritableStorage interface { PutVec(ctx context.Context, key string, blobVec [][]byte) error } // PeekableStorage is a feature-detection interface which a storage implementation can use to advertise // the ability to look at a piece of data, and return it in shared memory. // The PeekableStorage.Peek method is essentially the same as ReadableStorage.Get -- // but by contrast, ReadableStorage is expected to return a safe copy. // PeekableStorage can be used when the caller knows they will not mutate the returned slice. // // An io.Closer is returned along with the byte slice. // The Close method on the Closer must be called when the caller is done with the byte slice; // otherwise, memory leaks may result. // (Implementers of this interface may be expecting to reuse the byte slice after Close is called.) // // Note that Peek does not imply that the caller can use the byte slice freely; // doing so may result in storage corruption or other undefined behavior. type PeekableStorage interface { Peek(ctx context.Context, key string) ([]byte, io.Closer, error) } // the following are all hypothetical additional future interfaces (in varying degress of speculativeness): // FUTURE: an EnumerableStorage API, that lets you list all keys present? // FUTURE: a cleanup API (for getting rid of tmp files that might've been left behind on rough shutdown)? // FUTURE: a sync-forcing API? // FUTURE: a delete API? sure. (just document carefully what its consistency model is -- i.e. basically none.) // (hunch: if you do want some sort of consistency model -- consider offering a whole family of methods that have some sort of generation or sequencing number on them.) // FUTURE: a force-overwrite API? (not useful for a content-address system. but maybe a gesture towards wider reusability is acceptable to have on offer.) // FUTURE: a size estimation API? (unclear if we need to standardize this, but we could. an offer, anyway.) // FUTURE: a GC API? (dubious -- doing it well probably crosses logical domains, and should not be tied down here.) ================================================ FILE: storage/benchmarks/README.md ================================================ benchmarks ========== This is a small module that pulls in a bunch of storage implementations, as well as legacy implementations via the adapter modules, and benchmarks all of them on the same benchmarks. There's no reason to import this code, so the go.mod file uses relative paths shamelessly. (You can create your own benchmarks using the code in `../tests`, which contains most of the engine; this package is just tables of setup.) What variations do the benchmarks exercise? ------------------------------------------ - the various storage implementations! - in some cases: variations of parameters to individual storage implementations. (TODO) - puts and gets. (TODO: currently only puts.) - various distributions of data size. (TODO) - block mode vs streaming mode. (TODO) - end-to-end use via linksystem with small cbor objects. (TODO) - (this measures a lot of things that aren't to do with the storage itself -- but is useful to contextualize things.) Running the benchmarks on variations in hardware and filesystem may also be important! Many of these storage systems use the disk in some way. Why is the module structured like this? ---------------------------------------- Because many of the storage implementations are also their own modules, and we don't want to have the go-ipld-prime module pull in a huge universe of transitive dependencies. See similar discussion in `../README_adapters.md`. It may be worth pulling this out into a new git repo in the future, especially if we want to add more and more implementations to what we benchmark, or develop additional tools for deploying the benchmark on varying hardware, etc. For now, it incubates here. ================================================ FILE: storage/benchmarks/go.mod ================================================ module github.com/ipld/go-ipld-prime/storage/benchmarks go 1.25.7 replace github.com/ipld/go-ipld-prime => ../.. replace github.com/ipld/go-ipld-prime/storage/dsadapter => ../dsadapter require ( github.com/ipfs/go-ds-flatfs v0.6.0 github.com/ipld/go-ipld-prime v0.20.0 github.com/ipld/go-ipld-prime/storage/dsadapter v0.20.0 ) require ( github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 // indirect github.com/google/uuid v1.6.0 // indirect github.com/ipfs/go-datastore v0.9.1 // indirect github.com/ipfs/go-log/v2 v2.9.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/sys v0.43.0 // indirect ) ================================================ FILE: storage/benchmarks/go.sum ================================================ github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 h1:iW0a5ljuFxkLGPNem5Ui+KBjFJzKg4Fv2fnxe4dvzpM= github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5/go.mod h1:Y2QMoi1vgtOIfc+6DhrMOGkLoGzqSV2rKp4Sm+opsyA= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/ipfs/go-datastore v0.9.1 h1:67Po2epre/o0UxrmkzdS9ZTe2GFGODgTd2odx8Wh6Yo= github.com/ipfs/go-datastore v0.9.1/go.mod h1:zi07Nvrpq1bQwSkEnx3bfjz+SQZbdbWyCNvyxMh9pN0= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= github.com/ipfs/go-ds-flatfs v0.6.0 h1:olAEnDNBK1VMoTRZvfzgo90H5kBP4qIZPpYMtNlBBws= github.com/ipfs/go-ds-flatfs v0.6.0/go.mod h1:p8a/YhmAFYyuonxDbvuIANlDCgS69uqVv+iH5f8fAxY= github.com/ipfs/go-log/v2 v2.9.0 h1:l4b06AwVXwldIzbVPZy5z7sKp9lHFTX0KWfTBCtHaOk= github.com/ipfs/go-log/v2 v2.9.0/go.mod h1:UhIYAwMV7Nb4ZmihUxfIRM2Istw/y9cAk3xaK+4Zs2c= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= ================================================ FILE: storage/benchmarks/storageBenchmarks_test.go ================================================ package benchmarks import ( "encoding/base32" "fmt" "os" "testing" flatfs "github.com/ipfs/go-ds-flatfs" "github.com/ipld/go-ipld-prime/storage" "github.com/ipld/go-ipld-prime/storage/dsadapter" "github.com/ipld/go-ipld-prime/storage/fsstore" "github.com/ipld/go-ipld-prime/storage/memstore" "github.com/ipld/go-ipld-prime/storage/tests" ) func BenchmarkPut(b *testing.B) { // - memstore // - dsadapter with flatfs // - bsrvadapter wrapped around that (todo) // - fsstore tt := []struct { storeName string // used in test name storeConstructor func() storage.WritableStorage }{{ storeName: "memstore", storeConstructor: func() storage.WritableStorage { return &memstore.Store{} }, }, { storeName: "dsadapter-flatfs-base32-defaultshard", storeConstructor: func() storage.WritableStorage { shardFn, err := flatfs.ParseShardFunc("/repo/flatfs/shard/v1/next-to-last/2") if err != nil { panic(err) } ds, err := flatfs.CreateOrOpen(".", shardFn, false) if err != nil { panic(err) } return &dsadapter.Adapter{ Wrapped: ds, EscapingFunc: func(raw string) string { return base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString([]byte(raw)) }, } }, }, { storeName: "fsstore-base32-defaultshard", storeConstructor: func() storage.WritableStorage { store := &fsstore.Store{} if err := store.InitDefaults("."); err != nil { panic(err) } return store }, }} for _, ttr := range tt { for _, scale := range []int{ // 1 << 8, // probably too small to be useful; b.N will always be much bigger than this. 1 << 12, 1 << 16, // 1 << 20, // already getting too big to fit the setup phase into the default benchmark time windows when using disk storage. // 1 << 24, } { b.Run(fmt.Sprintf("%s/scale=%d", ttr.storeName, scale), func(b *testing.B) { // Make a tempdir. Change cwd to it. // We'll assume the storage system, if it needs filesystem, can use the cwd. // Using b.TempDir means the cleanup happens handled by the test system (and critically, not on our clock). dir := b.TempDir() retreat, err := os.Getwd() if err != nil { panic(err) } defer os.Chdir(retreat) if err := os.Chdir(dir); err != nil { panic(err) } // Create the store, and put it to work! store := ttr.storeConstructor() gen := tests.NewCounterGen(1000000) // Use a large enough number that any sharding function kicks in (e.g. the b10 string is >=7 chars). tests.BenchPut(b, store, gen, scale) }) } } } ================================================ FILE: storage/bsadapter/README.md ================================================ bsadapter ========= The `bsadapter` package/module is a small piece of glue code to connect the `github.com/ipfs/go-blockstore` package, and packages implementing its interfaces, forward into the `go-ipld-prime/storage` interfaces. Why structured like this? ------------------------- See `../README_adapters.md` for details about why adapter code is needed, why this is in a module, why it's here, etc. Which of `dsadapter` vs `bsadapter` vs `bsrvadapter` should I use? ------------------------------------------------------------------ In short: you should prefer direct implementations of the storage APIs over any of these adapters, if one is available with the features you need. Otherwise, if that's not an option (yet) for some reason, use whichever adapter gets you most directly connected to the code you need. See `../README_adapters.md` for more details and discussion. ================================================ FILE: storage/bsadapter/bsadapter.go ================================================ package bsadapter import ( "context" "fmt" "github.com/ipfs/boxo/blockstore" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" ) // Adapter implements go-ipld-prime/storage.ReadableStorage // and go-ipld-prime/storage.WritableStorage // backed by a go-ipfs-blockstore.Blockstore. // // The go-ipfs-blockstore.Blockstore may internally have other configuration. // We don't interfere with that here; // such configuration should be handled when creating the go-ipfs-blockstore value. // // Note that this system will only work for certain structures of keys -- // this is because the blockstore API works on the level of CIDs. // As long as your key string is the binary form of a CID, it will work correctly. // Other keys are not possible to support with this adapter. // // Contexts given to this system are checked for errors at the beginning of an operation, // but otherwise have no effect, because the Blockstore API doesn't accept context parameters. type Adapter struct { Wrapped blockstore.Blockstore } // Has implements go-ipld-prime/storage.Storage.Has. func (a *Adapter) Has(ctx context.Context, key string) (bool, error) { // Return early if the context is already closed. // This is also the last time we'll check the context, // since the Has method on Blockstore doesn't take them. if ctx.Err() != nil { return false, ctx.Err() } // Do the inverse of cid.KeyString(), // which is how a valid key for this adapter must've been produced. k, err := cidFromBinString(key) if err != nil { return false, err } // Delegate the Has call. return a.Wrapped.Has(ctx, k) } // Get implements go-ipld-prime/storage.ReadableStorage.Get. func (a *Adapter) Get(ctx context.Context, key string) ([]byte, error) { // Return early if the context is already closed. // This is also the last time we'll check the context, // since the Put method on Blockstore doesn't take them. if ctx.Err() != nil { return nil, ctx.Err() } // Do the inverse of cid.KeyString(), // which is how a valid key for this adapter must've been produced. k, err := cidFromBinString(key) if err != nil { return nil, err } // Delegate the Get call. block, err := a.Wrapped.Get(ctx, k) if err != nil { return nil, err } // Unwrap the actual raw data for return. // Discard the rest. (It's a shame there was an alloc for that structure.) return block.RawData(), nil } // Put implements go-ipld-prime/storage.WritableStorage.Put. func (a *Adapter) Put(ctx context.Context, key string, content []byte) error { // Return early if the context is already closed. // This is also the last time we'll check the context, // since the Put method on Blockstore doesn't take them. if ctx.Err() != nil { return ctx.Err() } // Do the inverse of cid.KeyString(), // which is how a valid key for this adapter must've been produced. k, err := cidFromBinString(key) if err != nil { return err } // Create a structure that has the cid and the raw content together. // This is necessary because it's the format demanded by Blockstore. // (Unfortunately, it also provokes an allocation, because it uses interfaces; // but we can't avoid that without changing the code in go-ipfs-blockstore.) // The error is treated as a panic because it's only possible if a global debug var is set, // and is for behavior that is not meant to be part of the contract of the storage APIs. block, err := blocks.NewBlockWithCid(content, k) if err != nil { panic(err) } // Delegate the Put call. return a.Wrapped.Put(ctx, block) } // Do the inverse of cid.KeyString(). // (Unclear why go-cid doesn't offer a function for this itself.) func cidFromBinString(key string) (cid.Cid, error) { l, k, err := cid.CidFromBytes([]byte(key)) if err != nil { return cid.Undef, fmt.Errorf("bsrvadapter: key was not a cid: %w", err) } if l != len(key) { return cid.Undef, fmt.Errorf("bsrvadapter: key was not a cid: had %d bytes leftover", len(key)-l) } return k, nil } ================================================ FILE: storage/bsadapter/go.mod ================================================ module github.com/ipld/go-ipld-prime/storage/bsadapter go 1.25.7 require ( github.com/ipfs/boxo v0.39.0 github.com/ipfs/go-block-format v0.2.3 github.com/ipfs/go-cid v0.6.1 ) require ( github.com/gammazero/chanqueue v1.1.2 // indirect github.com/gammazero/deque v1.2.1 // indirect github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/ipfs/bbloom v0.1.0 // indirect github.com/ipfs/go-cidutil v0.1.1 // indirect github.com/ipfs/go-datastore v0.9.1 // indirect github.com/ipfs/go-dsqueue v0.2.0 // indirect github.com/ipfs/go-ipld-format v0.6.3 // indirect github.com/ipfs/go-log/v2 v2.9.1 // indirect github.com/ipfs/go-metrics-interface v0.3.0 // indirect github.com/ipld/go-ipld-prime v0.23.0 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mr-tron/base58 v1.3.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect github.com/multiformats/go-multibase v0.3.0 // indirect github.com/multiformats/go-multicodec v0.10.0 // indirect github.com/multiformats/go-multihash v0.2.3 // indirect github.com/multiformats/go-varint v0.1.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.1 // indirect golang.org/x/crypto v0.50.0 // indirect golang.org/x/sys v0.43.0 // indirect lukechampine.com/blake3 v1.4.1 // indirect ) ================================================ FILE: storage/bsadapter/go.sum ================================================ github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 h1:5RVFMOWjMyRy8cARdy79nAmgYw3hK/4HUq48LQ6Wwqo= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= github.com/filecoin-project/go-clock v0.1.0 h1:SFbYIM75M8NnFm1yMHhN9Ahy3W5bEZV9gd6MPfXbKVU= github.com/filecoin-project/go-clock v0.1.0/go.mod h1:4uB/O4PvOjlx1VCMdZ9MyDZXRm//gkj1ELEbxfI1AZs= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/gammazero/chanqueue v1.1.2 h1:dZEsxlyANZMyeTRemABqZF8QM9BnE4NBI43Oh3y5fIU= github.com/gammazero/chanqueue v1.1.2/go.mod h1:XDN1X/jjAbmSceNFOQbtKToeSkxtdVdpKu90LiEdBEE= github.com/gammazero/deque v1.2.1 h1:9fnQVFCCZ9/NOc7ccTNqzoKd1tCWOqeI05/lPqFPMGQ= github.com/gammazero/deque v1.2.1/go.mod h1:5nSFkzVm+afG9+gy0VIowlqVAW4N8zNcMne+CMQVD2g= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/ipfs/bbloom v0.1.0 h1:nIWwfIE3AaG7RCDQIsrUonGCOTp7qSXzxH7ab/ss964= github.com/ipfs/bbloom v0.1.0/go.mod h1:lDy3A3i6ndgEW2z1CaRFvDi5/ZTzgM1IxA/pkL7Wgts= github.com/ipfs/boxo v0.39.0 h1:u9jLf5pLx5SWROXjHtj8VMvv+iDlMbiTyZ/vVTQ4VhI= github.com/ipfs/boxo v0.39.0/go.mod h1:k9YCvMjytFguMHndEiGdCGMMj4b7CkdOT44vtgAxOdk= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.2.3 h1:mpCuDaNXJ4wrBJLrtEaGFGXkferrw5eqVvzaHhtFKQk= github.com/ipfs/go-block-format v0.2.3/go.mod h1:WJaQmPAKhD3LspLixqlqNFxiZ3BZ3xgqxxoSR/76pnA= github.com/ipfs/go-cid v0.6.1 h1:T5TnNb08+ueovG76Z5gx1L4Y7QOaGTXHg1F6raWFxIc= github.com/ipfs/go-cid v0.6.1/go.mod h1:zrY0SwOhjrrIdfPQ/kf+k1sXyJ0QE7cMxfCployLBs0= github.com/ipfs/go-cidutil v0.1.1 h1:COuby6H8C2ml0alvHYX3WdbFM4F07YtbY0UlT5j+sgI= github.com/ipfs/go-cidutil v0.1.1/go.mod h1:SCoUftGEUgoXe5Hjeyw5CiLZF8cwYn/TbtpFQXJCP6k= github.com/ipfs/go-datastore v0.9.1 h1:67Po2epre/o0UxrmkzdS9ZTe2GFGODgTd2odx8Wh6Yo= github.com/ipfs/go-datastore v0.9.1/go.mod h1:zi07Nvrpq1bQwSkEnx3bfjz+SQZbdbWyCNvyxMh9pN0= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= github.com/ipfs/go-ds-leveldb v0.5.2 h1:6nmxlQ2zbp4LCNdJVsmHfs9GP0eylfBNxpmY1csp0x0= github.com/ipfs/go-ds-leveldb v0.5.2/go.mod h1:2fAwmcvD3WoRT72PzEekHBkQmBDhc39DJGoREiuGmYo= github.com/ipfs/go-dsqueue v0.2.0 h1:MBi9w3oSiX98Xc+Y7NuJ9G8MI6mAT4IGdO9dHEMCZzU= github.com/ipfs/go-dsqueue v0.2.0/go.mod h1:8FfNQC4DMF/KkzBXRNB9Rb3MKDW0Sh98HMtXYl1mLQE= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-pq v0.0.4 h1:U7jjENWJd1jhcrR8X/xHTaph14PTAK9O+yaLJbjqgOw= github.com/ipfs/go-ipfs-pq v0.0.4/go.mod h1:9UdLOIIb99IFrgT0Fc53pvbvlJBhpUb4GJuAQf3+O2A= github.com/ipfs/go-ipld-format v0.6.3 h1:9/lurLDTotJpZSuL++gh3sTdmcFhVkCwsgx2+rAh4j8= github.com/ipfs/go-ipld-format v0.6.3/go.mod h1:74ilVN12NXVMIV+SrBAyC05UJRk0jVvGqdmrcYZvCBk= github.com/ipfs/go-ipld-legacy v0.3.0 h1:7XhFKkRyCvP5upOlQfKUFIqL3S5DEZnbUE4bQmQ/tNE= github.com/ipfs/go-ipld-legacy v0.3.0/go.mod h1:Ukef9ARQiX+RVetwH2XiReLgJvQDEXcUPszrZ1KRjKI= github.com/ipfs/go-log/v2 v2.9.1 h1:3JXwHWU31dsCpvQ+7asz6/QsFJHqFr4gLgQ0FWteujk= github.com/ipfs/go-log/v2 v2.9.1/go.mod h1:evFx7sBiohUN3AG12mXlZBw5hacBQld3ZPHrowlJYoo= github.com/ipfs/go-metrics-interface v0.3.0 h1:YwG7/Cy4R94mYDUuwsBfeziJCVm9pBMJ6q/JR9V40TU= github.com/ipfs/go-metrics-interface v0.3.0/go.mod h1:OxxQjZDGocXVdyTPocns6cOLwHieqej/jos7H4POwoY= github.com/ipfs/go-peertaskqueue v0.8.3 h1:tBPpGJy+A92RqtRFq5amJn0Uuj8Pw8tXi0X3eHfHM8w= github.com/ipfs/go-peertaskqueue v0.8.3/go.mod h1:OqVync4kPOcXEGdj/LKvox9DCB5mkSBeXsPczCxLtYA= github.com/ipfs/go-test v0.3.0 h1:0Y4Uve3tp9HI+2lIJjfOliOrOgv/YpXg/l1y3P4DEYE= github.com/ipfs/go-test v0.3.0/go.mod h1:JK+U8pRpATZb7lsYNSJlCj3WYB3cFfWIbI6nWRM/GFk= github.com/ipfs/go-unixfsnode v1.10.3 h1:c8sJjuGNkxXAQH75P+f5ngPda/9T+DrboVA0TcDGvGI= github.com/ipfs/go-unixfsnode v1.10.3/go.mod h1:2Jlc7DoEwr12W+7l8Hr6C7XF4NHST3gIkqSArLhGSxU= github.com/ipld/go-codec-dagpb v1.7.0 h1:hpuvQjCSVSLnTnHXn+QAMR0mLmb1gA6wl10LExo2Ts0= github.com/ipld/go-codec-dagpb v1.7.0/go.mod h1:rD3Zg+zub9ZnxcLwfol/OTQRVjaLzXypgy4UqHQvilM= github.com/ipld/go-ipld-prime v0.23.0 h1:csqdPZH60BsTC+AZrv7fpa27v+09I/oTqyHYYYE27eE= github.com/ipld/go-ipld-prime v0.23.0/go.mod h1:46YCFSFNFBJHPjB0pfMuv7Ly7df2eChpkpyPo5SE0bA= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/koron/go-ssdp v0.0.6 h1:Jb0h04599eq/CY7rB5YEqPS83HmRfHP2azkxMN2rFtU= github.com/koron/go-ssdp v0.0.6/go.mod h1:0R9LfRJGek1zWTjN3JUNlm5INCDYGpRDfAptnct63fI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/libp2p/go-flow-metrics v0.3.0 h1:q31zcHUvHnwDO0SHaukewPYgwOBSxtt830uJtUx6784= github.com/libp2p/go-flow-metrics v0.3.0/go.mod h1:nuhlreIwEguM1IvHAew3ij7A8BMlyHQJ279ao24eZZo= github.com/libp2p/go-libp2p v0.48.0 h1:h2BrLAgrj7X8bEN05K7qmrjpNHYA+6tnsGRdprjTnvo= github.com/libp2p/go-libp2p v0.48.0/go.mod h1:Q1fBZNdmC2Hf82husCTfkKJVfHm2we5zk+NWmOGEmWk= github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94= github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8= github.com/libp2p/go-libp2p-record v0.3.1 h1:cly48Xi5GjNw5Wq+7gmjfBiG9HCzQVkiZOUZ8kUl+Fg= github.com/libp2p/go-libp2p-record v0.3.1/go.mod h1:T8itUkLcWQLCYMqtX7Th6r7SexyUJpIyPgks757td/E= github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA= github.com/libp2p/go-libp2p-testing v0.12.0/go.mod h1:KcGDRXyN7sQCllucn1cOOS+Dmm7ujhfEyXQL5lvkcPg= github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0= github.com/libp2p/go-msgio v0.3.0/go.mod h1:nyRM819GmVaF9LX3l03RMh10QdOroF++NBbxAb0mmDM= github.com/libp2p/go-netroute v0.4.0 h1:sZZx9hyANYUx9PZyqcgE/E1GUG3iEtTZHUEvdtXT7/Q= github.com/libp2p/go-netroute v0.4.0/go.mod h1:Nkd5ShYgSMS5MUKy/MU2T57xFoOKvvLR92Lic48LEyA= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mr-tron/base58 v1.3.0 h1:K6Y13R2h+dku0wOqKtecgRnBUBPrZzLZy5aIj8lCcJI= github.com/mr-tron/base58 v1.3.0/go.mod h1:2BuubE67DCSWwVfx37JWNG8emOC0sHEU4/HpcYgCLX8= github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= github.com/multiformats/go-multiaddr v0.16.1 h1:fgJ0Pitow+wWXzN9do+1b8Pyjmo8m5WhGfzpL82MpCw= github.com/multiformats/go-multiaddr v0.16.1/go.mod h1:JSVUmXDjsVFiW7RjIFMP7+Ev+h1DTbiJgVeTV/tcmP0= github.com/multiformats/go-multiaddr-dns v0.5.0 h1:p/FTyHKX0nl59f+S+dEUe8HRK+i5Ow/QHMw8Nh3gPCo= github.com/multiformats/go-multiaddr-dns v0.5.0/go.mod h1:yJ349b8TPIAANUyuOzn1oz9o22tV9f+06L+cCeMxC14= github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= github.com/multiformats/go-multiaddr-fmt v0.1.0/go.mod h1:hGtDIW4PU4BqJ50gW2quDuPVjyWNZxToGUh/HwTZYJo= github.com/multiformats/go-multibase v0.3.0 h1:8helZD2+4Db7NNWFiktk2NePbF0boolBe6bDQvM4r68= github.com/multiformats/go-multibase v0.3.0/go.mod h1:MoBLQPCkRTOL3eveIPO81860j2AQY8JwcnNlRkGRUfI= github.com/multiformats/go-multicodec v0.10.0 h1:UpP223cig/Cx8J76jWt91njpK3GTAO1w02sdcjZDSuc= github.com/multiformats/go-multicodec v0.10.0/go.mod h1:wg88pM+s2kZJEQfRCKBNU+g32F5aWBEjyFHXvZLTcLI= github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= github.com/multiformats/go-multistream v0.6.1 h1:4aoX5v6T+yWmc2raBHsTvzmFhOI8WVOer28DeBBEYdQ= github.com/multiformats/go-multistream v0.6.1/go.mod h1:ksQf6kqHAb6zIsyw7Zm+gAuVo57Qbq84E27YlYqavqw= github.com/multiformats/go-varint v0.1.0 h1:i2wqFp4sdl3IcIxfAonHQV9qU5OsZ4Ts9IOoETFs5dI= github.com/multiformats/go-varint v0.1.0/go.mod h1:5KVAVXegtfmNQQm/lCY+ATvDzvJJhSkUlGQV9wgObdI= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/polydawn/refmt v0.89.1-0.20231129105047-37766d95467a h1:cgqrm0F3zwf9IPzca7xN4w+Zy6MC9ZkPvAC8QEWa/iQ= github.com/polydawn/refmt v0.89.1-0.20231129105047-37766d95467a/go.mod h1:ocZfO/tLSHqfScRDNTJbAJR1by4D1lewauX9OwTaPuY= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/warpfork/go-testmark v0.12.1 h1:rMgCpJfwy1sJ50x0M0NgyphxYYPMOODIJHhsXyEHU0s= github.com/warpfork/go-testmark v0.12.1/go.mod h1:kHwy7wfvGSPh1rQJYKayD4AbtNaeyZdcGi9tNJTaa5Y= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho= go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc= go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4= go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI= go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY= go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= lukechampine.com/blake3 v1.4.1 h1:I3Smz7gso8w4/TunLKec6K2fn+kyKtDxr/xcQEN84Wg= lukechampine.com/blake3 v1.4.1/go.mod h1:QFosUxmjB8mnrWFSNwKmvxHpfY72bmD2tQ0kBMM3kwo= ================================================ FILE: storage/bsrvadapter/README.md ================================================ bsrvadapter =========== The `bsrvadapter` package/module is a small piece of glue code to connect the `github.com/ipfs/go-blockservice` package, and packages implementing its interfaces, forward into the `go-ipld-prime/storage` interfaces. This can be used to rig systems like Bitswap up behind go-ipld-prime storage APIs. (Whether or not this is a good idea is debatable. It should be noted that both the `ipfs/go-blockservice` API, as well as Bitswap in particular as an implementation, are inherently prone to the infamous "N+1 Query Problem". Treating a remote network fetch as equivalent to a local low-latency operation just isn't a good idea for performance or predictability, no matter how you slice it. Nonetheless: it's possible, using this code, if you really want to do it.) Why structured like this? ------------------------- See `../README_adapters.md` for details about why adapter code is needed, why this is in a module, why it's here, etc. Which of `dsadapter` vs `bsadapter` vs `bsrvadapter` should I use? ------------------------------------------------------------------ In short: you should prefer direct implementations of the storage APIs over any of these adapters, if one is available with the features you need. Otherwise, if that's not an option (yet) for some reason, use whichever adapter gets you most directly connected to the code you need. See `../README_adapters.md` for more details and discussion. ================================================ FILE: storage/bsrvadapter/bsrvadapter.go ================================================ package bsrvadapter import ( "context" "fmt" "github.com/ipfs/boxo/blockservice" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" ) // Adapter implements go-ipld-prime/storage.ReadableStorage // and go-ipld-prime/storage.WritableStorage // backed by a go-blockservice.BlockService. // // The go-blockservice.BlockService may internally have other configuration, // and contain whole other systems like Bitswap for transport. // We don't interfere with that here; // such configuration should be handled when creating the go-blockservice value. // // Note that this system will only work for certain structures of keys -- // this is because the blockservice API works on the level of CIDs. // As long as your key string is the binary form of a CID, it will work correctly. // Other keys are not possible to support with this adapter. // // Contexts given to this system are passed through where possible, but it is not possible in all cases. // For operations where the underlying interface doesn't accept a context parameter, // this adapter will check the context for errors before beginning an operation, // but the context will otherwise have no effect. // For operations where BlockService does accept a context, we pass it on. type Adapter struct { Wrapped blockservice.BlockService } // Has implements go-ipld-prime/storage.Storage.Has. // // Note that for a BlockService, the Has operation has rather unusual semantics. // Has may return false, and an immediately subsequent Get for the same key might return data! // This is because the Has operation is defined as whether // the Blockstore that the BlockService wraps has the requested key, immediately, locally; // while the Get operation might use the BlockService to go _find_ the requested key // and its content, even remotely! func (a *Adapter) Has(ctx context.Context, key string) (bool, error) { // Return early if the context is already closed. // This is also the last time we'll check the context, // since the Has method is on Blockstore, which doesn't take them. if ctx.Err() != nil { return false, ctx.Err() } // Do the inverse of cid.KeyString(), // which is how a valid key for this adapter must've been produced. k, err := cidFromBinString(key) if err != nil { return false, err } // Delegate the Has call. return a.Wrapped.Blockstore().Has(ctx, k) } // Get implements go-ipld-prime/storage.ReadableStorage.Get. func (a *Adapter) Get(ctx context.Context, key string) ([]byte, error) { // No need to check the context proactively here -- // the BlockService API actually accepts context. // Do the inverse of cid.KeyString(), // which is how a valid key for this adapter must've been produced. k, err := cidFromBinString(key) if err != nil { return nil, err } // Delegate the Get call. // It's called "GetBlock" in BlockService. block, err := a.Wrapped.GetBlock(ctx, k) if err != nil { return nil, err } // Unwrap the actual raw data for return. // Discard the rest. (It's a shame there was an alloc for that structure.) return block.RawData(), nil } // Put implements go-ipld-prime/storage.WritableStorage.Put. func (a *Adapter) Put(ctx context.Context, key string, content []byte) error { // Return early if the context is already closed. // This is also the last time we'll check the context, // since the AddBlock method on BlockService that we'll eventually be delegating to doesn't take them. if ctx.Err() != nil { return ctx.Err() } // Do the inverse of cid.KeyString(), // which is how a valid key for this adapter must've been produced. k, err := cidFromBinString(key) if err != nil { return err } // Create a structure that has the cid and the raw content together. // This is necessary because it's the format demanded by BlockService. // (Unfortunately, it also provokes an allocation, because it uses interfaces; // but we can't avoid that without changing the code in go-blockservice.) // The error is treated as a panic because it's only possible if a global debug var is set, // and is for behavior that is not meant to be part of the contract of the storage APIs. block, err := blocks.NewBlockWithCid(content, k) if err != nil { panic(err) } // Delegate the Put call. // It's called "AddBlock" in BlockService. return a.Wrapped.AddBlock(ctx, block) } // Do the inverse of cid.KeyString(). // (Unclear why go-cid doesn't offer a function for this itself.) func cidFromBinString(key string) (cid.Cid, error) { l, k, err := cid.CidFromBytes([]byte(key)) if err != nil { return cid.Undef, fmt.Errorf("bsrvadapter: key was not a cid: %w", err) } if l != len(key) { return cid.Undef, fmt.Errorf("bsrvadapter: key was not a cid: had %d bytes leftover", len(key)-l) } return k, nil } ================================================ FILE: storage/bsrvadapter/go.mod ================================================ module github.com/ipld/go-ipld-prime/storage/bsrvadapter go 1.25.7 require ( github.com/ipfs/boxo v0.39.0 github.com/ipfs/go-block-format v0.2.3 github.com/ipfs/go-cid v0.6.1 ) require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/gammazero/chanqueue v1.1.2 // indirect github.com/gammazero/deque v1.2.1 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/ipfs/bbloom v0.1.0 // indirect github.com/ipfs/go-cidutil v0.1.1 // indirect github.com/ipfs/go-datastore v0.9.1 // indirect github.com/ipfs/go-dsqueue v0.2.0 // indirect github.com/ipfs/go-ipld-format v0.6.3 // indirect github.com/ipfs/go-log/v2 v2.9.1 // indirect github.com/ipfs/go-metrics-interface v0.3.0 // indirect github.com/ipld/go-ipld-prime v0.23.0 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mr-tron/base58 v1.3.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect github.com/multiformats/go-multibase v0.3.0 // indirect github.com/multiformats/go-multicodec v0.10.0 // indirect github.com/multiformats/go-multihash v0.2.3 // indirect github.com/multiformats/go-varint v0.1.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/otel v1.42.0 // indirect go.opentelemetry.io/otel/metric v1.42.0 // indirect go.opentelemetry.io/otel/trace v1.42.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.1 // indirect golang.org/x/crypto v0.50.0 // indirect golang.org/x/sys v0.43.0 // indirect lukechampine.com/blake3 v1.4.1 // indirect ) ================================================ FILE: storage/bsrvadapter/go.sum ================================================ github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 h1:5RVFMOWjMyRy8cARdy79nAmgYw3hK/4HUq48LQ6Wwqo= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= github.com/filecoin-project/go-clock v0.1.0 h1:SFbYIM75M8NnFm1yMHhN9Ahy3W5bEZV9gd6MPfXbKVU= github.com/filecoin-project/go-clock v0.1.0/go.mod h1:4uB/O4PvOjlx1VCMdZ9MyDZXRm//gkj1ELEbxfI1AZs= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/gammazero/chanqueue v1.1.2 h1:dZEsxlyANZMyeTRemABqZF8QM9BnE4NBI43Oh3y5fIU= github.com/gammazero/chanqueue v1.1.2/go.mod h1:XDN1X/jjAbmSceNFOQbtKToeSkxtdVdpKu90LiEdBEE= github.com/gammazero/deque v1.2.1 h1:9fnQVFCCZ9/NOc7ccTNqzoKd1tCWOqeI05/lPqFPMGQ= github.com/gammazero/deque v1.2.1/go.mod h1:5nSFkzVm+afG9+gy0VIowlqVAW4N8zNcMne+CMQVD2g= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/ipfs/bbloom v0.1.0 h1:nIWwfIE3AaG7RCDQIsrUonGCOTp7qSXzxH7ab/ss964= github.com/ipfs/bbloom v0.1.0/go.mod h1:lDy3A3i6ndgEW2z1CaRFvDi5/ZTzgM1IxA/pkL7Wgts= github.com/ipfs/boxo v0.39.0 h1:u9jLf5pLx5SWROXjHtj8VMvv+iDlMbiTyZ/vVTQ4VhI= github.com/ipfs/boxo v0.39.0/go.mod h1:k9YCvMjytFguMHndEiGdCGMMj4b7CkdOT44vtgAxOdk= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.2.3 h1:mpCuDaNXJ4wrBJLrtEaGFGXkferrw5eqVvzaHhtFKQk= github.com/ipfs/go-block-format v0.2.3/go.mod h1:WJaQmPAKhD3LspLixqlqNFxiZ3BZ3xgqxxoSR/76pnA= github.com/ipfs/go-cid v0.6.1 h1:T5TnNb08+ueovG76Z5gx1L4Y7QOaGTXHg1F6raWFxIc= github.com/ipfs/go-cid v0.6.1/go.mod h1:zrY0SwOhjrrIdfPQ/kf+k1sXyJ0QE7cMxfCployLBs0= github.com/ipfs/go-cidutil v0.1.1 h1:COuby6H8C2ml0alvHYX3WdbFM4F07YtbY0UlT5j+sgI= github.com/ipfs/go-cidutil v0.1.1/go.mod h1:SCoUftGEUgoXe5Hjeyw5CiLZF8cwYn/TbtpFQXJCP6k= github.com/ipfs/go-datastore v0.9.1 h1:67Po2epre/o0UxrmkzdS9ZTe2GFGODgTd2odx8Wh6Yo= github.com/ipfs/go-datastore v0.9.1/go.mod h1:zi07Nvrpq1bQwSkEnx3bfjz+SQZbdbWyCNvyxMh9pN0= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= github.com/ipfs/go-ds-leveldb v0.5.2 h1:6nmxlQ2zbp4LCNdJVsmHfs9GP0eylfBNxpmY1csp0x0= github.com/ipfs/go-ds-leveldb v0.5.2/go.mod h1:2fAwmcvD3WoRT72PzEekHBkQmBDhc39DJGoREiuGmYo= github.com/ipfs/go-dsqueue v0.2.0 h1:MBi9w3oSiX98Xc+Y7NuJ9G8MI6mAT4IGdO9dHEMCZzU= github.com/ipfs/go-dsqueue v0.2.0/go.mod h1:8FfNQC4DMF/KkzBXRNB9Rb3MKDW0Sh98HMtXYl1mLQE= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-pq v0.0.4 h1:U7jjENWJd1jhcrR8X/xHTaph14PTAK9O+yaLJbjqgOw= github.com/ipfs/go-ipfs-pq v0.0.4/go.mod h1:9UdLOIIb99IFrgT0Fc53pvbvlJBhpUb4GJuAQf3+O2A= github.com/ipfs/go-ipld-format v0.6.3 h1:9/lurLDTotJpZSuL++gh3sTdmcFhVkCwsgx2+rAh4j8= github.com/ipfs/go-ipld-format v0.6.3/go.mod h1:74ilVN12NXVMIV+SrBAyC05UJRk0jVvGqdmrcYZvCBk= github.com/ipfs/go-ipld-legacy v0.3.0 h1:7XhFKkRyCvP5upOlQfKUFIqL3S5DEZnbUE4bQmQ/tNE= github.com/ipfs/go-ipld-legacy v0.3.0/go.mod h1:Ukef9ARQiX+RVetwH2XiReLgJvQDEXcUPszrZ1KRjKI= github.com/ipfs/go-log/v2 v2.9.1 h1:3JXwHWU31dsCpvQ+7asz6/QsFJHqFr4gLgQ0FWteujk= github.com/ipfs/go-log/v2 v2.9.1/go.mod h1:evFx7sBiohUN3AG12mXlZBw5hacBQld3ZPHrowlJYoo= github.com/ipfs/go-metrics-interface v0.3.0 h1:YwG7/Cy4R94mYDUuwsBfeziJCVm9pBMJ6q/JR9V40TU= github.com/ipfs/go-metrics-interface v0.3.0/go.mod h1:OxxQjZDGocXVdyTPocns6cOLwHieqej/jos7H4POwoY= github.com/ipfs/go-peertaskqueue v0.8.3 h1:tBPpGJy+A92RqtRFq5amJn0Uuj8Pw8tXi0X3eHfHM8w= github.com/ipfs/go-peertaskqueue v0.8.3/go.mod h1:OqVync4kPOcXEGdj/LKvox9DCB5mkSBeXsPczCxLtYA= github.com/ipfs/go-test v0.3.0 h1:0Y4Uve3tp9HI+2lIJjfOliOrOgv/YpXg/l1y3P4DEYE= github.com/ipfs/go-test v0.3.0/go.mod h1:JK+U8pRpATZb7lsYNSJlCj3WYB3cFfWIbI6nWRM/GFk= github.com/ipfs/go-unixfsnode v1.10.3 h1:c8sJjuGNkxXAQH75P+f5ngPda/9T+DrboVA0TcDGvGI= github.com/ipfs/go-unixfsnode v1.10.3/go.mod h1:2Jlc7DoEwr12W+7l8Hr6C7XF4NHST3gIkqSArLhGSxU= github.com/ipld/go-codec-dagpb v1.7.0 h1:hpuvQjCSVSLnTnHXn+QAMR0mLmb1gA6wl10LExo2Ts0= github.com/ipld/go-codec-dagpb v1.7.0/go.mod h1:rD3Zg+zub9ZnxcLwfol/OTQRVjaLzXypgy4UqHQvilM= github.com/ipld/go-ipld-prime v0.23.0 h1:csqdPZH60BsTC+AZrv7fpa27v+09I/oTqyHYYYE27eE= github.com/ipld/go-ipld-prime v0.23.0/go.mod h1:46YCFSFNFBJHPjB0pfMuv7Ly7df2eChpkpyPo5SE0bA= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/koron/go-ssdp v0.0.6 h1:Jb0h04599eq/CY7rB5YEqPS83HmRfHP2azkxMN2rFtU= github.com/koron/go-ssdp v0.0.6/go.mod h1:0R9LfRJGek1zWTjN3JUNlm5INCDYGpRDfAptnct63fI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/libp2p/go-flow-metrics v0.3.0 h1:q31zcHUvHnwDO0SHaukewPYgwOBSxtt830uJtUx6784= github.com/libp2p/go-flow-metrics v0.3.0/go.mod h1:nuhlreIwEguM1IvHAew3ij7A8BMlyHQJ279ao24eZZo= github.com/libp2p/go-libp2p v0.48.0 h1:h2BrLAgrj7X8bEN05K7qmrjpNHYA+6tnsGRdprjTnvo= github.com/libp2p/go-libp2p v0.48.0/go.mod h1:Q1fBZNdmC2Hf82husCTfkKJVfHm2we5zk+NWmOGEmWk= github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94= github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8= github.com/libp2p/go-libp2p-record v0.3.1 h1:cly48Xi5GjNw5Wq+7gmjfBiG9HCzQVkiZOUZ8kUl+Fg= github.com/libp2p/go-libp2p-record v0.3.1/go.mod h1:T8itUkLcWQLCYMqtX7Th6r7SexyUJpIyPgks757td/E= github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA= github.com/libp2p/go-libp2p-testing v0.12.0/go.mod h1:KcGDRXyN7sQCllucn1cOOS+Dmm7ujhfEyXQL5lvkcPg= github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0= github.com/libp2p/go-msgio v0.3.0/go.mod h1:nyRM819GmVaF9LX3l03RMh10QdOroF++NBbxAb0mmDM= github.com/libp2p/go-netroute v0.4.0 h1:sZZx9hyANYUx9PZyqcgE/E1GUG3iEtTZHUEvdtXT7/Q= github.com/libp2p/go-netroute v0.4.0/go.mod h1:Nkd5ShYgSMS5MUKy/MU2T57xFoOKvvLR92Lic48LEyA= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mr-tron/base58 v1.3.0 h1:K6Y13R2h+dku0wOqKtecgRnBUBPrZzLZy5aIj8lCcJI= github.com/mr-tron/base58 v1.3.0/go.mod h1:2BuubE67DCSWwVfx37JWNG8emOC0sHEU4/HpcYgCLX8= github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= github.com/multiformats/go-multiaddr v0.16.1 h1:fgJ0Pitow+wWXzN9do+1b8Pyjmo8m5WhGfzpL82MpCw= github.com/multiformats/go-multiaddr v0.16.1/go.mod h1:JSVUmXDjsVFiW7RjIFMP7+Ev+h1DTbiJgVeTV/tcmP0= github.com/multiformats/go-multiaddr-dns v0.5.0 h1:p/FTyHKX0nl59f+S+dEUe8HRK+i5Ow/QHMw8Nh3gPCo= github.com/multiformats/go-multiaddr-dns v0.5.0/go.mod h1:yJ349b8TPIAANUyuOzn1oz9o22tV9f+06L+cCeMxC14= github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= github.com/multiformats/go-multiaddr-fmt v0.1.0/go.mod h1:hGtDIW4PU4BqJ50gW2quDuPVjyWNZxToGUh/HwTZYJo= github.com/multiformats/go-multibase v0.3.0 h1:8helZD2+4Db7NNWFiktk2NePbF0boolBe6bDQvM4r68= github.com/multiformats/go-multibase v0.3.0/go.mod h1:MoBLQPCkRTOL3eveIPO81860j2AQY8JwcnNlRkGRUfI= github.com/multiformats/go-multicodec v0.10.0 h1:UpP223cig/Cx8J76jWt91njpK3GTAO1w02sdcjZDSuc= github.com/multiformats/go-multicodec v0.10.0/go.mod h1:wg88pM+s2kZJEQfRCKBNU+g32F5aWBEjyFHXvZLTcLI= github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= github.com/multiformats/go-multistream v0.6.1 h1:4aoX5v6T+yWmc2raBHsTvzmFhOI8WVOer28DeBBEYdQ= github.com/multiformats/go-multistream v0.6.1/go.mod h1:ksQf6kqHAb6zIsyw7Zm+gAuVo57Qbq84E27YlYqavqw= github.com/multiformats/go-varint v0.1.0 h1:i2wqFp4sdl3IcIxfAonHQV9qU5OsZ4Ts9IOoETFs5dI= github.com/multiformats/go-varint v0.1.0/go.mod h1:5KVAVXegtfmNQQm/lCY+ATvDzvJJhSkUlGQV9wgObdI= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/polydawn/refmt v0.89.1-0.20231129105047-37766d95467a h1:cgqrm0F3zwf9IPzca7xN4w+Zy6MC9ZkPvAC8QEWa/iQ= github.com/polydawn/refmt v0.89.1-0.20231129105047-37766d95467a/go.mod h1:ocZfO/tLSHqfScRDNTJbAJR1by4D1lewauX9OwTaPuY= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/warpfork/go-testmark v0.12.1 h1:rMgCpJfwy1sJ50x0M0NgyphxYYPMOODIJHhsXyEHU0s= github.com/warpfork/go-testmark v0.12.1/go.mod h1:kHwy7wfvGSPh1rQJYKayD4AbtNaeyZdcGi9tNJTaa5Y= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho= go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc= go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4= go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI= go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY= go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= lukechampine.com/blake3 v1.4.1 h1:I3Smz7gso8w4/TunLKec6K2fn+kyKtDxr/xcQEN84Wg= lukechampine.com/blake3 v1.4.1/go.mod h1:QFosUxmjB8mnrWFSNwKmvxHpfY72bmD2tQ0kBMM3kwo= ================================================ FILE: storage/doc.go ================================================ // The storage package contains interfaces for storage systems, and functions for using them. // // These are very low-level storage primitives. // The interfaces here deal only with raw keys and raw binary blob values. // // In IPLD, you can often avoid dealing with storage directly yourself, // and instead use linking.LinkSystem to handle serialization, hashing, and storage all at once. // (You'll hand some values that match interfaces from this package to LinkSystem when configuring it.) // It's probably best to work at that level and above as much as possible. // If you do need to interact with storage more directly, the read on. // // The most basic APIs are ReadableStorage and WritableStorage. // When writing code that works with storage systems, these two interfaces should be seen in almost all situations: // user code is recommended to think in terms of these types; // functions provided by this package will accept parameters of these types and work on them; // implementations are expected to provide these types first; // and any new library code is recommended to keep with the theme: use these interfaces preferentially. // // Users should decide which actions they want to take using a storage system, // find the appropriate function in this package (n.b., package function -- not a method on an interface! // You will likely find one of each, with the same name: pick the package function!), // and use that function, providing it the storage system (e.g. either ReadableStorage, WritableStorage, or sometimes just Storage) // as a parameter. // That function will then use feature-detection (checking for matches to the other, // more advanced and more specific interfaces in this package) and choose the best way // to satisfy the request; or, if it can't feature-detect any relevant features, // the function will fall back to synthesizing the requested behavior out of the most basic API. // Using the package functions, and letting them do the feature detection for you, // should provide the most consistent user experience and minimize the amount of work you need to do. // (Bonus: It also gives us a convenient place to smooth out any future library migrations for you!) // // If writing new APIs that are meant to work reusably for any storage implementation: // APIs should usually be designed around accepting ReadableStorage or WritableStorage as parameters // (depending on which direction of data flow the API is regarding). // and use the other interfaces (e.g. StreamingReadableStorage) thereafter internally for feature detection. // For APIs which may sometimes be found relating to either a read or a write direction of data flow, // the Storage interface may be used in order to define a function that should accept either ReadableStorage or WritableStorage. // In other words: when writing reusable APIs, one should follow the same pattern as this package's own functions do. // // Similarly, implementers of storage systems should always implement either ReadableStorage or WritableStorage first. // Only after satisfying one of those should the implementation then move on to further supporting // additional interfaces in this package (all of which are meant to support feature-detection). // Beyond one of the basic two, all the other interfaces are optional: // you can implement them if you want to advertise additional features, // or advertise fastpaths that your storage system supports; // but you don't have implement any of those additional interfaces if you don't want to, // or if your implementation can't offer useful fastpaths for them. // // Storage systems as described by this package are allowed to make some interesting trades. // Generally, write operations are allowed to be first-write-wins. // Furthermore, there is no requirement that the system return an error if a subsequent write to the same key has different content. // These rules are reasonable for a content-addressed storage system, and allow great optimizations to be made. // // Note that all of the interfaces in this package only use types that are present in the golang standard library. // This is intentional, and was done very carefully. // If implementing a storage system, you should find it possible to do so *without* importing this package. // Because only standard library types are present in the interface contracts, // it's possible to implement types that align with the interfaces without referring to them. // // Note that where keys are discussed in this package, they use the golang string type -- // however, they may be binary. (The golang string type allows arbitrary bytes in general, // and here, we both use that, and explicitly disavow the usual "norm" that the string type implies UTF-8. // This is roughly the same as the practical truth that appears when using e.g. os.OpenFile and other similar functions.) // If you are creating a storage implementation where the underlying medium does not support arbitrary binary keys, // then it is strongly recommend that your storage implementation should support being configured with // an "escaping function", which should typically simply be of the form `func(string) string`. // Additional, your storage implementation's documentation should also clearly describe its internal limitations, // so that users have enough information to write an escaping function which // maps their domain into the domain your storage implementation can handle. package storage // also note: // LinkContext stays *out* of this package. It's a chooser-related thing. // LinkSystem can think about it (and your callbacks over there can think about it), and that's the end of its road. // (Future: probably LinkSystem should have SetStorage and SetupStorageChooser methods for helping you set things up -- where the former doesn't discuss LinkContext at all.) ================================================ FILE: storage/dsadapter/README.md ================================================ dsadapter ========= The `dsadapter` package/module is a small piece of glue code to connect the `github.com/ipfs/go-datastore` package, and packages implementing its interfaces, forward into the `go-ipld-prime/storage` interfaces. For example, this can be used to use "flatfs" and other datastore plugins with go-ipld-prime storage APIs. Why structured like this? ------------------------- See `../README_adapters.md` for details about why adapter code is needed, why this is in a module, why it's here, etc. Which of `dsadapter` vs `bsadapter` vs `bsrvadapter` should I use? ------------------------------------------------------------------ In short: you should prefer direct implementations of the storage APIs over any of these adapters, if one is available with the features you need. Otherwise, if that's not an option (yet) for some reason, use whichever adapter gets you most directly connected to the code you need. See `../README_adapters.md` for more details and discussion. ================================================ FILE: storage/dsadapter/dsadapter.go ================================================ package dsadapter import ( "context" "github.com/ipfs/go-datastore" ) // Adapter implements go-ipld-prime/storage.ReadableStorage // and go-ipld-prime/storage.WritableStorage // backed by a go-datastore.Datastore. // // Optionally, an EscapingFunc may also be set, // which transforms the (possibly binary) keys considered acceptable // by the go-ipld-prime/storage APIs into a subset that // the go-datastore can accept. // (Be careful to use any escaping consistently, // and be wary of potential unexpected behavior if the escaping function might // collapse two distinct keys into the same "escaped" key.) // // The go-datastore.Datastore may internally have other configuration, // such as key sharding functions, etc, and we don't interfere with that here; // such configuration should be handled when creating the go-datastore value. // // Contexts given to this system are checked for errors at the beginning of an operation, // but otherwise have no effect, because the Datastore API doesn't accept context parameters. type Adapter struct { Wrapped datastore.Datastore EscapingFunc func(string) string } // Has implements go-ipld-prime/storage.Storage.Has. func (a *Adapter) Has(ctx context.Context, key string) (bool, error) { // Return early if the context is already closed. // This is also the last time we'll check the context, // since go-datastore doesn't take them. if ctx.Err() != nil { return false, ctx.Err() } // If we have an EscapingFunc, apply it. if a.EscapingFunc != nil { key = a.EscapingFunc(key) } // Wrap the key into go-datastore's concrete type that it requires. // Note that this does a bunch of actual work, which may be surprising. // The key may be transformed (as per path.Clean). // There will also be an allocation, if the key doesn't start with "/". // (Avoiding these performance drags is part of why we started // new interfaces in go-ipld-prime/storage.) k := datastore.NewKey(key) // Delegate the has call. // Note that for some datastore implementations, this will do *yet more* // validation on the key, and may return errors from that. return a.Wrapped.Has(ctx, k) } // Get implements go-ipld-prime/storage.ReadableStorage.Get. func (a *Adapter) Get(ctx context.Context, key string) ([]byte, error) { // Return early if the context is already closed. // This is also the last time we'll check the context, // since go-datastore doesn't take them. if ctx.Err() != nil { return nil, ctx.Err() } // If we have an EscapingFunc, apply it. if a.EscapingFunc != nil { key = a.EscapingFunc(key) } // Wrap the key into go-datastore's concrete type that it requires. // Note that this does a bunch of actual work, which may be surprising. // The key may be transformed (as per path.Clean). // There will also be an allocation, if the key doesn't start with "/". // (Avoiding these performance drags is part of why we started // new interfaces in go-ipld-prime/storage.) k := datastore.NewKey(key) // Delegate the get call. // Note that for some datastore implementations, this will do *yet more* // validation on the key, and may return errors from that. return a.Wrapped.Get(ctx, k) } // Put implements go-ipld-prime/storage.WritableStorage.Put. func (a *Adapter) Put(ctx context.Context, key string, content []byte) error { // Return early if the context is already closed. // This is also the last time we'll check the context, // since go-datastore doesn't take them. if ctx.Err() != nil { return ctx.Err() } // If we have an EscapingFunc, apply it. if a.EscapingFunc != nil { key = a.EscapingFunc(key) } // Wrap the key into go-datastore's concrete type that it requires. // Note that this does a bunch of actual work, which may be surprising. // The key may be transformed (as per path.Clean). // There will also be an allocation, if the key doesn't start with "/". // (Avoiding these performance drags is part of why we started // new interfaces in go-ipld-prime/storage.) k := datastore.NewKey(key) // Delegate the put call. // Note that for some datastore implementations, this will do *yet more* // validation on the key, and may return errors from that. return a.Wrapped.Put(ctx, k, content) } ================================================ FILE: storage/dsadapter/go.mod ================================================ module github.com/ipld/go-ipld-prime/storage/dsadapter go 1.25 require github.com/ipfs/go-datastore v0.9.1 require github.com/google/uuid v1.6.0 // indirect ================================================ FILE: storage/dsadapter/go.sum ================================================ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/ipfs/go-datastore v0.9.1 h1:67Po2epre/o0UxrmkzdS9ZTe2GFGODgTd2odx8Wh6Yo= github.com/ipfs/go-datastore v0.9.1/go.mod h1:zi07Nvrpq1bQwSkEnx3bfjz+SQZbdbWyCNvyxMh9pN0= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= ================================================ FILE: storage/fsstore/fsstore.go ================================================ package fsstore import ( "context" "crypto/rand" "encoding/base32" "encoding/hex" "fmt" "io" "os" "path/filepath" "github.com/ipld/go-ipld-prime/storage/sharding" ) // Store is implements storage.ReadableStorage and storage.WritableStorage, // as well as quite a few of the other extended storage feature interfaces, // backing it with simple filesystem operations. // // This implementation uses golang's usual `os` package for IO, // so it should be highly portable. // // Both the sharding and escaping functions are configurable, // but a typical recommended setup is to use base32 encoding, // and a sharding function that returns two shards of two characters each. // The escaping and sharding functions should be chosen with regard to each other -- // the sharding function is applied to the escaped form. type Store struct { basepath string escapingFunc func(string) string shardingFunc func(key string, shards *[]string) } func (store *Store) InitDefaults(basepath string) error { return store.Init( basepath, b32enc, // The same function as go-ipfs uses: see https://github.com/ipfs/go-ipfs-ds-help/blob/48b9cc210923d23b39582b5fa6670ed0d08dc2af/key.go#L20-L22 . sharding.Shard_r12, // Equivalent to what go-ipfs uses by default with flatfs: see https://github.com/ipfs/go-ipfs/blob/52a747763f6c4e85b33ca051cda9cc4b75c815f9/docs/config.md#datastorespec and grep for "shard/v1/next-to-last/2". ) } func (store *Store) Init( basepath string, escapingFunc func(string) string, shardingFunc func(key string, shards *[]string), ) error { // Simple args and state check. if basepath == "" { return fmt.Errorf("fsstore: invalid setup args: need a path") } if store.basepath != "" { return fmt.Errorf("fsstore: cannot init: is already initialized") } store.basepath = basepath store.escapingFunc = escapingFunc store.shardingFunc = shardingFunc // Make sure basepath is a dir, and make sure the staging and content dirs exist. if err := CheckAndMakeBasepath(basepath); err != nil { return err } // That's it for setup on this one. return nil } var b32encoder = base32.StdEncoding.WithPadding(base32.NoPadding) func b32enc(in string) string { return b32encoder.EncodeToString([]byte(in)) } // pathForKey applies sharding funcs as well as adds the basepath prefix, // returning a string ready to use as a filesystem path. func (store *Store) pathForKey(key string) string { shards := make([]string, 1, 4) // future work: would be nice if we could reuse this rather than fresh allocating. shards[0] = store.basepath // not part of the path shard, but will be a param to Join, so, practical to put here. //shards[1] = storageDir // not part of the path shard, but will be a param to Join, so, practical to put here. store.shardingFunc(key, &shards) return filepath.Join(shards...) } // Has implements go-ipld-prime/storage.Storage.Has. func (store *Store) Has(ctx context.Context, key string) (bool, error) { _, err := os.Stat(store.pathForKey(key)) if err == nil { return true, nil } if os.IsNotExist(err) { return false, nil } return false, err } // Get implements go-ipld-prime/storage.ReadableStorage.Get. func (store *Store) Get(ctx context.Context, key string) ([]byte, error) { f, err := store.GetStream(ctx, key) if err != nil { return nil, err } defer f.(io.Closer).Close() return io.ReadAll(f) } // Put implements go-ipld-prime/storage.WritableStorage.Put. func (store *Store) Put(ctx context.Context, key string, content []byte) error { // We can't improve much on what we get by wrapping the stream interface; // we always end up using a streaming action on the very bottom because that's how file writing works // (especially since we care about controlling the write flow enough to be able to do the atomic move at the end). wr, wrCommitter, err := store.PutStream(ctx) if err != nil { return err } // Write, all at once. // Note we can ignore the size return, because the contract of io.Writer states "Write must return a non-nil error if it returns n < len(p)". _, err = wr.Write(content) if err != nil { wrCommitter("") return err } // Commit. return wrCommitter(key) } // GetStream implements go-ipld-prime/storage.StreamingReadableStorage.GetStream. func (store *Store) GetStream(ctx context.Context, key string) (io.ReadCloser, error) { if ctx.Err() != nil { return nil, ctx.Err() } // Figure out where we expect it to be. destpath := store.pathForKey(key) // Open and return. // TODO: we should normalize things like "not exists" errors before hurling them up the stack. return os.OpenFile(destpath, os.O_RDONLY, 0) } // PutStream implements go-ipld-prime/storage.StreamingWritableStorage.PutStream. func (store *Store) PutStream(ctx context.Context) (io.Writer, func(string) error, error) { for { if ctx.Err() != nil { return nil, nil, ctx.Err() } // Open a new file in the staging area, with a random name. var bs [8]byte rand.Read(bs[:]) stagepath := filepath.Join(store.basepath, stagingDir, hex.EncodeToString(bs[:])) f, err := os.OpenFile(stagepath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0666) if os.IsExist(err) { continue } if err != nil { return nil, nil, fmt.Errorf("fsstore.BeginWrite: could not create a staging file: %w", err) } // Okay, got a handle. Return it... and its commit closure. return f, func(key string) error { // Close the staging file. if err := f.Close(); err != nil { return err } if key == "" { return os.Remove(stagepath) } // n.b. there is a lack of fsync here. I am going to choose to believe that a sane filesystem will not let me do a 'move' without flushing somewhere in between. // Fun little note: there are some times in history where this belief is not backed -- but, mostly, the evolution of kernel and filesystem development seems to have considered that a mistake, // and things do again typically take 'move' as a strong cue to flush, unless you've actively configured your system oddly. // See https://en.wikipedia.org/wiki/Ext4#Delayed_allocation_and_potential_data_loss for some fun history regarding Ext4; // but ultimately, note that the kernel decided to again make 'move' cause flush, and has done so since 2.6.30, which came out sometime in 2009. // Accordingly, our lack of fsync here seems justified. // However, if you *really* find a system in the wild where this is problematic, // *and* you cannot make your application recover gracefully (which should be relatively easy, because... content addressing; you can't have inconsistency, at least!), // *and* you cannot configure your filesystem to have the level of durability and sanity that you want, so you must fix it in application land... // then... patches welcome. :) // // History also seems to indicate that if we add fsyncs hereabouts, people will usually just turn around and seek to disable them for performance reasons; // so by default, it seems best to just not do the dance of having a default that people hate. // Figure out where we want it to go. destpath := store.pathForKey(key) // Get it there. return move(stagepath, destpath) }, nil } } const stagingDir = ".temp" // same as flatfs uses. func CheckAndMakeBasepath(basepath string) error { // Is this basepath a dir? // (This is TOCTOU, obviously, but also it's nice to sanity check early and return error quickly because it's probably a setup error.) if fi, err := os.Stat(basepath); err != nil { return fmt.Errorf("fsstore: cannot init: basepath must be a directory: %w", err) } else { if !fi.IsDir() { return fmt.Errorf("fsstore: cannot init: basepath must be a directory") } } // Make sure the staging dir exists. err := os.Mkdir(filepath.Join(basepath, stagingDir), 0777) switch { case err == nil: // excellent. case os.IsExist(err): // sanity check it's a directory already. fi, err := os.Stat(filepath.Join(basepath, stagingDir)) if err != nil { return fmt.Errorf("fsstore: failed to make staging dir: %w", err) } if !fi.IsDir() { return fmt.Errorf("fsstore: staging dir path contains not a dir") } default: return fmt.Errorf("fsstore: failed to make staging dir: %w", err) } return nil } // move file at stagepath to destpath. // First, attempt to directly rename to the destination; // if we get a ENOENT error code, that means the parent didn't exist, and we make that and then retry. // If making the parent failed: recurse, and use similar logic. // // This optimistic approach should have fewer syscall RTTs when most of the parents exist // than would be taken if we checked that each parent segment exists. // // (An alternative approach would be to blindly mkdir the parent segments every time, // rather than do this backwards stepping. Have not benchmarked these against each other.) func move(stagepath, destpath string) error { err := os.Rename(stagepath, destpath) if os.IsNotExist(err) { // This probably means parent of destpath doesn't exist yet, so we'll make it. // It's technically a race condition to assume that this is because destpath has no parents vs that stagepath hasn't been removed out from underneath us, but, alas; kernel ABIs. // If we did this will all fds, it could be somewhat better. // (This is certainly possible, at least in linux; but we'd have to import the syscall package and do it ourselves, which is not a rubicon we're willing to cross in this package.) // In practice, this is probably not going to kerfuffle things. if err := haveDir(filepath.Dir(destpath)); err != nil { return err } // Now try again. // (And don't return quite yet; there's one more check to do, because someone might've raced us.) err = os.Rename(stagepath, destpath) } if os.IsExist(err) { // Oh! Some content is already there? // We're a write-once (presumed-to-be-)content-addressable blob store -- that means *we keep what already exists*. // FIXME: no, I wish this is how the Rename function worked, but it is not, actually. return os.Remove(stagepath) } return err } // haveDir tries to make sure a directory exists at pth. // If this sounds a lot like os.MkdirAll: yes, // except this function is going to assume if it exists, it's a dir, // and that saves us some stat syscalls. func haveDir(pth string) error { err := os.Mkdir(pth, 0777) if os.IsNotExist(err) { if err := haveDir(filepath.Dir(pth)); err != nil { return err } return os.Mkdir(pth, 0777) } return err } ================================================ FILE: storage/funcs.go ================================================ package storage import ( "bytes" "context" "fmt" "io" ) /* This file contains equivalents of every method that can be feature-detected on a storage system. You can always call these functions, and give them the most basic storage interface, and they'll attempt to feature-detect their way to the best possible implementation of the behavior, or they'll fall back to synthesizing the same behavior from more basic interfaces. Long story short: you can always use these functions as an end user, and get the behavior you want -- regardless of how much explicit support the storage implementation has for the exact behavior you requested. */ func Has(ctx context.Context, store Storage, key string) (bool, error) { // Okay, not much going on here -- this function is only here for consistency of style. return store.Has(ctx, key) } func Get(ctx context.Context, store ReadableStorage, key string) ([]byte, error) { // Okay, not much going on here -- this function is only here for consistency of style. return store.Get(ctx, key) } func Put(ctx context.Context, store WritableStorage, key string, content []byte) error { // Okay, not much going on here -- this function is only here for consistency of style. return store.Put(ctx, key, content) } // GetStream returns a streaming reader. // This function will feature-detect the StreamingReadableStorage interface, and use that if possible; // otherwise it will fall back to using basic ReadableStorage methods transparently // (at the cost of loading all the data into memory at once and up front). func GetStream(ctx context.Context, store ReadableStorage, key string) (io.ReadCloser, error) { // Prefer the feature itself, first. if streamable, ok := store.(StreamingReadableStorage); ok { return streamable.GetStream(ctx, key) } // Fallback to basic. blob, err := store.Get(ctx, key) return noopCloser{bytes.NewReader(blob)}, err } // PutStream returns an io.Writer and a WriteCommitter callback. // (See the docs on StreamingWritableStorage.PutStream for details on what that means.) // This function will feature-detect the StreamingWritableStorage interface, and use that if possible; // otherwise it will fall back to using basic WritableStorage methods transparently // (at the cost of needing to buffer all of the content in memory while the write is in progress). func PutStream(ctx context.Context, store WritableStorage) (io.Writer, func(key string) error, error) { // Prefer the feature itself, first. if streamable, ok := store.(StreamingWritableStorage); ok { return streamable.PutStream(ctx) } // Fallback to basic. var buf bytes.Buffer var written bool return &buf, func(key string) error { if written { return fmt.Errorf("WriteCommitter already used") } written = true return store.Put(ctx, key, buf.Bytes()) }, nil } // PutVec is an API for writing several slices of bytes at once into storage. // This kind of API can be useful for maximizing performance in scenarios where // data is already loaded completely into memory, but scattered across several non-contiguous regions. // This function will feature-detect the VectorWritableStorage interface, and use that if possible; // otherwise it will fall back to using StreamingWritableStorage, // or failing that, fall further back to basic WritableStorage methods, transparently. func PutVec(ctx context.Context, store WritableStorage, key string, blobVec [][]byte) error { // Prefer the feature itself, first. if putvable, ok := store.(VectorWritableStorage); ok { return putvable.PutVec(ctx, key, blobVec) } // Fallback to streaming mode. // ... or, fallback to basic, and use emulated streaming. Still presumably preferable to doing a big giant memcopy. // Conveniently, the PutStream function makes that transparent for our implementation, too. wr, wrcommit, err := PutStream(ctx, store) if err != nil { return err } for _, blob := range blobVec { _, err := wr.Write(blob) if err != nil { return err } } return wrcommit(key) } // Peek accessess the same data as Get, but indicates that the caller promises not to mutate the returned byte slice. // (By contrast, Get is expected to return a safe copy.) // This function will feature-detect the PeekableStorage interface, and use that if possible; // otherwise it will fall back to using basic ReadableStorage methods transparently // (meaning that a no-copy fastpath simply wasn't available). // // An io.Closer is returned along with the byte slice. // The Close method on the Closer must be called when the caller is done with the byte slice; // otherwise, memory leaks may result. // (Implementers of this interface may be expecting to reuse the byte slice after Close is called.) func Peek(ctx context.Context, store ReadableStorage, key string) ([]byte, io.Closer, error) { // Prefer the feature itself, first. if peekable, ok := store.(PeekableStorage); ok { return peekable.Peek(ctx, key) } // Fallback to basic. bs, err := store.Get(ctx, key) return bs, noopCloser{nil}, err } type noopCloser struct { io.Reader } func (noopCloser) Close() error { return nil } ================================================ FILE: storage/memstore/memstore.go ================================================ package memstore import ( "bytes" "context" "fmt" "io" ) // Store is a simple in-memory storage. // (It's little more than a map -- in fact, the map is exported, // and you can poke it directly.) // // Store conforms to the storage.ReadableStorage and storage.WritableStorage APIs. // Additionally, it supports storage.PeekableStorage and storage.StreamingReadableStorage, // because it can do so while provoking fewer copies. // // If you want to use this store with streaming APIs, // you can still do so by using the functions in the storage package, // such as storage.GetStream and storage.PutStream, which will synthesize the correct behavior. // // You can use this storage with a linking.LinkSystem easily, // by using the LinkSystem.SetReadStorage and/or LinkSystem.SetWriteStorage methods. // // There are no construction parameters for sharding functions nor escaping functions. // Any keys are acceptable. // // This storage is mostly expected to be used for testing and demos, // and as an example of how you can implement and integrate your own storage systems. // It does not provide persistence beyond memory. type Store struct { Bag map[string][]byte } func (store *Store) beInitialized() { if store.Bag != nil { return } store.Bag = make(map[string][]byte) } // Has implements go-ipld-prime/storage.Storage.Has. func (store *Store) Has(ctx context.Context, key string) (bool, error) { if store.Bag == nil { return false, nil } _, exists := store.Bag[key] return exists, nil } // Get implements go-ipld-prime/storage.ReadableStorage.Get. // // Note that this internally performs a defensive copy; // use Peek for higher performance if you are certain you won't mutate the returned slice. func (store *Store) Get(ctx context.Context, key string) ([]byte, error) { store.beInitialized() content, exists := store.Bag[key] if !exists { return nil, fmt.Errorf("404") // FIXME this needs a standard error type } cpy := make([]byte, len(content)) copy(cpy, content) return cpy, nil } // Put implements go-ipld-prime/storage.WritableStorage.Put. func (store *Store) Put(ctx context.Context, key string, content []byte) error { store.beInitialized() if _, exists := store.Bag[key]; exists { return nil } cpy := make([]byte, len(content)) copy(cpy, content) store.Bag[key] = cpy return nil } // GetStream implements go-ipld-prime/storage.StreamingReadableStorage.GetStream. // // It's useful for this storage implementation to explicitly support this, // because returning a reader gives us room to avoid needing a defensive copy. func (store *Store) GetStream(ctx context.Context, key string) (io.ReadCloser, error) { content, exists := store.Bag[key] if !exists { return nil, fmt.Errorf("404") // FIXME this needs a standard error type } return noopCloser{bytes.NewReader(content)}, nil } // Peek implements go-ipld-prime/storage.PeekableStorage.Peek. func (store *Store) Peek(ctx context.Context, key string) ([]byte, io.Closer, error) { content, exists := store.Bag[key] if !exists { return nil, nil, fmt.Errorf("404") // FIXME this needs a standard error type } return content, noopCloser{nil}, nil } type noopCloser struct { io.Reader } func (noopCloser) Close() error { return nil } ================================================ FILE: storage/sharding/sharding.go ================================================ /* This package contains several useful readymade sharding functions, which should plug nicely into most storage implementations. The API contract for a sharding function is: func(key string, shards *[]string) In other words, the return is actually by a pointer to a slice which will be mutated. This API allows the calling code to hand in a slice with existing capacity, and thus allows for sharding functions to work without allocations. There is not a named type for this contract, because we prefer that packages implementing the storage APIs should be possible to write without being required to import any code from the go-ipld-prime module. However, the function type definition above can be seen in many packages. Not all packages use this API convention. The `fsstore` package does; some other storage implementations don't use sharding functions because they don't need them; most of the adapter packages which target older code do not, because those modules have their own sharding APIs already. */ package sharding // Shard_r133 is a sharding function which will return three hunks, // the last of which is the full original key, // and the first two of which are three bytes long. // The prefix hunks are taken from the end of the original key, // after skipping one byte. // If the key is too short, padding of the ascii "0" character is used. // // (This somewhat odd-sounding procedure is a useful one in practice, // because if applying it on a base32 string that's a CID or multihash (which is the typical usage), // it avoids the uneven distribution of the trailing characters of a base32 string, // and also avoids the uneven distribution of the prefixes of CIDs or mulithashes.) // // If the shards parameter is a pointer to a slice that starts at zero length // and a capacity of at least 3, this function will operate with no allocations. // // Supposing the key is a base32 string (where each byte effectively contains 2^5 bits), // if a sufficient range of keys is present that all shards are seen, // each group of shards will contain (2^5)^3=32768 entries. func Shard_r133(key string, shards *[]string) { l := len(key) switch { case l > 6: *shards = append(*shards, key[l-7:l-4], key[l-4:l-1], key) case l > 3: *shards = append(*shards, "000", key[l-4:l-1], key) default: *shards = append(*shards, "000", "000", key) } } // Shard_r133 is a sharding function which will return three hunks. // It is very similar to Shard_r133, but with shorter hunks. // The last hunk is the full original key, // and the first two hunks are two bytes long each. // The prefix hunks are taken from the end of the original key, // after skipping one byte. // If the key is too short, padding of the ascii "0" character is used. // // If the shards parameter is a pointer to a slice that starts at zero length // and a capacity of at least 3, this function will operate with no allocations. // // Supposing the key is a base32 string (where each byte effectively contains 2^5 bits), // if a sufficient range of keys is present that all shards are seen, // each group of shards will contain (2^5)^2=1024 entries. // (This is often a useful number in practice, because if one is mapping shards // onto filesystem directories, 1024 entries is almost certainly going to fit // efficiently within any filesystem format you're likely to encounter; // 1024-within-1024 also means you'll see about a billion entries before // directories on the second layer of sharding will contain more than 1024 files. // (If we're assuming 1MB blocks of data asthe actual contents, that would be quite // a few terabytes of storage, so this is a very nice balanced trade for // most practical systems.)) func Shard_r122(key string, shards *[]string) { l := len(key) switch { case l > 4: *shards = append(*shards, key[l-5:l-3], key[l-3:l-1], key) case l > 2: *shards = append(*shards, "00", key[l-3:l-1], key) default: *shards = append(*shards, "00", "00", key) } } // Shard_r12 is a sharding function which will return two hunks. // The last hunk is the full original key, // and the first hunk is two bytes long. // The prefix is are taken from the end of the original key, // after skipping one byte. // If the key is too short, the first hunk is just the ascii characters "00" instead. // // If the shards parameter is a pointer to a slice that starts at zero length // and a capacity of at least 2, this function will operate with no allocations. // // Shard_r122 is functionally equivalent to "flatfs/shard/v1/next-to-last/2", // as it's known in some other code -- it may be familiar as the default // for block storage in go-ipfs. func Shard_r12(key string, shards *[]string) { l := len(key) switch { case l > 2: *shards = append(*shards, key[l-3:l-1], key) default: *shards = append(*shards, "00", key) } } ================================================ FILE: storage/sharding/sharding_bench_test.go ================================================ package sharding import ( "testing" ) var globalSink interface{} // This doesn't benchmark each of the sharding functions because... they're all roughly the same, really. // It's mainly to make sure that our documentation's claim about zero-alloc operation is true. func Benchmark(b *testing.B) { b.ReportAllocs() k := "abcdefgh" v := make([]string, 0, 3) var sink string for n := 0; n < b.N; n++ { v = v[0:0] Shard_r133(k, &v) sink = v[1] } globalSink = sink // make very very sure the compiler can't optimize our 'v' into oblivion. } ================================================ FILE: storage/sharding/sharding_test.go ================================================ package sharding import ( "fmt" "path" ) func printShard(fn func(string, *[]string), key string) { v := make([]string, 0, 4) fn(key, &v) fmt.Printf("%s => %s\n", key, path.Join(v...)) } func Example_shard_R133() { printShard(Shard_r133, "abcdefgh") printShard(Shard_r133, "abcdefg") printShard(Shard_r133, "abcdef") printShard(Shard_r133, "abcde") printShard(Shard_r133, "abcd") printShard(Shard_r133, "abc") // Output: // abcdefgh => bcd/efg/abcdefgh // abcdefg => abc/def/abcdefg // abcdef => 000/cde/abcdef // abcde => 000/bcd/abcde // abcd => 000/abc/abcd // abc => 000/000/abc } func Example_shard_r122() { printShard(Shard_r122, "abcdefgh") printShard(Shard_r122, "abcdefg") printShard(Shard_r122, "abcdef") printShard(Shard_r122, "abcde") printShard(Shard_r122, "abcd") printShard(Shard_r122, "abc") // Output: // abcdefgh => de/fg/abcdefgh // abcdefg => cd/ef/abcdefg // abcdef => bc/de/abcdef // abcde => ab/cd/abcde // abcd => 00/bc/abcd // abc => 00/ab/abc } func Example_shard_r12() { printShard(Shard_r12, "abcde") printShard(Shard_r12, "abcd") printShard(Shard_r12, "abc") printShard(Shard_r12, "ab") // Output: // abcde => cd/abcde // abcd => bc/abcd // abc => ab/abc // ab => 00/ab } ================================================ FILE: storage/tests/benchmarks.go ================================================ package tests import ( "context" "testing" "github.com/ipld/go-ipld-prime/storage" ) /* General note: It's important to be careful to benchmark the cost *per op* -- and not mistake b.N for the scale of corpus to work on. The corpus size should be a parameter that you supply, and your benchmark table should have a column for them! */ func BenchPut(b *testing.B, store storage.WritableStorage, gen Gen, scale int) { b.ReportAllocs() b.Logf("benchmarking with b.N=%d", b.N) // Use a fixed context throughout; it's not really relevant. ctx := context.Background() // Setup phase: create data up to the scale provided. // Reset the timer afterwards. b.Logf("prepopulating %d entries into storage...", scale) for n := 0; n < scale; n++ { key, content := gen() err := store.Put(ctx, key, content) if err != nil { b.Fatal(err) } } b.Logf("prepopulating %d entries into storage: done.", scale) b.ResetTimer() // Now continue doing puts in the benchmark loop. // Note that if 'scale' was initially small, and b.N is big, results may be skewed, // because the last put of the series will actually be working at scale+b.N-1. for n := 0; n < b.N; n++ { // Attempt to avoid counting any time spent by the gen func. // ... except don't, because the overhead of starting and stopping is actually really high compared to a likely gen function; // in practice, starting and stopping this frequently causes: // - alloc count to be reported *correctly* (which is nice) // - but reported ns/op to become erratic, and inflated (not at all nice) // - and actual wall-clock run time to increase drastically (~22x!) (not deadly, but certainly unpleasant) // It may be best to write a synthetic dummy benchmark to see how much the gen function costs, and subtract that from the other results. //b.StopTimer() key, content := gen() //b.StartTimer() // Do the put. err := store.Put(ctx, key, content) if err != nil { b.Fatal(err) } } } ================================================ FILE: storage/tests/generators.go ================================================ package tests import ( "strconv" ) // Gen is a func which should generate key-value pairs. // It's used to configure benchmarks. // // How exactly to use this is up to you, but // a good gen function should probably return a wide variety of keys, // and some known distribution of key and content sizes. // If it returns the same key frequently, it should be documented, // because key collision rates will affect benchmark results. type Gen func() (key string, content []byte) // NewCounterGen returns a Gen func which yields a unique value on each subsequent call, // which is simply the base-10 string representation of an incrementing integer. // The content and the key are the same. func NewCounterGen(start int64) Gen { return func() (key string, content []byte) { k := strconv.FormatInt(start, 10) start++ return k, []byte(k) } } ================================================ FILE: testutil/garbage/garbage.go ================================================ package garbage import ( "math" mathrand "math/rand" "strings" "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime/datamodel" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipld/go-ipld-prime/must" basicnode "github.com/ipld/go-ipld-prime/node/basic" "github.com/multiformats/go-multihash" ) type Options struct { initialWeights map[datamodel.Kind]int weights map[datamodel.Kind]int blockSize uint64 } type generator func(rand *mathrand.Rand, count uint64, opts Options) (uint64, datamodel.Node) type hasher struct { code uint64 length int } const charset = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789`~!@#$%^&*()-_=+[]{}|\\:;'\",.<>?/ \t\n☺💩" var ( codecs = []uint64{0x55, 0x70, 0x71, 0x0129} hashes = []hasher{{0x12, 256}, {0x16, 256}, {0x1b, 256}, {0xb220, 256}, {0x13, 512}, {0x15, 384}, {0x14, 512}} kinds = append(datamodel.KindSet_Scalar, datamodel.KindSet_Recursive...) runes = []rune(charset) generators map[datamodel.Kind]generator ) // Generate produces random Nodes which can be useful for testing and benchmarking. By default, the // Nodes produced are relatively small, averaging near the 1024 byte range when encoded // (very roughly, with a wide spread). // // Options can be used to adjust the average size and weights of occurrences of different kinds // within the complete Node graph. // // Care should be taken when using a random source to generate garbage for testing purposes, that // the randomness is stable across test runs, or a seed is captured in such a way that a failure // can be reproduced (e.g. by printing it to stdout during the test run so it can be captured in // CI for a failure). func Generate(rand *mathrand.Rand, opts ...Option) datamodel.Node { options := applyOptions(opts...) _, n := generate(rand, options.blockSize, options) return n } func generate(rand *mathrand.Rand, count uint64, opts Options) (uint64, datamodel.Node) { weights := opts.weights if opts.initialWeights != nil { weights = opts.initialWeights opts = Options{weights: opts.weights} } totWeight := 0 for _, kind := range kinds { totWeight += weights[kind] } r := rand.Float64() * float64(totWeight) var wacc int for _, kind := range kinds { wacc += weights[kind] if float64(wacc) >= r { return generators[kind](rand, count, opts) } } panic("bad options") } func rndSize(rand *mathrand.Rand, bias uint64) uint64 { if bias == 0 { panic("size shouldn't be zero") } mean := float64(bias) stdev := mean / 10 for { s := math.Abs(rand.NormFloat64())*stdev + mean if s >= 1 { return uint64(s) } } } func rndRune(rand *mathrand.Rand) rune { return runes[rand.Intn(len(runes))] } func listGenerator(rand *mathrand.Rand, count uint64, opts Options) (uint64, datamodel.Node) { len := rndSize(rand, 10) lb := basicnode.Prototype.List.NewBuilder() la, err := lb.BeginList(int64(len)) if err != nil { panic(err) } size := uint64(0) for i := uint64(0); i < len && size < count; i++ { c, n := generate(rand, count-size, opts) err := la.AssembleValue().AssignNode(n) if err != nil { panic(err) } size += c } err = la.Finish() if err != nil { panic(err) } return size, lb.Build() } func mapGenerator(rand *mathrand.Rand, count uint64, opts Options) (uint64, datamodel.Node) { length := rndSize(rand, 10) mb := basicnode.Prototype.Map.NewBuilder() ma, err := mb.BeginMap(int64(length)) if err != nil { panic(err) } size := uint64(0) keys := make(map[string]struct{}) for i := uint64(0); i < length && size < count; i++ { var key string for { c, k := stringGenerator(rand, 5, opts) key = must.String(k) if _, ok := keys[key]; !ok && len(key) > 0 { keys[key] = struct{}{} size += c break } } sz := count - size if size >= count { // the case where we've blown our budget already on the key sz = 5 } c, value := generate(rand, sz, opts) size += c err := ma.AssembleKey().AssignString(key) if err != nil { panic(err) } err = ma.AssembleValue().AssignNode(value) if err != nil { panic(err) } } err = ma.Finish() if err != nil { panic(err) } return size, mb.Build() } func stringGenerator(rand *mathrand.Rand, count uint64, opts Options) (uint64, datamodel.Node) { len := rndSize(rand, count/2+1) sb := strings.Builder{} for i := uint64(0); i < len; i++ { sb.WriteRune(rndRune(rand)) } return len, basicnode.NewString(sb.String()) } func bytesGenerator(rand *mathrand.Rand, count uint64, opts Options) (uint64, datamodel.Node) { len := rndSize(rand, count/2+1) ba := make([]byte, len) _, err := rand.Read(ba) if err != nil { panic(err) } return len, basicnode.NewBytes(ba) } func boolGenerator(rand *mathrand.Rand, count uint64, opts Options) (uint64, datamodel.Node) { return 0, basicnode.NewBool(rand.Float64() > 0.5) } func intGenerator(rand *mathrand.Rand, count uint64, opts Options) (uint64, datamodel.Node) { i := rand.Int63() if rand.Float64() > 0.5 { i = -i } return 0, basicnode.NewInt(i) } func floatGenerator(rand *mathrand.Rand, count uint64, opts Options) (uint64, datamodel.Node) { return 0, basicnode.NewFloat(math.Tan((rand.Float64() - 0.5) * math.Pi)) } func nullGenerator(rand *mathrand.Rand, count uint64, opts Options) (uint64, datamodel.Node) { return 0, datamodel.Null } func linkGenerator(rand *mathrand.Rand, count uint64, opts Options) (uint64, datamodel.Node) { hasher := hashes[rand.Intn(len(hashes))] codec := codecs[rand.Intn(len(codecs))] ba := make([]byte, hasher.length/8) rand.Read(ba) mh, err := multihash.Encode(ba, hasher.code) if err != nil { panic(err) } return uint64(hasher.length / 8), basicnode.NewLink(cidlink.Link{Cid: cid.NewCidV1(codec, mh)}) } type Option func(*Options) func applyOptions(opt ...Option) Options { opts := Options{ blockSize: 1024, initialWeights: DefaultInitialWeights(), weights: DefaultWeights(), } for _, o := range opt { o(&opts) } return opts } // DefaultInitialWeights provides the default map of weights that can be // overridden by the InitialWeights option. The default is an equal weighting // of 1 for every scalar kind and 10 for the recursive kinds. func DefaultInitialWeights() map[datamodel.Kind]int { return map[datamodel.Kind]int{ datamodel.Kind_List: 10, datamodel.Kind_Map: 10, datamodel.Kind_Bool: 1, datamodel.Kind_Bytes: 1, datamodel.Kind_Float: 1, datamodel.Kind_Int: 1, datamodel.Kind_Link: 1, datamodel.Kind_Null: 1, datamodel.Kind_String: 1, } } // DefaultWeights provides the default map of weights that can be overridden by // the Weights option. The default is an equal weighting of 1 for every kind. func DefaultWeights() map[datamodel.Kind]int { return map[datamodel.Kind]int{ datamodel.Kind_List: 1, datamodel.Kind_Map: 1, datamodel.Kind_Bool: 1, datamodel.Kind_Bytes: 1, datamodel.Kind_Float: 1, datamodel.Kind_Int: 1, datamodel.Kind_Link: 1, datamodel.Kind_Null: 1, datamodel.Kind_String: 1, } } // InitialWeights sets a per-kind weighting for the root node. That is, the weights // set here will determine the liklihood of the returned Node's direct .Kind(). // These weights are ignored after the top-level Node (for recursive kinds, // obviously for scalar kinds there is only a top-level Node). // // The default initial weights bias toward Map and List kinds, by a ratio of // 10:1—i.e. the recursive kinds are more likely to appear at the top-level. func InitialWeights(initialWeights map[datamodel.Kind]int) Option { return func(o *Options) { o.initialWeights = initialWeights } } // Weights sets a per-kind weighting for nodes appearing throughout the returned // graph. When assembling a graph, these weights determine the liklihood that // a given kind will be selected for that node. // // A weight of 0 will turn that kind off entirely. So, for example, if you // wanted output data with no maps or bytes, then set both of those weights to // zero, leaving the rest >0 and do the same for InitialWeights. // // The default weights are set to 1—i.e. there is an equal liklihood that any of // the valid kinds will be selected for any point in the graph. // // This option is overridden by InitialWeights (which also has a default even // if not set explicitly) for the top-level node. func Weights(weights map[datamodel.Kind]int) Option { return func(o *Options) { o.weights = weights } } // TargetBlockSize sets a very rough bias in number of bytes that the resulting // Node may consume when encoded (i.e. the block size). This is a very // approximate measure, but over enough repeated Generate() calls, the resulting // Nodes, once encoded, should have a median that is somewhere in this vicinity. // // The default target block size is 1024. This should be tuned in accordance with // the anticipated average block size of the system under test. func TargetBlockSize(blockSize uint64) Option { return func(o *Options) { o.blockSize = blockSize } } func init() { // can't be declared statically because of some cycles through list & map to generate() generators = map[datamodel.Kind]generator{ datamodel.Kind_List: listGenerator, datamodel.Kind_Map: mapGenerator, datamodel.Kind_String: stringGenerator, datamodel.Kind_Bytes: bytesGenerator, datamodel.Kind_Bool: boolGenerator, datamodel.Kind_Int: intGenerator, datamodel.Kind_Float: floatGenerator, datamodel.Kind_Null: nullGenerator, datamodel.Kind_Link: linkGenerator, } } ================================================ FILE: testutil/garbage/garbage_test.go ================================================ package garbage import ( "bytes" "math/rand" "testing" "time" qt "github.com/frankban/quicktest" "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/codec/dagcbor" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/node/basicnode" ) func TestGarbageProducesAllKinds(t *testing.T) { kindCount := make(map[datamodel.Kind]int) seed := time.Now().Unix() t.Logf("randomness seed: %v\n", seed) rnd := rand.New(rand.NewSource(seed)) for i := 0; i < 10000; i++ { gbg := Generate(rnd) kindCount[gbg.Kind()]++ } for _, kind := range append(datamodel.KindSet_Scalar, datamodel.KindSet_Recursive...) { qt.Assert(t, kindCount[kind], qt.Not(qt.Equals), 0) } } func TestGarbageProducesValidNodes(t *testing.T) { // round-trip through a codec should pick up most possible problems with Node validity seed := time.Now().Unix() t.Logf("randomness seed: %v\n", seed) rnd := rand.New(rand.NewSource(seed)) for i := 0; i < 1000; i++ { var buf bytes.Buffer gbg := Generate(rnd) err := dagcbor.Encode(gbg, &buf) qt.Assert(t, err, qt.IsNil) nb := basicnode.Prototype.Any.NewBuilder() err = dagcbor.Decode(nb, &buf) qt.Assert(t, err, qt.IsNil) ipld.DeepEqual(gbg, nb.Build()) } } func TestGarbageProducesSameDataForSameRandomSource(t *testing.T) { gbg1 := Generate(rand.New(rand.NewSource(1))) gbg2 := Generate(rand.New(rand.NewSource(1))) qt.Assert(t, ipld.DeepEqual(gbg1, gbg2), qt.IsTrue) } func TestGarbageProducesSingleKind(t *testing.T) { seed := time.Now().Unix() t.Logf("randomness seed: %v\n", seed) rnd := rand.New(rand.NewSource(seed)) for _, kind := range append(datamodel.KindSet_Scalar, datamodel.KindSet_Recursive...) { t.Run(kind.String(), func(t *testing.T) { kindCount := make(map[datamodel.Kind]int) for i := 0; i < 1000; i++ { gbg := Generate(rnd, InitialWeights(map[datamodel.Kind]int{kind: 1})) kindCount[gbg.Kind()]++ } for _, k := range append(datamodel.KindSet_Scalar, datamodel.KindSet_Recursive...) { if k == kind { qt.Assert(t, kindCount[k], qt.Equals, 1000) } else { qt.Assert(t, kindCount[k], qt.Equals, 0) } } }) } } ================================================ FILE: testutil/indent.go ================================================ package testutil import "bytes" // Dedent strips leading tabs from every line of a string, taking a hint of // how many tabs should be stripped from the number of consecutive tabs found // on the first non-empty line. Dedent also strips one leading blank // line if it contains nothing but the linebreak. // // If later lines have fewer leading tab characters than the depth we intuited // from the first line, then stripping will still only remove tab characters. // // Roughly, Dedent is "Do What I Mean" to normalize a heredoc string // that contains leading indentation to make it congruent with the // surrounding source code. func Dedent(s string) string { // Originally from: https://github.com/warpfork/go-wish/blob/master/indent.go // Forked here to reduce dependencies in go-ipld-prime. return string(DedentBytes([]byte(s))) } // DedentBytes is identically to Dedent, but works on a byte slice. func DedentBytes(bs []byte) []byte { lines := bytes.SplitAfter(bs, []byte{'\n'}) buf := bytes.Buffer{} if len(lines[0]) == 1 && lines[0][0] == '\n' { lines = lines[1:] } if len(lines) == 0 { return []byte{} } depth := 0 for _, r := range lines[0] { depth++ if r != '\t' { depth-- break } } for _, line := range lines { for i, r := range line { if i < depth && r == '\t' { continue } buf.Write(line[i:]) break } } return buf.Bytes() } ================================================ FILE: testutil/indent_test.go ================================================ package testutil import ( "testing" qt "github.com/frankban/quicktest" ) func TestDedent(t *testing.T) { for _, tr := range []struct{ a, b string }{ {"", ""}, {"\t", ""}, {"\t\t", ""}, {"\n", ""}, {"\n\t", ""}, {"\n\t\t", ""}, {"\n\n", "\n"}, {"\n\t\n", "\n"}, {"\n\t\t\n", "\n"}, {"\n\n", "\n"}, {"\n\n\t", "\n\t"}, {"\n\n\t\t", "\n\t\t"}, {"a\nb\n\tc\n", "a\nb\n\tc\n"}, {"\ta\nb\n\tc\n", "a\nb\nc\n"}, {"\t\ta\nb\n\tc\n", "a\nb\nc\n"}, {"\ta\n\t\tb\n\tc\n", "a\n\tb\nc\n"}, {"\ta\n\t\t\tb\n\tc\n", "a\n\t\tb\nc\n"}, {"\n\t\t\ta\n\t\tb\n\t\t\t\n\t\t\t\tc\n\t\t", "a\nb\n\n\tc\n"}, } { actual := Dedent(tr.a) qt.Assert(t, actual, qt.Equals, tr.b) } } ================================================ FILE: testutil/multibytenode.go ================================================ package testutil import ( "io" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/node/basicnode" ) var _ datamodel.Node = MultiByteNode{} var _ datamodel.LargeBytesNode = (*MultiByteNode)(nil) // MultiByteNode is a node that is a concatenation of multiple byte slices. // It's not particularly sophisticated but lets us exercise LargeBytesNode in a // non-trivial way. // The novel behaviour of Read() and Seek() on the AsLargeBytes is similar to // that which would be expected from a LBN ADL, such as UnixFS sharded files. type MultiByteNode struct { bytes [][]byte } func NewMultiByteNode(bytes ...[]byte) MultiByteNode { return MultiByteNode{bytes: bytes} } func (mbn MultiByteNode) Kind() datamodel.Kind { return datamodel.Kind_Bytes } func (mbn MultiByteNode) AsBytes() ([]byte, error) { ret := make([]byte, 0, mbn.TotalLength()) for _, b := range mbn.bytes { ret = append(ret, b...) } return ret, nil } func (mbn MultiByteNode) TotalLength() int { var size int for _, b := range mbn.bytes { size += len(b) } return size } func (mbn MultiByteNode) AsLargeBytes() (io.ReadSeeker, error) { return &mbnReadSeeker{node: mbn}, nil } func (mbn MultiByteNode) AsBool() (bool, error) { return false, datamodel.ErrWrongKind{TypeName: "bool", MethodName: "AsBool", AppropriateKind: datamodel.KindSet_JustBytes} } func (mbn MultiByteNode) AsInt() (int64, error) { return 0, datamodel.ErrWrongKind{TypeName: "int", MethodName: "AsInt", AppropriateKind: datamodel.KindSet_JustBytes} } func (mbn MultiByteNode) AsFloat() (float64, error) { return 0, datamodel.ErrWrongKind{TypeName: "float", MethodName: "AsFloat", AppropriateKind: datamodel.KindSet_JustBytes} } func (mbn MultiByteNode) AsString() (string, error) { return "", datamodel.ErrWrongKind{TypeName: "string", MethodName: "AsString", AppropriateKind: datamodel.KindSet_JustBytes} } func (mbn MultiByteNode) AsLink() (datamodel.Link, error) { return nil, datamodel.ErrWrongKind{TypeName: "link", MethodName: "AsLink", AppropriateKind: datamodel.KindSet_JustBytes} } func (mbn MultiByteNode) AsNode() (datamodel.Node, error) { return nil, nil } func (mbn MultiByteNode) Size() int { return 0 } func (mbn MultiByteNode) IsAbsent() bool { return false } func (mbn MultiByteNode) IsNull() bool { return false } func (mbn MultiByteNode) Length() int64 { return 0 } func (mbn MultiByteNode) ListIterator() datamodel.ListIterator { return nil } func (mbn MultiByteNode) MapIterator() datamodel.MapIterator { return nil } func (mbn MultiByteNode) LookupByIndex(idx int64) (datamodel.Node, error) { return nil, datamodel.ErrWrongKind{} } func (mbn MultiByteNode) LookupByString(key string) (datamodel.Node, error) { return nil, datamodel.ErrWrongKind{} } func (mbn MultiByteNode) LookupByNode(key datamodel.Node) (datamodel.Node, error) { return nil, datamodel.ErrWrongKind{} } func (mbn MultiByteNode) LookupBySegment(seg datamodel.PathSegment) (datamodel.Node, error) { return nil, datamodel.ErrWrongKind{} } func (mbn MultiByteNode) Prototype() datamodel.NodePrototype { return basicnode.Prototype.Bytes // not really ... but it'll do for this test } type mbnReadSeeker struct { node MultiByteNode offset int } func (mbnrs *mbnReadSeeker) Read(p []byte) (int, error) { var acc int for _, byts := range mbnrs.node.bytes { if mbnrs.offset-acc >= len(byts) { acc += len(byts) continue } n := copy(p, byts[mbnrs.offset-acc:]) mbnrs.offset += n return n, nil } return 0, io.EOF } func (mbnrs *mbnReadSeeker) Seek(offset int64, whence int) (int64, error) { switch whence { case io.SeekStart: mbnrs.offset = int(offset) case io.SeekCurrent: mbnrs.offset += int(offset) case io.SeekEnd: mbnrs.offset = mbnrs.node.TotalLength() + int(offset) } return int64(mbnrs.offset), nil } ================================================ FILE: testutil/multibytenode_test.go ================================================ package testutil_test import ( "io" "testing" qt "github.com/frankban/quicktest" "github.com/ipld/go-ipld-prime/testutil" ) func TestMultiByteNode(t *testing.T) { mbn := testutil.NewMultiByteNode( []byte("foo"), []byte("bar"), []byte("baz"), []byte("!"), ) // Sanity check that the readseeker works. // (This is a test of the test, not the code under test.) for _, rl := range []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10} { t.Run("readseeker works with read length "+qt.Format(rl), func(t *testing.T) { rs, err := mbn.AsLargeBytes() qt.Assert(t, err, qt.IsNil) acc := make([]byte, 0, mbn.TotalLength()) buf := make([]byte, rl) for { n, err := rs.Read(buf) if err == io.EOF { qt.Check(t, n, qt.Equals, 0) break } qt.Assert(t, err, qt.IsNil) acc = append(acc, buf[0:n]...) } qt.Assert(t, string(acc), qt.DeepEquals, "foobarbaz!") }) } t.Run("readseeker can seek and read middle bytes", func(t *testing.T) { rs, err := mbn.AsLargeBytes() qt.Assert(t, err, qt.IsNil) _, err = rs.Seek(2, io.SeekStart) qt.Assert(t, err, qt.IsNil) buf := make([]byte, 2) acc := make([]byte, 0, 5) for len(acc) < 5 { n, err := rs.Read(buf) qt.Assert(t, err, qt.IsNil) acc = append(acc, buf[0:n]...) } qt.Assert(t, string(acc), qt.DeepEquals, "obarba") }) t.Run("readseeker can seek and read last byte", func(t *testing.T) { rs, err := mbn.AsLargeBytes() qt.Assert(t, err, qt.IsNil) _, err = rs.Seek(-1, io.SeekEnd) qt.Assert(t, err, qt.IsNil) buf := make([]byte, 1) n, err := rs.Read(buf) qt.Assert(t, err, qt.IsNil) qt.Check(t, n, qt.Equals, 1) qt.Check(t, string(buf[0]), qt.Equals, "!") }) } ================================================ FILE: testutil/simplebytes.go ================================================ package testutil import ( "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/node/mixins" ) var _ datamodel.Node = simpleBytes(nil) // simpleBytes is like basicnode's plainBytes but it doesn't implement // LargeBytesNode so we can exercise the non-LBN case. type simpleBytes []byte // NewSimpleBytes is identical to basicnode.NewBytes but the returned node // doesn't implement LargeBytesNode, which can be useful for testing cases // where we want to exercise non-LBN code paths. func NewSimpleBytes(value []byte) datamodel.Node { v := simpleBytes(value) return &v } // -- Node interface methods --> func (simpleBytes) Kind() datamodel.Kind { return datamodel.Kind_Bytes } func (simpleBytes) LookupByString(string) (datamodel.Node, error) { return mixins.Bytes{TypeName: "bytes"}.LookupByString("") } func (simpleBytes) LookupByNode(key datamodel.Node) (datamodel.Node, error) { return mixins.Bytes{TypeName: "bytes"}.LookupByNode(nil) } func (simpleBytes) LookupByIndex(idx int64) (datamodel.Node, error) { return mixins.Bytes{TypeName: "bytes"}.LookupByIndex(0) } func (simpleBytes) LookupBySegment(seg datamodel.PathSegment) (datamodel.Node, error) { return mixins.Bytes{TypeName: "bytes"}.LookupBySegment(seg) } func (simpleBytes) MapIterator() datamodel.MapIterator { return nil } func (simpleBytes) ListIterator() datamodel.ListIterator { return nil } func (simpleBytes) Length() int64 { return -1 } func (simpleBytes) IsAbsent() bool { return false } func (simpleBytes) IsNull() bool { return false } func (simpleBytes) AsBool() (bool, error) { return mixins.Bytes{TypeName: "bytes"}.AsBool() } func (simpleBytes) AsInt() (int64, error) { return mixins.Bytes{TypeName: "bytes"}.AsInt() } func (simpleBytes) AsFloat() (float64, error) { return mixins.Bytes{TypeName: "bytes"}.AsFloat() } func (simpleBytes) AsString() (string, error) { return mixins.Bytes{TypeName: "bytes"}.AsString() } func (n simpleBytes) AsBytes() ([]byte, error) { return []byte(n), nil } func (simpleBytes) AsLink() (datamodel.Link, error) { return mixins.Bytes{TypeName: "bytes"}.AsLink() } func (simpleBytes) Prototype() datamodel.NodePrototype { return basicnode.Prototype__Bytes{} } ================================================ FILE: traversal/common.go ================================================ package traversal import ( "context" "fmt" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/linking" "github.com/ipld/go-ipld-prime/schema" ) // init sets all the values in TraveralConfig to reasonable defaults // if they're currently the zero value. // // Note that you're absolutely going to need to replace the // LinkLoader and LinkNodeBuilderChooser if you want automatic link traversal; // the defaults return error and/or panic. func (tc *Config) init() { if tc.Ctx == nil { tc.Ctx = context.Background() } if tc.LinkTargetNodePrototypeChooser == nil { tc.LinkTargetNodePrototypeChooser = func(lnk datamodel.Link, lnkCtx linking.LinkContext) (datamodel.NodePrototype, error) { if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { return tlnkNd.LinkTargetNodePrototype(), nil } return nil, fmt.Errorf("no LinkTargetNodePrototypeChooser configured") } } } func (prog *Progress) init() { if prog.Cfg == nil { prog.Cfg = &Config{} } prog.Cfg.init() if prog.Cfg.LinkVisitOnlyOnce { prog.SeenLinks = make(map[datamodel.Link]struct{}) } } // asPathSegment figures out how to coerce a node into a PathSegment. // If it's a typed node: we take its representation. (Could be a struct with some string representation.) // If it's a string or an int, that's it. // Any other case will panic. (If you're using this one keys returned by a MapIterator, though, you can ignore this possibility; // any compliant map implementation should've already rejected that data long ago, and should not be able to yield it to you from an iterator.) func asPathSegment(n datamodel.Node) datamodel.PathSegment { if n2, ok := n.(schema.TypedNode); ok { n = n2.Representation() } switch n.Kind() { case datamodel.Kind_String: s, _ := n.AsString() return datamodel.PathSegmentOfString(s) case datamodel.Kind_Int: i, _ := n.AsInt() return datamodel.PathSegmentOfInt(i) default: panic(fmt.Errorf("cannot get pathsegment from a %s", n.Kind())) } } ================================================ FILE: traversal/doc.go ================================================ // Package traversal provides functional utilities for traversing and // transforming IPLD graphs. // // Two primary types of traversal are implemented in this package: "Focus" and // "Walk". Both types have a "Transforming" variant, which supports mutation // through emulated copy-on-write tree rebuilding. // // Traversal operations use the Progress type for configuration and state // tracking. Helper functions such as Focus and Walk exist to avoid manual setup // of a Progress struct, but they cannot cross link boundaries without a // LinkSystem, which needs to be configured on the Progress struct. // // A typical traversal operation involves creating a Progress struct, setting up // the LinkSystem, and calling one of the Focus or Walk functions on the // Progress object. Various other configuration options are available when // traversing this way. // // # Focus // // "Focus" and "Get" functions provide syntactic sugar for using ipld.Path to // access Nodes deep within a graph. // // "FocusedTransform" resembles "Focus" but supports user-defined mutation using // its TransformFn. // // # Walk // // "Walk" functions perform a recursive walk of a Node graph, applying visitor // functions to matched parts of the graph. // // The selector sub-package offers a declarative mechanism for guiding // traversals and filtering relevant Nodes. // (Refer to the selector sub-package for more details.) // // "WalkLocal" is a special case of Walk that doesn't require a selector. It // walks a local graph, not crossing link boundaries, and calls its VisitFn for // each encountered Node. // // "WalkMatching" traverses according to a selector, calling the VisitFn for // each match based on the selector's matching rules. // // "WalkAdv" performs the same traversal as WalkMatching, but calls its // AdvVisitFn on every Node, regardless of whether it matches the selector. // // "WalkTransforming" resembles "WalkMatching" but supports user-defined // mutation using its TransformFn. // // # Usage Notes // // These functions work via callbacks, performing traversal and calling a // user-provided function with a handle to the reached Node(s). Further "Focus" // and "Walk" operations can be performed recursively within this callback if // desired. // // All traversal functions operate on a Progress object, except "WalkLocal", // which can be configured with a LinkSystem for automatic resolution and // loading of new Node trees when IPLD Links are encountered. // // The "*Transform" methods are best suited for point-mutation patterns. For // more general transformations, use the read-only systems (e.g., Focus, // Traverse) and handle accumulation in the visitor functions. // // A common use case for walking traversal is running a selector over a graph // and noting all the blocks it uses. This is achieved by configuring a // LinkSystem that can handle and observe block loads. Be aware that a selector // might visit the same block multiple times during a traversal, as IPLD graphs // often form "diamond patterns" with the same block referenced from multiple // locations. // // The LinkVisitOnlyOnce option can be used to avoid duplicate loads, but it // must be used carefully with non-trivial selectors, where repeat visits of // the same block may be essential for traversal or visit callbacks. // // A Budget can be set at the beginning of a traversal to limit the number of // Nodes and/or Links encountered before failing the traversal (with the // ErrBudgetExceeded error). // // The "Preloader" option provides a way to parallelize block loading in // environments where block loading is a high-latency operation (such as // fetching over the network). // The traversal operation itself is not parallel and will proceed strictly // according to path or selector order. However, a Preloader can be used to load // blocks asynchronously, and prepare the LinkSystem that the traversal is using // with already-loaded blocks. // // A Preloader and a Budget option can be used on the same traversal, BUT the // Preloader may not receive the same links that the traversal wants to load // from the LinkSystem. Use with care. See notes below. package traversal // Why only "point-mutation"? This use-case gets core library support because // it's both high utility and highly clear how to implement it. // More advanced transformations are nontrivial to provide generalized support // for, for three reasons: efficiency is hard; not all existing research into // categorical recursion schemes is necessarily applicable without modification // (efficient behavior in a merkle-tree context is not the same as efficient // behavior on uniform memory!); and we have the further compounding complexity // of the range of choices available for underlying Node implementation. // Therefore, attempts at generalization are not included here; handling these // issues in concrete cases is easy, so we call it an application logic concern. // However, exploring categorical recursion schemes as a library is encouraged!) ================================================ FILE: traversal/example_select_links_test.go ================================================ package traversal_test import ( "fmt" "github.com/ipld/go-ipld-prime" // Import all the codecs so we can use them in our example; each of these will // set themselves up in our multicodec registry so the LinkSystem can use // them for encoding (when a LinkPrototype says so) and decoding (when a CID's // codec code says so). _ "github.com/ipld/go-ipld-prime/codec/dagcbor" _ "github.com/ipld/go-ipld-prime/codec/dagjson" _ "github.com/ipld/go-ipld-prime/codec/raw" "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/fluent/qp" "github.com/ipld/go-ipld-prime/linking" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/traversal" "github.com/multiformats/go-multihash" ) func ExampleSelectLinks() { // Setup: make some blocks, store them in our memory store blocks := make([]cid.Cid, 0) // make 3 raw blocks c1 := encodeAndStore(rawlp, basicnode.NewBytes([]byte{0xca, 0xfe, 0xbe, 0xef})) blocks = append(blocks, c1) c2 := encodeAndStore(rawlp, basicnode.NewBytes([]byte{0xde, 0xad, 0xbe, 0xef})) blocks = append(blocks, c2) c3 := encodeAndStore(rawlp, basicnode.NewBytes([]byte{0xba, 0xad, 0xf0, 0x0d})) blocks = append(blocks, c3) // Pretend we're doing a dag-pb encode here but since we don't want to pull in // the dagpb package we'll do it as dag-json. This should use // dagpb.Type.PBNode if it were real. pbn, err := qp.BuildMap(basicnode.Prototype.Map, 1, func(ma datamodel.MapAssembler) { qp.MapEntry(ma, "Links", qp.List(3, func(la datamodel.ListAssembler) { qp.ListEntry(la, qp.Map(2, func(ma datamodel.MapAssembler) { qp.MapEntry(ma, "Name", qp.String("01")) qp.MapEntry(ma, "Hash", qp.Link(cidlink.Link{Cid: c1})) })) qp.ListEntry(la, qp.Map(2, func(ma datamodel.MapAssembler) { qp.MapEntry(ma, "Name", qp.String("02")) qp.MapEntry(ma, "Hash", qp.Link(cidlink.Link{Cid: c2})) })) qp.ListEntry(la, qp.Map(2, func(ma datamodel.MapAssembler) { qp.MapEntry(ma, "Name", qp.String("03")) qp.MapEntry(ma, "Hash", qp.Link(cidlink.Link{Cid: c3})) })) })) }) if err != nil { panic(err) } cpb := encodeAndStore(pblp, pbn) blocks = append(blocks, cpb) // make a dag-cbor block with a bunch of links in it, stored in various ways cbn, err := qp.BuildList(basicnode.Prototype.List, -1, func(la datamodel.ListAssembler) { qp.ListEntry(la, qp.String("not a link!")) qp.ListEntry(la, qp.Link(cidlink.Link{Cid: c1})) qp.ListEntry(la, qp.Link(cidlink.Link{Cid: c2})) qp.ListEntry(la, qp.Int(42)) qp.ListEntry(la, qp.Link(cidlink.Link{Cid: c3})) qp.ListEntry(la, qp.Map(-1, func(ma datamodel.MapAssembler) { qp.MapEntry(ma, "Boop!", qp.String("boop!")) qp.MapEntry(ma, "This is a dag-pb link:", qp.Link(cidlink.Link{Cid: cpb})) qp.MapEntry(ma, "And this one the same link:", qp.Link(cidlink.Link{Cid: cpb})) qp.MapEntry(ma, "But thus one is a raw link:", qp.Link(cidlink.Link{Cid: c2})) })) }) if err != nil { panic(err) } ccb := encodeAndStore(dclp, cbn) blocks = append(blocks, ccb) // Example code: load the blocks as datamodel.Node form and traverse // them using the SelectLinks function to find all the links for _, c := range blocks { // load Node form of the block n := loadNode(c) // Select all links from the node links, err := traversal.SelectLinks(n) if err != nil { panic(err) } fmt.Printf("%s (%d links)\n", c, len(links)) // Print the links for _, l := range links { fmt.Printf("\t➜ %s\n", l) } } // Demonstrating how we might do this if we don't have a LinkSystem but do // have the bytes of a block we want to traverse. // load bytes of the block byts := loadBytes(ccb) // decode the block into Node form decoder, err := cidlink.DefaultLinkSystem().DecoderChooser(cidlink.Link{Cid: ccb}) if err != nil { panic(err) } // we could just use `dagcbor.Decode` here, but by using the DecoderChooser // we make sure we get the right one for the CID. n, err := ipld.Decode(byts, decoder) if err != nil { panic(err) } // Select all links from the node links, err := traversal.SelectLinks(n) if err != nil { panic(err) } fmt.Printf("Manually decoded %s has %d links too, surprise!\n", ccb, len(links)) // Output: // bafkreicwcm4sqhux7ipwbwro2inf4vbjoprzwoqnl3t4zta4gphofot7du (0 links) // bafkreic7pdbte5heh6u54vszezob3el6exadoiw4wc4ne7ny2x7kvajzkm (0 links) // bafkreiafc5f36dkaocd6iwysxkxboelue2cs745j4wgrfihlxgqqwqexim (0 links) // baguqeeraawfdfutq7b4dgmcelypy4r36d7ndngbzpbejzhjaxffcrbgq3wka (3 links) // ➜ bafkreicwcm4sqhux7ipwbwro2inf4vbjoprzwoqnl3t4zta4gphofot7du // ➜ bafkreic7pdbte5heh6u54vszezob3el6exadoiw4wc4ne7ny2x7kvajzkm // ➜ bafkreiafc5f36dkaocd6iwysxkxboelue2cs745j4wgrfihlxgqqwqexim // bafyreietdfd5sh743y6c4f4zrhwkqdadwtcuvziot3mvwhyqojjtvtqoxi (6 links) // ➜ bafkreicwcm4sqhux7ipwbwro2inf4vbjoprzwoqnl3t4zta4gphofot7du // ➜ bafkreic7pdbte5heh6u54vszezob3el6exadoiw4wc4ne7ny2x7kvajzkm // ➜ bafkreiafc5f36dkaocd6iwysxkxboelue2cs745j4wgrfihlxgqqwqexim // ➜ baguqeeraawfdfutq7b4dgmcelypy4r36d7ndngbzpbejzhjaxffcrbgq3wka // ➜ baguqeeraawfdfutq7b4dgmcelypy4r36d7ndngbzpbejzhjaxffcrbgq3wka // ➜ bafkreic7pdbte5heh6u54vszezob3el6exadoiw4wc4ne7ny2x7kvajzkm // Manually decoded bafyreietdfd5sh743y6c4f4zrhwkqdadwtcuvziot3mvwhyqojjtvtqoxi has 6 links too, surprise! } // LinkPrototypes to tell the LinkSystem how to store the Nodes in our memory // store and how to generate the CIDs (i.e. codec to store, multihash to use in // the CID). var dclp = cidlink.LinkPrototype{ Prefix: cid.Prefix{ Version: 1, Codec: cid.DagCBOR, MhType: multihash.SHA2_256, MhLength: 32, }, } var pblp = cidlink.LinkPrototype{ Prefix: cid.Prefix{ Version: 1, Codec: cid.DagJSON, // cid.DagProtobuf, but we're pretending MhType: multihash.SHA2_256, MhLength: 32, }, } var rawlp = cidlink.LinkPrototype{ Prefix: cid.Prefix{ Version: 1, Codec: cid.Raw, MhType: multihash.SHA2_256, MhLength: 32, }, } // Given a LinkPrototype and a Node, encode the Node and store it in our memory // store, returning the CID of the stored Node. func encodeAndStore(lp cidlink.LinkPrototype, n datamodel.Node) cid.Cid { lsys := cidlink.DefaultLinkSystem() // note this is in our package: var store = memstore.Store{} lsys.SetWriteStorage(&store) lsys.SetReadStorage(&store) lnk := lsys.MustStore(linking.LinkContext{}, lp, n) return lnk.(cidlink.Link).Cid } // Given a CID, load the Node from our memory store and return it. func loadNode(c cid.Cid) datamodel.Node { lsys := cidlink.DefaultLinkSystem() // note this is in our package: var store = memstore.Store{} lsys.SetReadStorage(&store) nb := lsys.MustLoad(linking.LinkContext{}, cidlink.Link{Cid: c}, basicnode.Prototype.Any) return nb } // Given a CID, load the bytes from our memory store and return them. func loadBytes(c cid.Cid) []byte { lsys := cidlink.DefaultLinkSystem() // note this is in our package: var store = memstore.Store{} lsys.SetReadStorage(&store) byts, err := lsys.LoadRaw(linking.LinkContext{}, cidlink.Link{Cid: c}) if err != nil { panic(err) } return byts } ================================================ FILE: traversal/fns.go ================================================ package traversal import ( "context" "fmt" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/linking" "github.com/ipld/go-ipld-prime/linking/preload" ) // This file defines interfaces for things users provide, // plus a few of the parameters they'll need to receieve. //-------------------------------------------------------- // VisitFn is a read-only visitor. type VisitFn func(Progress, datamodel.Node) error // TransformFn is like a visitor that can also return a new Node to replace the visited one. type TransformFn func(Progress, datamodel.Node) (datamodel.Node, error) // AdvVisitFn is like VisitFn, but for use with AdvTraversal: it gets additional arguments describing *why* this node is visited. type AdvVisitFn func(Progress, datamodel.Node, VisitReason) error // VisitReason provides additional information to traversals using AdvVisitFn. type VisitReason byte const ( // VisitReason_SelectionMatch tells AdvVisitFn that this node was explicitly selected. (This is the set of nodes that VisitFn is called for.) VisitReason_SelectionMatch VisitReason = 'm' // VisitReason_SelectionParent tells AdvVisitFn that this node is a parent of one that will be explicitly selected. (These calls only happen if the feature is enabled -- enabling parent detection requires a different algorithm and adds some overhead.) VisitReason_SelectionParent VisitReason = 'p' // VisitReason_SelectionCandidate tells AdvVisitFn that this node was visited while searching for selection matches. It is not necessarily implied that any explicit match will be a child of this node; only that we had to consider it. (Merkle-proofs generally need to include any node in this group.) VisitReason_SelectionCandidate VisitReason = 'x' ) // Progress tracks a traversal as it proceeds. It is used initially to begin a traversal, and it is then passed to the visit function as the traversal proceeds. // // As the traversal descends into the graph, new Progress values are created and passed to the visit function with updated properties representing the current state of the traversal. // // Most customization of a traversal is done by setting a Cfg property on a Progress before beginning the traversal. // Typical customization involves setting a LinkSystem for link loading and/or tracking. // // Advanced traversal control options, such as LinkVisitOnlyOnce and StartAtPath, are also available in the Cfg but may have surprising effects on traversal behavior; be careful when using them. // // Budgets are set on the Progress option because a Budget, while set at the beginning of a traversal, is also updated as the traversal proceeds, with its fields being monotonically decremented. // Beware of using Budgets in tandem with a Preloader! The preloader discovers links in a lateral scan of a whole block, before rewinding for a depth-first walk for traversal-proper. // Budgets are intended to be used for the depth-first walk, and there is no way to know ahead of time how the budget may impact the lateral parts of the graph that the preloader encounters. // Currently a best-guess approach is used to try and have the preloader adhere to the budget, but with typical real-world graphs, this is likely to be inaccurate. // In the case of inaccuracies, the budget will be properly applied to the traversal-proper, but the preloader may receive a different set of links than the traversal-proper will. type Progress struct { // Cfg is the configuration for the traversal, set by user. Cfg *Config // Budget, if present, tracks "budgets" for how many more steps we're willing to take before we should halt. // Budget is initially set by user, but is then updated as the traversal proceeds. Budget *Budget // Path is how we reached the current point in the traversal. Path datamodel.Path // LastBlock stores the Path and Link of the last block edge we had to load. (It will always be zero in traversals with no linkloader.) LastBlock struct { Path datamodel.Path Link datamodel.Link } // PastStartAtPath indicates whether the traversal has progressed passed the StartAtPath in the config -- use to avoid path checks when inside a sub portion of a DAG that is entirely inside the "not-skipped" portion of a traversal PastStartAtPath bool // SeenLinks is a set used to remember which links have been visited before, if Cfg.LinkVisitOnlyOnce is true. SeenLinks map[datamodel.Link]struct{} } // Config is a set of options for a traversal. Set a Config on a Progress to customize the traversal. type Config struct { // Ctx is the context carried through a traversal. // Optional; use it if you need cancellation. Ctx context.Context // LinkSystem is used for automatic link loading, and also any storing if mutation features (e.g. traversal.Transform) are used. LinkSystem linking.LinkSystem // LinkTargetNodePrototypeChooser is a chooser for Node implementations to produce during automatic link traversal. LinkTargetNodePrototypeChooser LinkTargetNodePrototypeChooser // LinkVisitOnlyOnce controls repeat-link visitation. // By default, we visit across links wherever we see them again, even if we've visited them before, because the reason for visiting might be different than it was before since we got to it via a different path. // If set to true, track links we've seen before in Progress.SeenLinks and do not visit them again. // Note that sufficiently complex selectors may require valid revisiting of some links, so setting this to true can change behavior noticably and should be done with care. LinkVisitOnlyOnce bool // StartAtPath, if set, causes a traversal to skip forward until passing this path, and only then begins calling visit functions. // Block loads will also be skipped wherever possible. StartAtPath datamodel.Path // Preloader receives links within each block prior to traversal-proper by performing a lateral scan of a block without descending into links themselves before backing up and doing a traversal-proper. // This can be used to asynchronously load blocks that will be required at a later phase of the retrieval, or even to load blocks in a different order than the traversal would otherwise do. // Preload calls are not de-duplicated, it is up to the receiver to do so if desired. // Beware of using both Budget and Preloader! See the documentation on Progress for more information on this usage and the likely surprising effects. Preloader preload.Loader } // Budget is a set of monotonically-decrementing "budgets" for how many more steps we're willing to take before we should halt. // // The fields of Budget are described as "monotonically-decrementing", because that's what the traversal library will do with them, // but they are user-accessable and can be reset to higher numbers again by code in the visitor callbacks. This is not recommended (why?), but possible. // If you set any budgets (by having a non-nil Progress.Budget field), you must set some value for all of them. // Traversal halts when _any_ of the budgets reaches zero. // The max value of an int (math.MaxInt64) is acceptable for any budget you don't care about. // // Beware of using both Budget and Preloader! See the documentation on Progress for more information on this usage and the likely surprising effects. type Budget struct { // NodeBudget is a monotonically-decrementing "budget" for how many more nodes we're willing to visit before halting. NodeBudget int64 // LinkBudget is a monotonically-decrementing "budget" for how many more links we're willing to load before halting. // (This is not aware of any caching; it's purely in terms of links encountered and traversed.) LinkBudget int64 } // Clone returns a copy of the budget. func (b *Budget) Clone() *Budget { if b == nil { return nil } return &Budget{ NodeBudget: b.NodeBudget, LinkBudget: b.LinkBudget, } } // LinkTargetNodePrototypeChooser is a function that returns a NodePrototype based on // the information in a Link and/or its LinkContext. // // A LinkTargetNodePrototypeChooser can be used in a traversal.Config to be clear about // what kind of Node implementation to use when loading a Link. // In a simple example, it could constantly return a `basicnode.Prototype.Any`. // In a more complex example, a program using `bind` over native Go types // could decide what kind of native type is expected, and return a // `bind.NodeBuilder` for that specific concrete native type. type LinkTargetNodePrototypeChooser func(datamodel.Link, linking.LinkContext) (datamodel.NodePrototype, error) // SkipMe is a signalling "error" which can be used to tell traverse to skip some data. // // SkipMe can be returned by the Config.LinkLoader to skip entire blocks without aborting the walk. // (This can be useful if you know you don't have data on hand, // but want to continue the walk in other areas anyway; // or, if you're doing a way where you know that it's valid to memoize seen // areas based on Link alone.) type SkipMe struct{} func (SkipMe) Error() string { return "skip" } type ErrBudgetExceeded struct { BudgetKind string // "node"|"link" Path datamodel.Path Link datamodel.Link // only present if BudgetKind=="link" } func (e *ErrBudgetExceeded) Error() string { msg := fmt.Sprintf("traversal budget exceeded: budget for %ss reached zero while on path %q", e.BudgetKind, e.Path) if e.Link != nil { msg += fmt.Sprintf(" (link: %q)", e.Link) } return msg } func (e *ErrBudgetExceeded) Is(target error) bool { _, ok := target.(*ErrBudgetExceeded) return ok } ================================================ FILE: traversal/focus.go ================================================ package traversal import ( "fmt" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/linking" ) // Focus traverses a Node graph according to a path, reaches a single Node, // and calls the given VisitFn on that reached node. // // This function is a helper function which starts a new traversal with default configuration. // It cannot cross links automatically (since this requires configuration). // Use the equivalent Focus function on the Progress structure // for more advanced and configurable walks. func Focus(n datamodel.Node, p datamodel.Path, fn VisitFn) error { return Progress{}.Focus(n, p, fn) } // Get is the equivalent of Focus, but returns the reached node (rather than invoking a callback at the target), // and does not yield Progress information. // // This function is a helper function which starts a new traversal with default configuration. // It cannot cross links automatically (since this requires configuration). // Use the equivalent Get function on the Progress structure // for more advanced and configurable walks. func Get(n datamodel.Node, p datamodel.Path) (datamodel.Node, error) { return Progress{}.Get(n, p) } // FocusedTransform traverses a datamodel.Node graph, reaches a single Node, // and calls the given TransformFn to decide what new node to replace the visited node with. // A new Node tree will be returned (the original is unchanged). // // This function is a helper function which starts a new traversal with default configuration. // It cannot cross links automatically (since this requires configuration). // Use the equivalent FocusedTransform function on the Progress structure // for more advanced and configurable walks. func FocusedTransform(n datamodel.Node, p datamodel.Path, fn TransformFn, createParents bool) (datamodel.Node, error) { return Progress{}.FocusedTransform(n, p, fn, createParents) } // Focus traverses a Node graph according to a path, reaches a single Node, // and calls the given VisitFn on that reached node. // // Focus is a read-only traversal. // See FocusedTransform if looking for a way to do an "update" to a Node. // // Provide configuration to this process using the Config field in the Progress object. // // This walk will automatically cross links, but requires some configuration // with link loading functions to do so. // // Focus (and the other traversal functions) can be used again again inside the VisitFn! // By using the traversal.Progress handed to the VisitFn, // the Path recorded of the traversal so far will continue to be extended, // and thus continued nested uses of Walk and Focus will see the fully contextualized Path. func (prog Progress) Focus(n datamodel.Node, p datamodel.Path, fn VisitFn) error { n, err := prog.get(n, p, true) if err != nil { return err } return fn(prog, n) } // Get is the equivalent of Focus, but returns the reached node (rather than invoking a callback at the target), // and does not yield Progress information. // // Provide configuration to this process using the Config field in the Progress object. // // This walk will automatically cross links, but requires some configuration // with link loading functions to do so. // // If doing several traversals which are nested, consider using the Focus funcion in preference to Get; // the Focus functions provide updated Progress objects which can be used to do nested traversals while keeping consistent track of progress, // such that continued nested uses of Walk or Focus or Get will see the fully contextualized Path. func (prog Progress) Get(n datamodel.Node, p datamodel.Path) (datamodel.Node, error) { return prog.get(n, p, false) } // get is the internal implementation for Focus and Get. // It *mutates* the Progress object it's called on, and returns reached nodes. // For Get calls, trackProgress=false, which avoids some allocations for state tracking that's not needed by that call. func (prog *Progress) get(n datamodel.Node, p datamodel.Path, trackProgress bool) (datamodel.Node, error) { prog.init() segments := p.Segments() var prev datamodel.Node // for LinkContext for i, seg := range segments { // Check the budget! if prog.Budget != nil { prog.Budget.NodeBudget-- if prog.Budget.NodeBudget <= 0 { return nil, &ErrBudgetExceeded{BudgetKind: "node", Path: prog.Path} } } // Traverse the segment. switch n.Kind() { case datamodel.Kind_Invalid: panic(fmt.Errorf("invalid node encountered at %q", p.Truncate(i))) case datamodel.Kind_Map: next, err := n.LookupByString(seg.String()) if err != nil { return nil, fmt.Errorf("error traversing segment %q on node at %q: %w", seg, p.Truncate(i), err) } prev, n = n, next case datamodel.Kind_List: intSeg, err := seg.Index() if err != nil { return nil, fmt.Errorf("error traversing segment %q on node at %q: the segment cannot be parsed as a number and the node is a list", seg, p.Truncate(i)) } next, err := n.LookupByIndex(intSeg) if err != nil { return nil, fmt.Errorf("error traversing segment %q on node at %q: %w", seg, p.Truncate(i), err) } prev, n = n, next default: return nil, fmt.Errorf("cannot traverse node at %q: %w", p.Truncate(i), fmt.Errorf("cannot traverse terminals")) } // Dereference any links. for n.Kind() == datamodel.Kind_Link { lnk, _ := n.AsLink() // Check the budget! if prog.Budget != nil { if prog.Budget.LinkBudget <= 0 { return nil, &ErrBudgetExceeded{BudgetKind: "link", Path: prog.Path, Link: lnk} } prog.Budget.LinkBudget-- } // Put together the context info we'll offer to the loader and prototypeChooser. lnkCtx := linking.LinkContext{ Ctx: prog.Cfg.Ctx, LinkPath: p.Truncate(i), LinkNode: n, ParentNode: prev, } // Pick what in-memory format we will build. np, err := prog.Cfg.LinkTargetNodePrototypeChooser(lnk, lnkCtx) if err != nil { return nil, fmt.Errorf("error traversing node at %q: could not load link %q: %w", p.Truncate(i+1), lnk, err) } // Load link! prev = n n, err = prog.Cfg.LinkSystem.Load(lnkCtx, lnk, np) if err != nil { return nil, fmt.Errorf("error traversing node at %q: could not load link %q: %w", p.Truncate(i+1), lnk, err) } if trackProgress { prog.LastBlock.Path = p.Truncate(i + 1) prog.LastBlock.Link = lnk } } } if trackProgress { prog.Path = prog.Path.Join(p) } return n, nil } // FocusedTransform traverses a datamodel.Node graph, reaches a single Node, // and calls the given TransformFn to decide what new node to replace the visited node with. // A new Node tree will be returned (the original is unchanged). // // If the TransformFn returns the same Node which it was called with, // then the transform is a no-op, and the Node returned from the // FocusedTransform call as a whole will also be the same as its starting Node. // // Otherwise, the reached node will be "replaced" with the new Node -- meaning // that new intermediate nodes will be constructed to also replace each // parent Node that was traversed to get here, thus propagating the changes in // a copy-on-write fashion -- and the FocusedTransform function as a whole will // return a new Node containing identical children except for those replaced. // // Returning nil from the TransformFn as the replacement node means "remove this". // // FocusedTransform can be used again inside the applied function! // This kind of composition can be useful for doing batches of updates. // E.g. if have a large Node graph which contains a 100-element list, and // you want to replace elements 12, 32, and 95 of that list: // then you should FocusedTransform to the list first, and inside that // TransformFn's body, you can replace the entire list with a new one // that is composed of copies of everything but those elements -- including // using more TransformFn calls as desired to produce the replacement elements // if it so happens that those replacement elements are easiest to construct // by regarding them as incremental updates to the previous values. // (This approach can also be used when doing other modifications like insertion // or reordering -- which would otherwise be tricky to define, since // each operation could change the meaning of subsequently used indexes.) // // As a special case, list appending is supported by using the path segment "-". // (This is determined by the node it applies to -- if that path segment // is applied to a map, it's just a regular map key of the string of dash.) // // Note that anything you can do with the Transform function, you can also // do with regular Node and NodeBuilder usage directly. Transform just // does a large amount of the intermediate bookkeeping that's useful when // creating new values which are partial updates to existing values. func (prog Progress) FocusedTransform(n datamodel.Node, p datamodel.Path, fn TransformFn, createParents bool) (datamodel.Node, error) { prog.init() nb := n.Prototype().NewBuilder() if err := prog.focusedTransform(n, nb, p, fn, createParents); err != nil { return nil, err } return nb.Build(), nil } // focusedTransform assumes that an update will actually happen, and as it recurses deeper, // begins building an updated node tree. // // As implemented, this is not actually efficient if the update will be a no-op; it won't notice until it gets there. func (prog Progress) focusedTransform(n datamodel.Node, na datamodel.NodeAssembler, p datamodel.Path, fn TransformFn, createParents bool) error { at := prog.Path // Base case: if we've reached the end of the path, do the replacement here. // (Note: in some cases within maps, there is another branch that is the base case, for reasons involving removes.) if p.Len() == 0 { n2, err := fn(prog, n) if err != nil { return err } return na.AssignNode(n2) } seg, p2 := p.Shift() // Check the budget! if prog.Budget != nil { if prog.Budget.NodeBudget <= 0 { return &ErrBudgetExceeded{BudgetKind: "node", Path: prog.Path} } prog.Budget.NodeBudget-- } // Special branch for if we've entered createParent mode in an earlier step. // This needs slightly different logic because there's no prior node to reference // (and we wouldn't want to waste time creating a dummy one). if n == nil { ma, err := na.BeginMap(1) if err != nil { return err } prog.Path = at.AppendSegment(seg) if err := ma.AssembleKey().AssignString(seg.String()); err != nil { return err } if err := prog.focusedTransform(nil, ma.AssembleValue(), p2, fn, createParents); err != nil { return err } return ma.Finish() } // Handle node based on kind. // If it's a recursive kind (map or list), we'll be recursing on it. // If it's a link, load it! And recurse on it. // If it's a scalar kind (any of the rest), we'll... be erroring, actually; // if we're at the end, it was already handled at the top of the function, // so we only get to this case if we were expecting to go deeper. switch n.Kind() { case datamodel.Kind_Map: ma, err := na.BeginMap(n.Length()) if err != nil { return err } // If we're approaching the end of the path, call the TransformFunc. // We need to know if it returns nil (meaning: do a deletion) _before_ we do the AssembleKey step. // (This results in the entire map branch having a different base case.) var end bool var n2 datamodel.Node if p2.Len() == 0 { end = true n3, err := n.LookupBySegment(seg) if n3 != datamodel.Absent && err != nil { // TODO badly need to simplify the standard treatment of "not found" here. Can't even fit it all in one line! See https://github.com/ipld/go-ipld-prime/issues/360. if _, ok := err.(datamodel.ErrNotExists); !ok { return err } } prog.Path = at.AppendSegment(seg) n2, err = fn(prog, n3) if err != nil { return err } } // Copy children over. Replace the target (preserving its current position!) while doing this, if found. // Note that we don't recurse into copying children (assuming AssignNode doesn't); this is as shallow/COW as the AssignNode implementation permits. var replaced bool for itr := n.MapIterator(); !itr.Done(); { k, v, err := itr.Next() if err != nil { return err } if asPathSegment(k).Equals(seg) { // for the segment that's either update, update within, or being removed: if end { // the last path segment in the overall instruction gets a different case because it may need to handle deletion if n2 == nil { replaced = true continue // replace with nil means delete, which means continue early here: don't even copy the key. } } // as long as we're not deleting, then this key will exist in the new data. if err := ma.AssembleKey().AssignNode(k); err != nil { return err } replaced = true if n2 != nil { // if we already produced the replacement because we're at the end... if err := ma.AssembleValue().AssignNode(n2); err != nil { return err } } else { // ... otherwise, recurse: prog.Path = at.AppendSegment(seg) if err := prog.focusedTransform(v, ma.AssembleValue(), p2, fn, createParents); err != nil { return err } } } else { // for any other siblings of the target: just copy. if err := ma.AssembleKey().AssignNode(k); err != nil { return err } if err := ma.AssembleValue().AssignNode(v); err != nil { return err } } } if replaced { return ma.Finish() } // If we didn't find the target yet: append it. // If we're at the end, always do this; // if we're in the middle, only do this if createParents mode is enabled. prog.Path = at.AppendSegment(seg) if p.Len() > 1 && !createParents { return fmt.Errorf("transform: parent position at %q did not exist (and createParents was false)", prog.Path) } if err := ma.AssembleKey().AssignString(seg.String()); err != nil { return err } if err := prog.focusedTransform(nil, ma.AssembleValue(), p2, fn, createParents); err != nil { return err } return ma.Finish() case datamodel.Kind_List: la, err := na.BeginList(n.Length()) if err != nil { return err } // First figure out if this path segment can apply to a list sanely at all. // Simultaneously, get it in numeric format, so subsequent operations are cheaper. ti, err := seg.Index() if err != nil { if seg.String() == "-" { ti = -1 } else { return fmt.Errorf("transform: cannot navigate path segment %q at %q because a list is here", seg, prog.Path) } } // Copy children over. Replace the target (preserving its current position!) while doing this, if found. // Note that we don't recurse into copying children (assuming AssignNode doesn't); this is as shallow/COW as the AssignNode implementation permits. var replaced bool for itr := n.ListIterator(); !itr.Done(); { i, v, err := itr.Next() if err != nil { return err } if ti == i { prog.Path = prog.Path.AppendSegment(seg) if err := prog.focusedTransform(v, la.AssembleValue(), p2, fn, createParents); err != nil { return err } replaced = true } else { if err := la.AssembleValue().AssignNode(v); err != nil { return err } } } if replaced { return la.Finish() } // If we didn't find the target yet: hopefully this was an append operation; // if it wasn't, then it's index out of bounds. We don't arbitrarily extend lists with filler. if ti >= 0 { return fmt.Errorf("transform: cannot navigate path segment %q at %q because it is beyond the list bounds", seg, prog.Path) } prog.Path = prog.Path.AppendSegment(datamodel.PathSegmentOfInt(n.Length())) if err := prog.focusedTransform(nil, la.AssembleValue(), p2, fn, createParents); err != nil { return err } return la.Finish() case datamodel.Kind_Link: lnk, _ := n.AsLink() // Check the budget! if prog.Budget != nil { if prog.Budget.LinkBudget <= 0 { return &ErrBudgetExceeded{BudgetKind: "link", Path: prog.Path, Link: lnk} } prog.Budget.LinkBudget-- } // Put together the context info we'll offer to the loader and prototypeChooser. lnkCtx := linking.LinkContext{ Ctx: prog.Cfg.Ctx, LinkPath: prog.Path, LinkNode: n, ParentNode: nil, // TODO inconvenient that we don't have this. maybe this whole case should be a helper function. } // Pick what in-memory format we will build. np, err := prog.Cfg.LinkTargetNodePrototypeChooser(lnk, lnkCtx) if err != nil { return fmt.Errorf("transform: error traversing node at %q: could not load link %q: %w", prog.Path, lnk, err) } // Load link! // We'll use LinkSystem.Fill here rather than Load, // because there's a nice opportunity to reuse the builder shortly. nb := np.NewBuilder() err = prog.Cfg.LinkSystem.Fill(lnkCtx, lnk, nb) if err != nil { return fmt.Errorf("transform: error traversing node at %q: could not load link %q: %w", prog.Path, lnk, err) } prog.LastBlock.Path = prog.Path prog.LastBlock.Link = lnk n = nb.Build() // Recurse. // Start a new builder for this, using the same prototype we just used for loading the link. // (Or more specifically: this is an opportunity for just resetting a builder and reusing memory!) // When we come back... we'll have to engage serialization and storage on the new node! // Path isn't updated here (neither progress nor to-go). nb.Reset() if err := prog.focusedTransform(n, nb, p, fn, createParents); err != nil { return err } n = nb.Build() lnk, err = prog.Cfg.LinkSystem.Store(lnkCtx, lnk.Prototype(), n) if err != nil { return fmt.Errorf("transform: error storing transformed node at %q: %w", prog.Path, err) } return na.AssignLink(lnk) default: return fmt.Errorf("transform: parent position at %q was a scalar, cannot go deeper", prog.Path) } } ================================================ FILE: traversal/focus_test.go ================================================ package traversal_test import ( "reflect" "testing" qt "github.com/frankban/quicktest" "github.com/google/go-cmp/cmp" "github.com/ipfs/go-cid" _ "github.com/ipld/go-ipld-prime/codec/dagjson" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/fluent" "github.com/ipld/go-ipld-prime/linking" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipld/go-ipld-prime/must" "github.com/ipld/go-ipld-prime/node/basicnode" nodetests "github.com/ipld/go-ipld-prime/node/tests" "github.com/ipld/go-ipld-prime/storage/memstore" "github.com/ipld/go-ipld-prime/traversal" ) // Do some fixture fabrication. // We assume all the builders and serialization must Just Work here. var deepEqualsAllowAllUnexported = qt.CmpEquals(cmp.Exporter(func(reflect.Type) bool { return true })) var store = memstore.Store{} var ( // baguqeeyexkjwnfy leafAlpha, leafAlphaLnk = encode(basicnode.NewString("alpha")) // baguqeeyeqvc7t3a leafBeta, leafBetaLnk = encode(basicnode.NewString("beta")) // baguqeeyezhlahvq middleMapNode, middleMapNodeLnk = encode(fluent.MustBuildMap(basicnode.Prototype.Map, 3, func(na fluent.MapAssembler) { na.AssembleEntry("foo").AssignBool(true) na.AssembleEntry("bar").AssignBool(false) na.AssembleEntry("nested").CreateMap(2, func(na fluent.MapAssembler) { na.AssembleEntry("alink").AssignLink(leafAlphaLnk) na.AssembleEntry("nonlink").AssignString("zoo") }) })) // baguqeeyehfkkfwa middleListNode, middleListNodeLnk = encode(fluent.MustBuildList(basicnode.Prototype.List, 4, func(na fluent.ListAssembler) { na.AssembleValue().AssignLink(leafAlphaLnk) na.AssembleValue().AssignLink(leafAlphaLnk) na.AssembleValue().AssignLink(leafBetaLnk) na.AssembleValue().AssignLink(leafAlphaLnk) })) // note that using `rootNode` directly will have a different field ordering than // the encoded form if you were to load `rootNodeLnk` due to dag-json field // reordering on encode, beware the difference for traversal order between // created, in-memory nodes and those that have passed through a codec with // field ordering rules // baguqeeyeie4ajfy rootNode, rootNodeLnk = encode(fluent.MustBuildMap(basicnode.Prototype.Map, 4, func(na fluent.MapAssembler) { na.AssembleEntry("plain").AssignString("olde string") na.AssembleEntry("linkedString").AssignLink(leafAlphaLnk) na.AssembleEntry("linkedMap").AssignLink(middleMapNodeLnk) na.AssembleEntry("linkedList").AssignLink(middleListNodeLnk) })) ) // encode hardcodes some encoding choices for ease of use in fixture generation; // just gimme a link and stuff the bytes in a map. // (also return the node again for convenient assignment.) func encode(n datamodel.Node) (datamodel.Node, datamodel.Link) { lp := cidlink.LinkPrototype{Prefix: cid.Prefix{ Version: 1, Codec: 0x0129, MhType: 0x13, MhLength: 4, }} lsys := cidlink.DefaultLinkSystem() lsys.SetWriteStorage(&store) lnk, err := lsys.Store(linking.LinkContext{}, lp, n) if err != nil { panic(err) } return n, lnk } // covers Focus used on one already-loaded Node; no link-loading exercised. func TestFocusSingleTree(t *testing.T) { t.Run("empty path on scalar node returns start node", func(t *testing.T) { err := traversal.Focus(basicnode.NewString("x"), datamodel.Path{}, func(prog traversal.Progress, n datamodel.Node) error { qt.Check(t, n, nodetests.NodeContentEquals, basicnode.NewString("x")) qt.Check(t, prog.Path.String(), qt.Equals, datamodel.Path{}.String()) return nil }) qt.Check(t, err, qt.IsNil) }) t.Run("one step path on map node works", func(t *testing.T) { err := traversal.Focus(middleMapNode, datamodel.ParsePath("foo"), func(prog traversal.Progress, n datamodel.Node) error { qt.Check(t, n, nodetests.NodeContentEquals, basicnode.NewBool(true)) qt.Check(t, prog.Path, deepEqualsAllowAllUnexported, datamodel.ParsePath("foo")) return nil }) qt.Check(t, err, qt.IsNil) }) t.Run("two step path on map node works", func(t *testing.T) { err := traversal.Focus(middleMapNode, datamodel.ParsePath("nested/nonlink"), func(prog traversal.Progress, n datamodel.Node) error { qt.Check(t, n, nodetests.NodeContentEquals, basicnode.NewString("zoo")) qt.Check(t, prog.Path, deepEqualsAllowAllUnexported, datamodel.ParsePath("nested/nonlink")) return nil }) qt.Check(t, err, qt.IsNil) }) } // covers Get used on one already-loaded Node; no link-loading exercised. // same fixtures as the test for Focus; just has fewer assertions, since Get does no progress tracking. func TestGetSingleTree(t *testing.T) { t.Run("empty path on scalar node returns start node", func(t *testing.T) { n, err := traversal.Get(basicnode.NewString("x"), datamodel.Path{}) qt.Check(t, err, qt.IsNil) qt.Check(t, n, nodetests.NodeContentEquals, basicnode.NewString("x")) }) t.Run("one step path on map node works", func(t *testing.T) { n, err := traversal.Get(middleMapNode, datamodel.ParsePath("foo")) qt.Check(t, err, qt.IsNil) qt.Check(t, n, nodetests.NodeContentEquals, basicnode.NewBool(true)) }) t.Run("two step path on map node works", func(t *testing.T) { n, err := traversal.Get(middleMapNode, datamodel.ParsePath("nested/nonlink")) qt.Check(t, err, qt.IsNil) qt.Check(t, n, nodetests.NodeContentEquals, basicnode.NewString("zoo")) }) } func TestFocusWithLinkLoading(t *testing.T) { t.Run("link traversal with no configured loader should fail", func(t *testing.T) { t.Run("terminal link should fail", func(t *testing.T) { err := traversal.Focus(middleMapNode, datamodel.ParsePath("nested/alink"), func(prog traversal.Progress, n datamodel.Node) error { t.Errorf("should not be reached; no way to load this path") return nil }) qt.Check(t, err.Error(), qt.Equals, `error traversing node at "nested/alink": could not load link "`+leafAlphaLnk.String()+`": no LinkTargetNodePrototypeChooser configured`) }) t.Run("mid-path link should fail", func(t *testing.T) { err := traversal.Focus(rootNode, datamodel.ParsePath("linkedMap/nested/nonlink"), func(prog traversal.Progress, n datamodel.Node) error { t.Errorf("should not be reached; no way to load this path") return nil }) qt.Check(t, err.Error(), qt.Equals, `error traversing node at "linkedMap": could not load link "`+middleMapNodeLnk.String()+`": no LinkTargetNodePrototypeChooser configured`) }) }) t.Run("link traversal with loader should work", func(t *testing.T) { lsys := cidlink.DefaultLinkSystem() lsys.SetReadStorage(&store) err := traversal.Progress{ Cfg: &traversal.Config{ LinkSystem: lsys, LinkTargetNodePrototypeChooser: basicnode.Chooser, }, }.Focus(rootNode, datamodel.ParsePath("linkedMap/nested/nonlink"), func(prog traversal.Progress, n datamodel.Node) error { qt.Check(t, n, nodetests.NodeContentEquals, basicnode.NewString("zoo")) qt.Check(t, prog.Path, deepEqualsAllowAllUnexported, datamodel.ParsePath("linkedMap/nested/nonlink")) qt.Check(t, prog.LastBlock.Link, deepEqualsAllowAllUnexported, middleMapNodeLnk) qt.Check(t, prog.LastBlock.Path, deepEqualsAllowAllUnexported, datamodel.ParsePath("linkedMap")) return nil }) qt.Check(t, err, qt.IsNil) }) } func TestGetWithLinkLoading(t *testing.T) { t.Run("link traversal with no configured loader should fail", func(t *testing.T) { t.Run("terminal link should fail", func(t *testing.T) { _, err := traversal.Get(middleMapNode, datamodel.ParsePath("nested/alink")) qt.Check(t, err.Error(), qt.Equals, `error traversing node at "nested/alink": could not load link "`+leafAlphaLnk.String()+`": no LinkTargetNodePrototypeChooser configured`) }) t.Run("mid-path link should fail", func(t *testing.T) { _, err := traversal.Get(rootNode, datamodel.ParsePath("linkedMap/nested/nonlink")) qt.Check(t, err.Error(), qt.Equals, `error traversing node at "linkedMap": could not load link "`+middleMapNodeLnk.String()+`": no LinkTargetNodePrototypeChooser configured`) }) }) t.Run("link traversal with loader should work", func(t *testing.T) { lsys := cidlink.DefaultLinkSystem() lsys.SetReadStorage(&store) n, err := traversal.Progress{ Cfg: &traversal.Config{ LinkSystem: lsys, LinkTargetNodePrototypeChooser: basicnode.Chooser, }, }.Get(rootNode, datamodel.ParsePath("linkedMap/nested/nonlink")) qt.Check(t, err, qt.IsNil) qt.Check(t, n, nodetests.NodeContentEquals, basicnode.NewString("zoo")) }) } func TestFocusedTransform(t *testing.T) { t.Run("UpdateMapEntry", func(t *testing.T) { n, err := traversal.FocusedTransform(rootNode, datamodel.ParsePath("plain"), func(progress traversal.Progress, prev datamodel.Node) (datamodel.Node, error) { qt.Check(t, progress.Path.String(), qt.Equals, "plain") qt.Check(t, must.String(prev), qt.Equals, "olde string") nb := prev.Prototype().NewBuilder() nb.AssignString("new string!") return nb.Build(), nil }, false) qt.Check(t, err, qt.IsNil) qt.Check(t, n.Kind(), qt.Equals, datamodel.Kind_Map) // updated value should be there qt.Check(t, must.Node(n.LookupByString("plain")), nodetests.NodeContentEquals, basicnode.NewString("new string!")) // everything else should be there qt.Check(t, must.Node(n.LookupByString("linkedString")), qt.Equals, must.Node(rootNode.LookupByString("linkedString"))) qt.Check(t, must.Node(n.LookupByString("linkedMap")), qt.Equals, must.Node(rootNode.LookupByString("linkedMap"))) qt.Check(t, must.Node(n.LookupByString("linkedList")), qt.Equals, must.Node(rootNode.LookupByString("linkedList"))) // everything should still be in the same order qt.Check(t, keys(n), qt.DeepEquals, []string{"plain", "linkedString", "linkedMap", "linkedList"}) }) t.Run("UpdateDeeperMap", func(t *testing.T) { n, err := traversal.FocusedTransform(middleMapNode, datamodel.ParsePath("nested/alink"), func(progress traversal.Progress, prev datamodel.Node) (datamodel.Node, error) { qt.Check(t, progress.Path.String(), qt.Equals, "nested/alink") qt.Check(t, prev, nodetests.NodeContentEquals, basicnode.NewLink(leafAlphaLnk)) return basicnode.NewString("new string!"), nil }, false) qt.Check(t, err, qt.IsNil) qt.Check(t, n.Kind(), qt.Equals, datamodel.Kind_Map) // updated value should be there qt.Check(t, must.Node(must.Node(n.LookupByString("nested")).LookupByString("alink")), nodetests.NodeContentEquals, basicnode.NewString("new string!")) // everything else in the parent map should should be there! qt.Check(t, must.Node(n.LookupByString("foo")), qt.Equals, must.Node(middleMapNode.LookupByString("foo"))) qt.Check(t, must.Node(n.LookupByString("bar")), qt.Equals, must.Node(middleMapNode.LookupByString("bar"))) // everything should still be in the same order qt.Check(t, keys(n), qt.DeepEquals, []string{"foo", "bar", "nested"}) }) t.Run("AppendIfNotExists", func(t *testing.T) { n, err := traversal.FocusedTransform(rootNode, datamodel.ParsePath("newpart"), func(progress traversal.Progress, prev datamodel.Node) (datamodel.Node, error) { qt.Check(t, progress.Path.String(), qt.Equals, "newpart") qt.Check(t, prev, qt.IsNil) // REVIEW: should datamodel.Absent be used here? I lean towards "no" but am unsure what's least surprising here. // An interesting thing to note about inserting a value this way is that you have no `prev.Prototype().NewBuilder()` to use if you wanted to. // But if that's an issue, then what you do is a focus or walk (transforming or not) to the parent node, get its child prototypes, and go from there. return basicnode.NewString("new string!"), nil }, false) qt.Check(t, err, qt.IsNil) qt.Check(t, n.Kind(), qt.Equals, datamodel.Kind_Map) // updated value should be there qt.Check(t, must.Node(n.LookupByString("newpart")), nodetests.NodeContentEquals, basicnode.NewString("new string!")) // everything should still be in the same order... with the new entry at the end. qt.Check(t, keys(n), qt.DeepEquals, []string{"plain", "linkedString", "linkedMap", "linkedList", "newpart"}) }) t.Run("CreateParents", func(t *testing.T) { n, err := traversal.FocusedTransform(rootNode, datamodel.ParsePath("newsection/newpart"), func(progress traversal.Progress, prev datamodel.Node) (datamodel.Node, error) { qt.Check(t, progress.Path.String(), qt.Equals, "newsection/newpart") qt.Check(t, prev, qt.IsNil) // REVIEW: should datamodel.Absent be used here? I lean towards "no" but am unsure what's least surprising here. return basicnode.NewString("new string!"), nil }, true) qt.Check(t, err, qt.IsNil) qt.Check(t, n.Kind(), qt.Equals, datamodel.Kind_Map) // a new map node in the middle should've been created n2 := must.Node(n.LookupByString("newsection")) qt.Check(t, n2.Kind(), qt.Equals, datamodel.Kind_Map) // updated value should in there qt.Check(t, must.Node(n2.LookupByString("newpart")), nodetests.NodeContentEquals, basicnode.NewString("new string!")) // everything in the root map should still be in the same order... with the new entry at the end. qt.Check(t, keys(n), qt.DeepEquals, []string{"plain", "linkedString", "linkedMap", "linkedList", "newsection"}) // and the created intermediate map of course has just one entry. qt.Check(t, keys(n2), qt.DeepEquals, []string{"newpart"}) }) t.Run("CreateParentsRequiresPermission", func(t *testing.T) { _, err := traversal.FocusedTransform(rootNode, datamodel.ParsePath("newsection/newpart"), func(progress traversal.Progress, prev datamodel.Node) (datamodel.Node, error) { qt.Check(t, true, qt.IsFalse) // ought not be reached return nil, nil }, false) qt.Check(t, err.Error(), qt.Equals, "transform: parent position at \"newsection\" did not exist (and createParents was false)") }) t.Run("UpdateListEntry", func(t *testing.T) { n, err := traversal.FocusedTransform(middleListNode, datamodel.ParsePath("2"), func(progress traversal.Progress, prev datamodel.Node) (datamodel.Node, error) { qt.Check(t, progress.Path.String(), qt.Equals, "2") qt.Check(t, prev, nodetests.NodeContentEquals, basicnode.NewLink(leafBetaLnk)) return basicnode.NewString("new string!"), nil }, false) qt.Check(t, err, qt.IsNil) qt.Check(t, n.Kind(), qt.Equals, datamodel.Kind_List) // updated value should be there qt.Check(t, must.Node(n.LookupByIndex(2)), nodetests.NodeContentEquals, basicnode.NewString("new string!")) // everything else should be there qt.Check(t, n.Length(), qt.Equals, int64(4)) qt.Check(t, must.Node(n.LookupByIndex(0)), nodetests.NodeContentEquals, basicnode.NewLink(leafAlphaLnk)) qt.Check(t, must.Node(n.LookupByIndex(1)), nodetests.NodeContentEquals, basicnode.NewLink(leafAlphaLnk)) qt.Check(t, must.Node(n.LookupByIndex(3)), nodetests.NodeContentEquals, basicnode.NewLink(leafAlphaLnk)) }) t.Run("AppendToList", func(t *testing.T) { n, err := traversal.FocusedTransform(middleListNode, datamodel.ParsePath("-"), func(progress traversal.Progress, prev datamodel.Node) (datamodel.Node, error) { qt.Check(t, progress.Path.String(), qt.Equals, "4") qt.Check(t, prev, qt.IsNil) return basicnode.NewString("new string!"), nil }, false) qt.Check(t, err, qt.IsNil) qt.Check(t, n.Kind(), qt.Equals, datamodel.Kind_List) // updated value should be there qt.Check(t, must.Node(n.LookupByIndex(4)), nodetests.NodeContentEquals, basicnode.NewString("new string!")) // everything else should be there qt.Check(t, n.Length(), qt.Equals, int64(5)) }) t.Run("ListBounds", func(t *testing.T) { _, err := traversal.FocusedTransform(middleListNode, datamodel.ParsePath("4"), func(progress traversal.Progress, prev datamodel.Node) (datamodel.Node, error) { qt.Check(t, true, qt.IsFalse) // ought not be reached return nil, nil }, false) qt.Check(t, err, qt.ErrorMatches, "transform: cannot navigate path segment \"4\" at \"\" because it is beyond the list bounds") }) t.Run("ReplaceRoot", func(t *testing.T) { // a fairly degenerate case and no reason to do this, but should work. n, err := traversal.FocusedTransform(middleListNode, datamodel.ParsePath(""), func(progress traversal.Progress, prev datamodel.Node) (datamodel.Node, error) { qt.Check(t, progress.Path.String(), qt.Equals, "") qt.Check(t, prev, nodetests.NodeContentEquals, middleListNode) nb := basicnode.Prototype.Any.NewBuilder() la, _ := nb.BeginList(0) la.Finish() return nb.Build(), nil }, false) qt.Check(t, err, qt.IsNil) qt.Check(t, n.Kind(), qt.Equals, datamodel.Kind_List) qt.Check(t, n.Length(), qt.Equals, int64(0)) }) } func TestFocusedTransformWithLinks(t *testing.T) { var store2 = memstore.Store{} lsys := cidlink.DefaultLinkSystem() lsys.SetReadStorage(&store) lsys.SetWriteStorage(&store2) cfg := traversal.Config{ LinkSystem: lsys, LinkTargetNodePrototypeChooser: basicnode.Chooser, } t.Run("UpdateMapBeyondLink", func(t *testing.T) { n, err := traversal.Progress{ Cfg: &cfg, }.FocusedTransform(rootNode, datamodel.ParsePath("linkedMap/nested/nonlink"), func(progress traversal.Progress, prev datamodel.Node) (datamodel.Node, error) { qt.Check(t, progress.Path.String(), qt.Equals, "linkedMap/nested/nonlink") qt.Check(t, must.String(prev), qt.Equals, "zoo") qt.Check(t, progress.LastBlock.Path.String(), qt.Equals, "linkedMap") qt.Check(t, progress.LastBlock.Link.String(), qt.Equals, "baguqeeyezhlahvq") nb := prev.Prototype().NewBuilder() nb.AssignString("new string!") return nb.Build(), nil }, false) qt.Check(t, err, qt.IsNil) qt.Check(t, n.Kind(), qt.Equals, datamodel.Kind_Map) // there should be a new object in our new storage! qt.Check(t, store2.Bag, qt.HasLen, 1) // cleanup for next test store2 = memstore.Store{} }) t.Run("UpdateNotBeyondLink", func(t *testing.T) { // This is replacing a link with a non-link. Doing so shouldn't hit storage. n, err := traversal.Progress{ Cfg: &cfg, }.FocusedTransform(rootNode, datamodel.ParsePath("linkedMap"), func(progress traversal.Progress, prev datamodel.Node) (datamodel.Node, error) { qt.Check(t, progress.Path.String(), qt.Equals, "linkedMap") nb := prev.Prototype().NewBuilder() nb.AssignString("new string!") return nb.Build(), nil }, false) qt.Check(t, err, qt.IsNil) qt.Check(t, n.Kind(), qt.Equals, datamodel.Kind_Map) // there should be no new objects in our new storage! qt.Check(t, store2.Bag, qt.HasLen, 0) // cleanup for next test store2 = memstore.Store{} }) // link traverse to scalar // this is unspecifiable using the current path syntax! you'll just end up replacing the link with the scalar! } func keys(n datamodel.Node) []string { v := make([]string, 0, n.Length()) for itr := n.MapIterator(); !itr.Done(); { k, _, _ := itr.Next() v = append(v, must.String(k)) } return v } ================================================ FILE: traversal/patch/eval.go ================================================ // Package patch provides an implementation of the IPLD Patch specification. // IPLD Patch is a system for declaratively specifying patches to a document, // which can then be applied to produce a new, modified document. // // This package is EXPERIMENTAL; its behavior and API might change as it's still // in development. package patch import ( "fmt" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/traversal" ) type Op string const ( Op_Add = "add" Op_Remove = "remove" Op_Replace = "replace" Op_Move = "move" Op_Copy = "copy" Op_Test = "test" ) type Operation struct { Op Op // Always required. Path datamodel.Path // Always required. Value datamodel.Node // Present on 'add', 'replace', 'test'. From datamodel.Path // Present on 'move', 'copy'. } func Eval(n datamodel.Node, ops []Operation) (datamodel.Node, error) { var err error for _, op := range ops { n, err = EvalOne(n, op) if err != nil { return nil, err } } return n, nil } func EvalOne(n datamodel.Node, op Operation) (datamodel.Node, error) { switch op.Op { case Op_Add: // The behavior of the 'add' op in jsonpatch varies based on if the parent of the target path is a list. // If the parent of the target path is a list, then 'add' is really more of an 'insert': it should slide the rest of the values down. // There's also a special case for "-", which means "append to the end of the list". // Otherwise, if the destination path exists, it's an error. (No upserting.) // Handling this requires looking at the parent of the destination node, so we split this into *two* traversal.FocusedTransform calls. return traversal.FocusedTransform(n, op.Path.Pop(), func(prog traversal.Progress, parent datamodel.Node) (datamodel.Node, error) { if parent.Kind() == datamodel.Kind_List { seg := op.Path.Last() var idx int64 if seg.String() == "-" { idx = -1 } var err error idx, err = seg.Index() if err != nil { return nil, fmt.Errorf("patch-invalid-path-through-list: at %q", op.Path) // TODO error structuralization and review the code } nb := parent.Prototype().NewBuilder() la, err := nb.BeginList(parent.Length() + 1) if err != nil { return nil, err } for itr := n.ListIterator(); !itr.Done(); { i, v, err := itr.Next() if err != nil { return nil, err } if idx == i { la.AssembleValue().AssignNode(op.Value) } if err := la.AssembleValue().AssignNode(v); err != nil { return nil, err } } // TODO: is one-past-the-end supposed to be supported or supposed to be ruled out? if idx == -1 { la.AssembleValue().AssignNode(op.Value) } if err := la.Finish(); err != nil { return nil, err } return nb.Build(), nil } return prog.FocusedTransform(parent, datamodel.NewPath([]datamodel.PathSegment{op.Path.Last()}), func(prog traversal.Progress, point datamodel.Node) (datamodel.Node, error) { if point != nil && !point.IsAbsent() { return nil, fmt.Errorf("patch-target-exists: at %q", op.Path) // TODO error structuralization and review the code } return op.Value, nil }, false) }, false) case "remove": return traversal.FocusedTransform(n, op.Path, func(_ traversal.Progress, point datamodel.Node) (datamodel.Node, error) { return nil, nil // Returning a nil value here means "remove what's here". }, false) case "replace": // TODO i think you need a check that it's not landing under itself here return traversal.FocusedTransform(n, op.Path, func(_ traversal.Progress, point datamodel.Node) (datamodel.Node, error) { return op.Value, nil // is this right? what does FocusedTransform do re upsert? }, false) case "move": // TODO i think you need a check that it's not landing under itself here source, err := traversal.Get(n, op.From) if err != nil { return nil, err } n, err := traversal.FocusedTransform(n, op.Path, func(_ traversal.Progress, point datamodel.Node) (datamodel.Node, error) { return source, nil // is this right? what does FocusedTransform do re upsert? }, false) if err != nil { return nil, err } return traversal.FocusedTransform(n, op.From, func(_ traversal.Progress, point datamodel.Node) (datamodel.Node, error) { return nil, nil // Returning a nil value here means "remove what's here". }, false) case "copy": // TODO i think you need a check that it's not landing under itself here source, err := traversal.Get(n, op.From) if err != nil { return nil, err } return traversal.FocusedTransform(n, op.Path, func(_ traversal.Progress, point datamodel.Node) (datamodel.Node, error) { return source, nil // is this right? what does FocusedTransform do re upsert? }, false) case "test": point, err := traversal.Get(n, op.Path) if err != nil { return nil, err } if datamodel.DeepEqual(point, op.Value) { return n, nil } return n, fmt.Errorf("test failed") // TODO real error handling and a code default: return nil, fmt.Errorf("misuse: invalid operation: %s", op.Op) // TODO real error handling and a code } } ================================================ FILE: traversal/patch/parse.go ================================================ package patch import ( _ "embed" "bytes" "io" "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/codec" "github.com/ipld/go-ipld-prime/node/bindnode" "github.com/ipld/go-ipld-prime/schema" "github.com/ipld/go-ipld-prime/codec/json" "github.com/ipld/go-ipld-prime/datamodel" ) //go:embed patch.ipldsch var embedSchema []byte var ts = func() *schema.TypeSystem { ts, err := ipld.LoadSchemaBytes(embedSchema) if err != nil { panic(err) } return ts }() func ParseBytes(b []byte, dec codec.Decoder) ([]Operation, error) { return Parse(bytes.NewReader(b), dec) } func Parse(r io.Reader, dec codec.Decoder) ([]Operation, error) { npt := bindnode.Prototype((*[]operationRaw)(nil), ts.TypeByName("OperationSequence")) nb := npt.Representation().NewBuilder() if err := json.Decode(nb, r); err != nil { return nil, err } opsRaw := bindnode.Unwrap(nb.Build()).(*[]operationRaw) var ops []Operation for _, opRaw := range *opsRaw { // TODO check the Op string op := Operation{ Op: Op(opRaw.Op), Path: datamodel.ParsePath(opRaw.Path), Value: opRaw.Value, } if opRaw.From != nil { op.From = datamodel.ParsePath(*opRaw.From) } ops = append(ops, op) } return ops, nil } // operationRaw is roughly the same structure as Operation, but more amenable to serialization // (it doesn't use high level library types that don't have a data model equivalent). type operationRaw struct { Op string Path string Value datamodel.Node From *string } ================================================ FILE: traversal/patch/patch.ipldsch ================================================ # Op represents the kind of operation to perform # The current set is based on the JSON Patch specification # We may end up adding more operations in the future type Op enum { | add | remove | replace | move | copy | test } # Operation and OperationSequence are the types that describe operations (but not what to apply them on). # See the Instruction type for describing both operations and what to apply them on. type Operation struct { op Op path String value optional Any from optional String } type OperationSequence [Operation] type Instruction struct { startAt Link operations OperationSequence # future: optional field for adl signalling and/or other lenses } type InstructionResult union { | Error "error" | Link "result" } representation keyed type Error struct { code String # enum forthcoming message String details {String:String} } ================================================ FILE: traversal/patch/patch_test.go ================================================ package patch import ( "bytes" "encoding/json" "os" "strings" "testing" qt "github.com/frankban/quicktest" "github.com/warpfork/go-testmark" "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/codec" "github.com/ipld/go-ipld-prime/codec/dagjson" ) func TestSpecFixtures(t *testing.T) { dir := "../../.ipld/specs/patch/fixtures/" testOneSpecFixtureFile(t, dir+"fixtures-1.md") } func testOneSpecFixtureFile(t *testing.T, filename string) { doc, err := testmark.ReadFile(filename) if os.IsNotExist(err) { t.Skipf("not running spec suite: %s (did you clone the submodule with the data?)", err) } if err != nil { t.Fatalf("spec file parse failed?!: %s", err) } // Data hunk in this spec file are in "directories" of a test scenario each. doc.BuildDirIndex() for _, dir := range doc.DirEnt.ChildrenList { t.Run(dir.Name, func(t *testing.T) { // Grab all the data hunks. // Each "directory" contains three piece of data: // - `initial` -- this is the "block". It's arbitrary example data. They're all in json (or dag-json) format, for simplicity. // - `patch` -- this is a list of patch ops. Again, as json. // - `result` -- this is the expected result object. Again, as json. initialBlob := dir.Children["initial"].Hunk.Body patchBlob := dir.Children["patch"].Hunk.Body resultBlob := dir.Children["result"].Hunk.Body // Parse everything. initial, err := ipld.Decode(initialBlob, dagjson.Decode) if err != nil { t.Fatalf("failed to parse fixture data: %s", err) } ops, err := ParseBytes(patchBlob, dagjson.Decode) if err != nil { t.Fatalf("failed to parse fixture patch: %s", err) } // We don't actually keep the decoded result object. We're just gonna serialize the result and textually diff that instead. _, err = ipld.Decode(resultBlob, dagjson.Decode) if err != nil { t.Fatalf("failed to parse fixture data: %s", err) } // Do the thing! actualResult, err := Eval(initial, ops) if strings.HasSuffix(dir.Name, "-fail") { if err == nil { t.Fatalf("patch was expected to fail") } else { return } } else { if err != nil { t.Fatalf("patch did not apply: %s", err) } } // Serialize (and pretty print) result, so that we can diff it. actualResultBlob, err := ipld.Encode(actualResult, dagjson.EncodeOptions{ EncodeLinks: true, EncodeBytes: true, MapSortMode: codec.MapSortMode_None, }.Encode) if err != nil { t.Errorf("failed to reserialize result: %s", err) } var actualResultBlobPretty bytes.Buffer json.Indent(&actualResultBlobPretty, actualResultBlob, "", "\t") // Diff! qt.Assert(t, actualResultBlobPretty.String()+"\n", qt.Equals, string(resultBlob)) }) } } ================================================ FILE: traversal/select_links.go ================================================ package traversal import ( "github.com/ipld/go-ipld-prime/datamodel" ) // SelectLinks walks a Node tree and returns a slice of all Links encountered. // SelectLinks will recurse down into any maps and lists, // but does not attempt to load any of the links it encounters nor recurse further through them // (in other words, it's confined to one "block"). // // SelectLinks only returns the list of links; it does not return any other information // about them such as position in the tree, etc. // // An error may be returned if any of the nodes returns errors during iteration; // this is generally only possible if one of the Nodes is an ADL, // and unable to be fully walked because of the inability to load or process some data inside the ADL. // Nodes already fully in memory should not encounter such errors, // and it should be safe to ignore errors from this method when used in that situation. // In case of an error, a partial list will still be returned. // // If an identical link is found several times during the walk, // it is reported several times in the resulting list; // no deduplication is performed by this method. func SelectLinks(n datamodel.Node) ([]datamodel.Link, error) { var answer []datamodel.Link err := accumulateLinks(&answer, n) return answer, err } func accumulateLinks(a *[]datamodel.Link, n datamodel.Node) error { switch n.Kind() { case datamodel.Kind_Map: for itr := n.MapIterator(); !itr.Done(); { _, v, err := itr.Next() if err != nil { return err } accumulateLinks(a, v) } case datamodel.Kind_List: for itr := n.ListIterator(); !itr.Done(); { _, v, err := itr.Next() if err != nil { return err } accumulateLinks(a, v) } case datamodel.Kind_Link: lnk, _ := n.AsLink() *a = append(*a, lnk) } return nil } ================================================ FILE: traversal/select_links_test.go ================================================ package traversal_test import ( "testing" qt "github.com/frankban/quicktest" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/traversal" ) func TestSelectLinks(t *testing.T) { t.Run("Scalar", func(t *testing.T) { lnks, _ := traversal.SelectLinks(leafAlpha) qt.Check(t, lnks, deepEqualsAllowAllUnexported, []datamodel.Link(nil)) }) t.Run("DeepMap", func(t *testing.T) { lnks, _ := traversal.SelectLinks(middleMapNode) qt.Check(t, lnks, deepEqualsAllowAllUnexported, []datamodel.Link{leafAlphaLnk}) }) t.Run("List", func(t *testing.T) { lnks, _ := traversal.SelectLinks(rootNode) qt.Check(t, lnks, deepEqualsAllowAllUnexported, []datamodel.Link{leafAlphaLnk, middleMapNodeLnk, middleListNodeLnk}) }) } ================================================ FILE: traversal/selector/builder/builder.go ================================================ package builder import ( "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/fluent" "github.com/ipld/go-ipld-prime/traversal/selector" ) // SelectorSpec is a specification for a selector that can build // a selector datamodel.Node or an actual parsed Selector type SelectorSpec interface { Node() datamodel.Node Selector() (selector.Selector, error) } // SelectorSpecBuilder is a utility interface to build selector ipld nodes // quickly. // // It serves two purposes: // 1. Save the user of go-ipld time and mental overhead with an easy // interface for making selector nodes in much less code without having to remember // the selector sigils // 2. Provide a level of protection from selector schema changes, at least in terms // of naming, if not structure type SelectorSpecBuilder interface { ExploreRecursiveEdge() SelectorSpec ExploreRecursive(limit selector.RecursionLimit, sequence SelectorSpec) SelectorSpec ExploreUnion(...SelectorSpec) SelectorSpec ExploreAll(next SelectorSpec) SelectorSpec ExploreIndex(index int64, next SelectorSpec) SelectorSpec ExploreRange(start, end int64, next SelectorSpec) SelectorSpec ExploreFields(ExploreFieldsSpecBuildingClosure) SelectorSpec ExploreInterpretAs(as string, next SelectorSpec) SelectorSpec Matcher() SelectorSpec MatcherSubset(from, to int64) SelectorSpec } // ExploreFieldsSpecBuildingClosure is a function that provided to SelectorSpecBuilder's // ExploreFields method that assembles the fields map in the selector using // an ExploreFieldsSpecBuilder type ExploreFieldsSpecBuildingClosure func(ExploreFieldsSpecBuilder) // ExploreFieldsSpecBuilder is an interface for assemble the map of fields to // selectors in ExploreFields type ExploreFieldsSpecBuilder interface { Insert(k string, v SelectorSpec) } type selectorSpecBuilder struct { np datamodel.NodePrototype } type selectorSpec struct { n datamodel.Node } func (ss selectorSpec) Node() datamodel.Node { return ss.n } func (ss selectorSpec) Selector() (selector.Selector, error) { return selector.ParseSelector(ss.n) } // NewSelectorSpecBuilder creates a SelectorSpecBuilder which will store // data in the format determined by the given datamodel.NodePrototype. func NewSelectorSpecBuilder(np datamodel.NodePrototype) SelectorSpecBuilder { return &selectorSpecBuilder{np} } func (ssb *selectorSpecBuilder) ExploreRecursiveEdge() SelectorSpec { return selectorSpec{ fluent.MustBuildMap(ssb.np, 1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_ExploreRecursiveEdge).CreateMap(0, func(na fluent.MapAssembler) {}) }), } } func (ssb *selectorSpecBuilder) ExploreRecursive(limit selector.RecursionLimit, sequence SelectorSpec) SelectorSpec { return selectorSpec{ fluent.MustBuildMap(ssb.np, 1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_ExploreRecursive).CreateMap(2, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_Limit).CreateMap(1, func(na fluent.MapAssembler) { switch limit.Mode() { case selector.RecursionLimit_Depth: na.AssembleEntry(selector.SelectorKey_LimitDepth).AssignInt(limit.Depth()) case selector.RecursionLimit_None: na.AssembleEntry(selector.SelectorKey_LimitNone).CreateMap(0, func(na fluent.MapAssembler) {}) default: panic("Unsupported recursion limit type") } }) na.AssembleEntry(selector.SelectorKey_Sequence).AssignNode(sequence.Node()) }) }), } } func (ssb *selectorSpecBuilder) ExploreAll(next SelectorSpec) SelectorSpec { return selectorSpec{ fluent.MustBuildMap(ssb.np, 1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_ExploreAll).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_Next).AssignNode(next.Node()) }) }), } } func (ssb *selectorSpecBuilder) ExploreIndex(index int64, next SelectorSpec) SelectorSpec { return selectorSpec{ fluent.MustBuildMap(ssb.np, 1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_ExploreIndex).CreateMap(2, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_Index).AssignInt(index) na.AssembleEntry(selector.SelectorKey_Next).AssignNode(next.Node()) }) }), } } func (ssb *selectorSpecBuilder) ExploreRange(start, end int64, next SelectorSpec) SelectorSpec { return selectorSpec{ fluent.MustBuildMap(ssb.np, 1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_ExploreRange).CreateMap(3, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_Start).AssignInt(start) na.AssembleEntry(selector.SelectorKey_End).AssignInt(end) na.AssembleEntry(selector.SelectorKey_Next).AssignNode(next.Node()) }) }), } } func (ssb *selectorSpecBuilder) ExploreUnion(members ...SelectorSpec) SelectorSpec { return selectorSpec{ fluent.MustBuildMap(ssb.np, 1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_ExploreUnion).CreateList(int64(len(members)), func(na fluent.ListAssembler) { for _, member := range members { na.AssembleValue().AssignNode(member.Node()) } }) }), } } func (ssb *selectorSpecBuilder) ExploreFields(specBuilder ExploreFieldsSpecBuildingClosure) SelectorSpec { return selectorSpec{ fluent.MustBuildMap(ssb.np, 1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_ExploreFields).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_Fields).CreateMap(-1, func(na fluent.MapAssembler) { specBuilder(exploreFieldsSpecBuilder{na}) }) }) }), } } func (ssb *selectorSpecBuilder) ExploreInterpretAs(as string, next SelectorSpec) SelectorSpec { return selectorSpec{ fluent.MustBuildMap(ssb.np, 1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_ExploreInterpretAs).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_As).AssignString(as) na.AssembleEntry(selector.SelectorKey_Next).AssignNode(next.Node()) }) }), } } func (ssb *selectorSpecBuilder) Matcher() SelectorSpec { return selectorSpec{ fluent.MustBuildMap(ssb.np, 1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_Matcher).CreateMap(0, func(na fluent.MapAssembler) {}) }), } } func (ssb *selectorSpecBuilder) MatcherSubset(from, to int64) SelectorSpec { return selectorSpec{ fluent.MustBuildMap(ssb.np, 1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_Matcher).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_Subset).CreateMap(2, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_From).AssignInt(from) na.AssembleEntry(selector.SelectorKey_To).AssignInt(to) }) }) }), } } type exploreFieldsSpecBuilder struct { na fluent.MapAssembler } func (efsb exploreFieldsSpecBuilder) Insert(field string, s SelectorSpec) { efsb.na.AssembleEntry(field).AssignNode(s.Node()) } ================================================ FILE: traversal/selector/builder/builder_test.go ================================================ package builder_test import ( "testing" qt "github.com/frankban/quicktest" "github.com/ipld/go-ipld-prime/fluent" "github.com/ipld/go-ipld-prime/node/basicnode" nodetests "github.com/ipld/go-ipld-prime/node/tests" "github.com/ipld/go-ipld-prime/traversal/selector" "github.com/ipld/go-ipld-prime/traversal/selector/builder" ) func TestBuildingSelectors(t *testing.T) { np := basicnode.Prototype.Any ssb := builder.NewSelectorSpecBuilder(np) t.Run("Matcher builds matcher nodes", func(t *testing.T) { sn := ssb.Matcher().Node() esn := fluent.MustBuildMap(np, 1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_Matcher).CreateMap(0, func(na fluent.MapAssembler) {}) }) qt.Check(t, sn, nodetests.NodeContentEquals, esn) }) t.Run("ExploreRecursiveEdge builds ExploreRecursiveEdge nodes", func(t *testing.T) { sn := ssb.ExploreRecursiveEdge().Node() esn := fluent.MustBuildMap(np, 1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_ExploreRecursiveEdge).CreateMap(0, func(na fluent.MapAssembler) {}) }) qt.Check(t, sn, nodetests.NodeContentEquals, esn) }) t.Run("ExploreAll builds ExploreAll nodes", func(t *testing.T) { sn := ssb.ExploreAll(ssb.Matcher()).Node() esn := fluent.MustBuildMap(np, 1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_ExploreAll).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_Next).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_Matcher).CreateMap(0, func(na fluent.MapAssembler) {}) }) }) }) qt.Check(t, sn, nodetests.NodeContentEquals, esn) }) t.Run("ExploreIndex builds ExploreIndex nodes", func(t *testing.T) { sn := ssb.ExploreIndex(2, ssb.Matcher()).Node() esn := fluent.MustBuildMap(np, 1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_ExploreIndex).CreateMap(2, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_Index).AssignInt(2) na.AssembleEntry(selector.SelectorKey_Next).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_Matcher).CreateMap(0, func(na fluent.MapAssembler) {}) }) }) }) qt.Check(t, sn, nodetests.NodeContentEquals, esn) }) t.Run("ExploreRange builds ExploreRange nodes", func(t *testing.T) { sn := ssb.ExploreRange(2, 3, ssb.Matcher()).Node() esn := fluent.MustBuildMap(np, 1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_ExploreRange).CreateMap(3, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_Start).AssignInt(2) na.AssembleEntry(selector.SelectorKey_End).AssignInt(3) na.AssembleEntry(selector.SelectorKey_Next).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_Matcher).CreateMap(0, func(na fluent.MapAssembler) {}) }) }) }) qt.Check(t, sn, nodetests.NodeContentEquals, esn) }) t.Run("ExploreRecursive builds ExploreRecursive nodes", func(t *testing.T) { sn := ssb.ExploreRecursive(selector.RecursionLimitDepth(2), ssb.ExploreAll(ssb.ExploreRecursiveEdge())).Node() esn := fluent.MustBuildMap(np, 1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_ExploreRecursive).CreateMap(2, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_Limit).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_LimitDepth).AssignInt(2) }) na.AssembleEntry(selector.SelectorKey_Sequence).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_ExploreAll).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_Next).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_ExploreRecursiveEdge).CreateMap(0, func(na fluent.MapAssembler) {}) }) }) }) }) }) qt.Check(t, sn, nodetests.NodeContentEquals, esn) sn = ssb.ExploreRecursive(selector.RecursionLimitNone(), ssb.ExploreAll(ssb.ExploreRecursiveEdge())).Node() esn = fluent.MustBuildMap(np, 1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_ExploreRecursive).CreateMap(2, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_Limit).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_LimitNone).CreateMap(0, func(na fluent.MapAssembler) {}) }) na.AssembleEntry(selector.SelectorKey_Sequence).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_ExploreAll).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_Next).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_ExploreRecursiveEdge).CreateMap(0, func(na fluent.MapAssembler) {}) }) }) }) }) }) qt.Check(t, sn, nodetests.NodeContentEquals, esn) }) t.Run("ExploreUnion builds ExploreUnion nodes", func(t *testing.T) { sn := ssb.ExploreUnion(ssb.Matcher(), ssb.ExploreIndex(2, ssb.Matcher())).Node() esn := fluent.MustBuildMap(np, 1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_ExploreUnion).CreateList(2, func(na fluent.ListAssembler) { na.AssembleValue().CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_Matcher).CreateMap(0, func(na fluent.MapAssembler) {}) }) na.AssembleValue().CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_ExploreIndex).CreateMap(2, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_Index).AssignInt(2) na.AssembleEntry(selector.SelectorKey_Next).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_Matcher).CreateMap(0, func(na fluent.MapAssembler) {}) }) }) }) }) }) qt.Check(t, sn, nodetests.NodeContentEquals, esn) }) t.Run("ExploreFields builds ExploreFields nodes", func(t *testing.T) { sn := ssb.ExploreFields(func(efsb builder.ExploreFieldsSpecBuilder) { efsb.Insert("applesauce", ssb.Matcher()) }).Node() esn := fluent.MustBuildMap(np, 1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_ExploreFields).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_Fields).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry("applesauce").CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_Matcher).CreateMap(0, func(na fluent.MapAssembler) {}) }) }) }) }) qt.Check(t, sn, nodetests.NodeContentEquals, esn) }) } ================================================ FILE: traversal/selector/condition.go ================================================ package selector import ( "fmt" "github.com/ipld/go-ipld-prime/datamodel" cidlink "github.com/ipld/go-ipld-prime/linking/cid" ) // Condition provides a mechanism for matching and limiting matching and // exploration of selectors. // Not all types of conditions which are imagined in the selector specification // are encoded at present, instead we currently only implement a subset that // is sufficient for initial pressing use cases. type Condition struct { mode ConditionMode match datamodel.Node } // A ConditionMode is the keyed representation for the union that is the condition type ConditionMode string const ( ConditionMode_Link ConditionMode = "/" ) // Match decides if a given datamodel.Node matches the condition. func (c *Condition) Match(n datamodel.Node) bool { switch c.mode { case ConditionMode_Link: if n.Kind() != datamodel.Kind_Link { return false } lnk, err := n.AsLink() if err != nil { return false } match, err := c.match.AsLink() if err != nil { return false } cidlnk, ok := lnk.(cidlink.Link) cidmatch, ok2 := match.(cidlink.Link) if ok && ok2 { return cidmatch.Equals(cidlnk.Cid) } return match.String() == lnk.String() default: return false } } // ParseCondition assembles a Condition from a condition selector node func (pc ParseContext) ParseCondition(n datamodel.Node) (Condition, error) { if n.Kind() != datamodel.Kind_Map { return Condition{}, fmt.Errorf("selector spec parse rejected: condition body must be a map") } if n.Length() != 1 { return Condition{}, fmt.Errorf("selector spec parse rejected: condition is a keyed union and thus must be single-entry map") } kn, v, _ := n.MapIterator().Next() kstr, _ := kn.AsString() // Switch over the single key to determine which condition body comes next. // (This switch is where the keyed union discriminators concretely happen.) switch ConditionMode(kstr) { case ConditionMode_Link: if _, err := v.AsLink(); err != nil { return Condition{}, fmt.Errorf("selector spec parse rejected: condition_link must be a link") } return Condition{mode: ConditionMode_Link, match: v}, nil default: return Condition{}, fmt.Errorf("selector spec parse rejected: %q is not a known member of the condition union", kstr) } } ================================================ FILE: traversal/selector/condition_test.go ================================================ package selector import ( "reflect" "testing" qt "github.com/frankban/quicktest" "github.com/google/go-cmp/cmp" "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime/fluent" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipld/go-ipld-prime/node/basicnode" ) var deepEqualsAllowAllUnexported = qt.CmpEquals(cmp.Exporter(func(reflect.Type) bool { return true })) func TestParseCondition(t *testing.T) { t.Run("parsing non map node should error", func(t *testing.T) { sn := basicnode.NewInt(0) _, err := ParseContext{}.ParseCondition(sn) qt.Check(t, err, qt.ErrorMatches, "selector spec parse rejected: condition body must be a map") }) t.Run("parsing map node without field should error", func(t *testing.T) { sn := fluent.MustBuildMap(basicnode.Prototype.Map, 0, func(na fluent.MapAssembler) {}) _, err := ParseContext{}.ParseCondition(sn) qt.Check(t, err, qt.ErrorMatches, "selector spec parse rejected: condition is a keyed union and thus must be single-entry map") }) t.Run("parsing map node keyed to invalid type should error", func(t *testing.T) { sn := fluent.MustBuildMap(basicnode.Prototype.Map, 1, func(na fluent.MapAssembler) { na.AssembleEntry(string(ConditionMode_Link)).AssignInt(0) }) _, err := ParseContext{}.ParseCondition(sn) qt.Check(t, err, qt.ErrorMatches, "selector spec parse rejected: condition_link must be a link") }) t.Run("parsing map node with condition field with valid selector node should parse", func(t *testing.T) { lnk := cidlink.Link{Cid: cid.Undef} sn := fluent.MustBuildMap(basicnode.Prototype.Map, 1, func(na fluent.MapAssembler) { na.AssembleEntry(string(ConditionMode_Link)).AssignLink(lnk) }) s, err := ParseContext{}.ParseCondition(sn) qt.Check(t, err, qt.IsNil) lnkNode := basicnode.NewLink(lnk) qt.Check(t, s, deepEqualsAllowAllUnexported, Condition{mode: ConditionMode_Link, match: lnkNode}) }) } ================================================ FILE: traversal/selector/exploreAll.go ================================================ package selector import ( "fmt" "github.com/ipld/go-ipld-prime/datamodel" ) // ExploreAll is similar to a `*` -- it traverses all elements of an array, // or all entries in a map, and applies a next selector to the reached nodes. type ExploreAll struct { next Selector // selector for element we're interested in } // Interests for ExploreAll is nil (meaning traverse everything) func (s ExploreAll) Interests() []datamodel.PathSegment { return nil } // Explore returns the node's selector for all fields func (s ExploreAll) Explore(n datamodel.Node, p datamodel.PathSegment) (Selector, error) { return s.next, nil } // Decide always returns false because this is not a matcher func (s ExploreAll) Decide(n datamodel.Node) bool { return false } // Match always returns false because this is not a matcher func (s ExploreAll) Match(node datamodel.Node) (datamodel.Node, error) { return nil, nil } // ParseExploreAll assembles a Selector from a ExploreAll selector node func (pc ParseContext) ParseExploreAll(n datamodel.Node) (Selector, error) { if n.Kind() != datamodel.Kind_Map { return nil, fmt.Errorf("selector spec parse rejected: selector body must be a map") } next, err := n.LookupByString(SelectorKey_Next) if err != nil { return nil, fmt.Errorf("selector spec parse rejected: next field must be present in ExploreAll selector") } selector, err := pc.ParseSelector(next) if err != nil { return nil, err } return ExploreAll{selector}, nil } ================================================ FILE: traversal/selector/exploreAll_test.go ================================================ package selector import ( "testing" qt "github.com/frankban/quicktest" "github.com/ipld/go-ipld-prime/fluent" "github.com/ipld/go-ipld-prime/node/basicnode" ) func TestParseExploreAll(t *testing.T) { t.Run("parsing non map node should error", func(t *testing.T) { sn := basicnode.NewInt(0) _, err := ParseContext{}.ParseExploreAll(sn) qt.Check(t, err, qt.ErrorMatches, "selector spec parse rejected: selector body must be a map") }) t.Run("parsing map node without next field should error", func(t *testing.T) { sn := fluent.MustBuildMap(basicnode.Prototype.Map, 0, func(na fluent.MapAssembler) {}) _, err := ParseContext{}.ParseExploreAll(sn) qt.Check(t, err, qt.ErrorMatches, "selector spec parse rejected: next field must be present in ExploreAll selector") }) t.Run("parsing map node without next field with invalid selector node should return child's error", func(t *testing.T) { sn := fluent.MustBuildMap(basicnode.Prototype.Map, 1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Next).AssignInt(0) }) _, err := ParseContext{}.ParseExploreAll(sn) qt.Check(t, err, qt.ErrorMatches, "selector spec parse rejected: selector is a keyed union and thus must be a map") }) t.Run("parsing map node with next field with valid selector node should parse", func(t *testing.T) { sn := fluent.MustBuildMap(basicnode.Prototype.Map, 1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Next).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Matcher).CreateMap(0, func(na fluent.MapAssembler) {}) }) }) s, err := ParseContext{}.ParseExploreAll(sn) qt.Check(t, err, qt.IsNil) qt.Check(t, s, qt.Equals, ExploreAll{Matcher{}}) }) } ================================================ FILE: traversal/selector/exploreFields.go ================================================ package selector import ( "fmt" "github.com/ipld/go-ipld-prime/datamodel" ) // ExploreFields traverses named fields in a map (or equivalently, struct, if // traversing on typed/schema nodes) and applies a next selector to the // reached nodes. // // Note that a concept of "ExplorePath" (e.g. "foo/bar/baz") can be represented // as a set of three nexted ExploreFields selectors, each specifying one field. // (For this reason, we don't have a special "ExplorePath" feature; use this.) // // ExploreFields also works for selecting specific elements out of a list; // if a "field" is a base-10 int, it will be coerced and do the right thing. // ExploreIndex or ExploreRange is more appropriate, however, and should be preferred. type ExploreFields struct { selections map[string]Selector interests []datamodel.PathSegment // keys of above; already boxed as that's the only way we consume them } // Interests for ExploreFields are the fields listed in the selector node func (s ExploreFields) Interests() []datamodel.PathSegment { return s.interests } // Explore returns the selector for the given path if it is a field in // the selector node or nil if not func (s ExploreFields) Explore(n datamodel.Node, p datamodel.PathSegment) (Selector, error) { return s.selections[p.String()], nil } // Decide always returns false because this is not a matcher func (s ExploreFields) Decide(n datamodel.Node) bool { return false } // Match always returns false because this is not a matcher func (s ExploreFields) Match(node datamodel.Node) (datamodel.Node, error) { return nil, nil } // ParseExploreFields assembles a Selector // from a ExploreFields selector node func (pc ParseContext) ParseExploreFields(n datamodel.Node) (Selector, error) { if n.Kind() != datamodel.Kind_Map { return nil, fmt.Errorf("selector spec parse rejected: selector body must be a map") } fields, err := n.LookupByString(SelectorKey_Fields) if err != nil { return nil, fmt.Errorf("selector spec parse rejected: fields in ExploreFields selector must be present") } if fields.Kind() != datamodel.Kind_Map { return nil, fmt.Errorf("selector spec parse rejected: fields in ExploreFields selector must be a map") } x := ExploreFields{ make(map[string]Selector, fields.Length()), make([]datamodel.PathSegment, 0, fields.Length()), } for itr := fields.MapIterator(); !itr.Done(); { kn, v, err := itr.Next() if err != nil { return nil, fmt.Errorf("error during selector spec parse: %w", err) } kstr, _ := kn.AsString() x.interests = append(x.interests, datamodel.PathSegmentOfString(kstr)) x.selections[kstr], err = pc.ParseSelector(v) if err != nil { return nil, err } } return x, nil } ================================================ FILE: traversal/selector/exploreFields_test.go ================================================ package selector import ( "testing" qt "github.com/frankban/quicktest" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/fluent" "github.com/ipld/go-ipld-prime/node/basicnode" ) func TestParseExploreFields(t *testing.T) { t.Run("parsing non map node should error", func(t *testing.T) { sn := basicnode.NewInt(0) _, err := ParseContext{}.ParseExploreFields(sn) qt.Check(t, err, qt.ErrorMatches, "selector spec parse rejected: selector body must be a map") }) t.Run("parsing map node without fields value should error", func(t *testing.T) { sn := fluent.MustBuildMap(basicnode.Prototype.Map, 0, func(na fluent.MapAssembler) {}) _, err := ParseContext{}.ParseExploreFields(sn) qt.Check(t, err, qt.ErrorMatches, "selector spec parse rejected: fields in ExploreFields selector must be present") }) t.Run("parsing map node with fields value that is not a map should error", func(t *testing.T) { sn := fluent.MustBuildMap(basicnode.Prototype.Map, 1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Fields).AssignString("cheese") }) _, err := ParseContext{}.ParseExploreFields(sn) qt.Check(t, err, qt.ErrorMatches, "selector spec parse rejected: fields in ExploreFields selector must be a map") }) t.Run("parsing map node with selector node in fields that is invalid should return child's error", func(t *testing.T) { sn := fluent.MustBuildMap(basicnode.Prototype.Map, 1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Fields).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry("applesauce").AssignInt(0) }) }) _, err := ParseContext{}.ParseExploreFields(sn) qt.Check(t, err, qt.ErrorMatches, "selector spec parse rejected: selector is a keyed union and thus must be a map") }) t.Run("parsing map node with fields value that is map of only valid selector node should parse", func(t *testing.T) { sn := fluent.MustBuildMap(basicnode.Prototype.Map, 1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Fields).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry("applesauce").CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Matcher).CreateMap(0, func(na fluent.MapAssembler) {}) }) }) }) s, err := ParseContext{}.ParseExploreFields(sn) qt.Check(t, err, qt.IsNil) qt.Check(t, s, deepEqualsAllowAllUnexported, ExploreFields{map[string]Selector{"applesauce": Matcher{}}, []datamodel.PathSegment{datamodel.PathSegmentOfString("applesauce")}}) }) } ================================================ FILE: traversal/selector/exploreIndex.go ================================================ package selector import ( "fmt" "github.com/ipld/go-ipld-prime/datamodel" ) // ExploreIndex traverses a specific index in a list, and applies a next // selector to the reached node. type ExploreIndex struct { next Selector // selector for element we're interested in interest [1]datamodel.PathSegment // index of element we're interested in } // Interests for ExploreIndex is just the index specified by the selector node func (s ExploreIndex) Interests() []datamodel.PathSegment { return s.interest[:] } // Explore returns the node's selector if // the path matches the index for this selector or nil if not func (s ExploreIndex) Explore(n datamodel.Node, p datamodel.PathSegment) (Selector, error) { if n.Kind() != datamodel.Kind_List { return nil, nil } expectedIndex, expectedErr := p.Index() actualIndex, actualErr := s.interest[0].Index() if expectedErr != nil || actualErr != nil || expectedIndex != actualIndex { return nil, nil } return s.next, nil } // Decide always returns false because this is not a matcher func (s ExploreIndex) Decide(n datamodel.Node) bool { return false } // Match always returns false because this is not a matcher func (s ExploreIndex) Match(node datamodel.Node) (datamodel.Node, error) { return nil, nil } // ParseExploreIndex assembles a Selector // from a ExploreIndex selector node func (pc ParseContext) ParseExploreIndex(n datamodel.Node) (Selector, error) { if n.Kind() != datamodel.Kind_Map { return nil, fmt.Errorf("selector spec parse rejected: selector body must be a map") } indexNode, err := n.LookupByString(SelectorKey_Index) if err != nil { return nil, fmt.Errorf("selector spec parse rejected: index field must be present in ExploreIndex selector") } indexValue, err := indexNode.AsInt() if err != nil { return nil, fmt.Errorf("selector spec parse rejected: index field must be a number in ExploreIndex selector") } next, err := n.LookupByString(SelectorKey_Next) if err != nil { return nil, fmt.Errorf("selector spec parse rejected: next field must be present in ExploreIndex selector") } selector, err := pc.ParseSelector(next) if err != nil { return nil, err } return ExploreIndex{selector, [1]datamodel.PathSegment{datamodel.PathSegmentOfInt(indexValue)}}, nil } ================================================ FILE: traversal/selector/exploreIndex_test.go ================================================ package selector import ( "testing" qt "github.com/frankban/quicktest" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/fluent" "github.com/ipld/go-ipld-prime/node/basicnode" ) func TestParseExploreIndex(t *testing.T) { t.Run("parsing non map node should error", func(t *testing.T) { sn := basicnode.NewInt(0) _, err := ParseContext{}.ParseExploreIndex(sn) qt.Check(t, err, qt.ErrorMatches, "selector spec parse rejected: selector body must be a map") }) t.Run("parsing map node without next field should error", func(t *testing.T) { sn := fluent.MustBuildMap(basicnode.Prototype__Map{}, 1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Index).AssignInt(2) }) _, err := ParseContext{}.ParseExploreIndex(sn) qt.Check(t, err, qt.ErrorMatches, "selector spec parse rejected: next field must be present in ExploreIndex selector") }) t.Run("parsing map node without index field should error", func(t *testing.T) { sn := fluent.MustBuildMap(basicnode.Prototype__Map{}, 1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Next).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Matcher).CreateMap(0, func(na fluent.MapAssembler) {}) }) }) _, err := ParseContext{}.ParseExploreIndex(sn) qt.Check(t, err, qt.ErrorMatches, "selector spec parse rejected: index field must be present in ExploreIndex selector") }) t.Run("parsing map node with index field that is not an int should error", func(t *testing.T) { sn := fluent.MustBuildMap(basicnode.Prototype__Map{}, 2, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Index).AssignString("cheese") na.AssembleEntry(SelectorKey_Next).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Matcher).CreateMap(0, func(na fluent.MapAssembler) {}) }) }) _, err := ParseContext{}.ParseExploreIndex(sn) qt.Check(t, err, qt.ErrorMatches, "selector spec parse rejected: index field must be a number in ExploreIndex selector") }) t.Run("parsing map node with next field with invalid selector node should return child's error", func(t *testing.T) { sn := fluent.MustBuildMap(basicnode.Prototype__Map{}, 2, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Index).AssignInt(2) na.AssembleEntry(SelectorKey_Next).AssignInt(0) }) _, err := ParseContext{}.ParseExploreIndex(sn) qt.Check(t, err, qt.ErrorMatches, "selector spec parse rejected: selector is a keyed union and thus must be a map") }) t.Run("parsing map node with next field with valid selector node should parse", func(t *testing.T) { sn := fluent.MustBuildMap(basicnode.Prototype__Map{}, 2, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Index).AssignInt(2) na.AssembleEntry(SelectorKey_Next).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Matcher).CreateMap(0, func(na fluent.MapAssembler) {}) }) }) s, err := ParseContext{}.ParseExploreIndex(sn) qt.Check(t, err, qt.IsNil) qt.Check(t, s, qt.Equals, ExploreIndex{Matcher{}, [1]datamodel.PathSegment{datamodel.PathSegmentOfInt(2)}}) }) } func TestExploreIndexExplore(t *testing.T) { s := ExploreIndex{Matcher{}, [1]datamodel.PathSegment{datamodel.PathSegmentOfInt(3)}} t.Run("exploring should return nil unless node is a list", func(t *testing.T) { n := fluent.MustBuildMap(basicnode.Prototype__Map{}, 0, func(na fluent.MapAssembler) {}) returnedSelector, _ := s.Explore(n, datamodel.PathSegmentOfInt(3)) qt.Check(t, returnedSelector, qt.IsNil) }) n := fluent.MustBuildList(basicnode.Prototype__List{}, 4, func(na fluent.ListAssembler) { na.AssembleValue().AssignInt(0) na.AssembleValue().AssignInt(1) na.AssembleValue().AssignInt(2) na.AssembleValue().AssignInt(3) }) t.Run("exploring should return nil when given a path segment with a different index", func(t *testing.T) { returnedSelector, _ := s.Explore(n, datamodel.PathSegmentOfInt(2)) qt.Check(t, returnedSelector, qt.IsNil) }) t.Run("exploring should return nil when given a path segment that isn't an index", func(t *testing.T) { returnedSelector, _ := s.Explore(n, datamodel.PathSegmentOfString("cheese")) qt.Check(t, returnedSelector, qt.IsNil) }) t.Run("exploring should return the next selector when given a path segment with the right index", func(t *testing.T) { returnedSelector, _ := s.Explore(n, datamodel.PathSegmentOfInt(3)) qt.Check(t, returnedSelector, qt.Equals, Matcher{}) }) } ================================================ FILE: traversal/selector/exploreInterpretAs.go ================================================ package selector import ( "fmt" "github.com/ipld/go-ipld-prime/datamodel" ) type ExploreInterpretAs struct { next Selector // selector for element we're interested in adl string // reifier for the ADL we're interested in } // Interests for ExploreIndex is just the index specified by the selector node func (s ExploreInterpretAs) Interests() []datamodel.PathSegment { return s.next.Interests() } // Explore returns the node's selector if // the path matches the index for this selector or nil if not func (s ExploreInterpretAs) Explore(n datamodel.Node, p datamodel.PathSegment) (Selector, error) { return s.next, nil } // Decide always returns false because this is not a matcher func (s ExploreInterpretAs) Decide(n datamodel.Node) bool { return false } // Match always returns false because this is not a matcher func (s ExploreInterpretAs) Match(node datamodel.Node) (datamodel.Node, error) { return nil, nil } // NamedReifier indicates how this selector expects to Reify the current datamodel.Node. func (s ExploreInterpretAs) NamedReifier() string { return s.adl } // Reifiable provides a feature detection interface on selectors to understand when // and if Reification of the datamodel.node should be attempted when performing traversals. type Reifiable interface { NamedReifier() string } // ParseExploreInterpretAs assembles a Selector // from a ExploreInterpretAs selector node func (pc ParseContext) ParseExploreInterpretAs(n datamodel.Node) (Selector, error) { if n.Kind() != datamodel.Kind_Map { return nil, fmt.Errorf("selector spec parse rejected: selector body must be a map") } adlNode, err := n.LookupByString(SelectorKey_As) if err != nil { return nil, fmt.Errorf("selector spec parse rejected: the 'as' field must be present in ExploreInterpretAs clause") } next, err := n.LookupByString(SelectorKey_Next) if err != nil { return nil, fmt.Errorf("selector spec parse rejected: the 'next' field must be present in ExploreInterpretAs clause") } selector, err := pc.ParseSelector(next) if err != nil { return nil, err } adl, err := adlNode.AsString() if err != nil { return nil, err } return ExploreInterpretAs{selector, adl}, nil } ================================================ FILE: traversal/selector/exploreRange.go ================================================ package selector import ( "fmt" "github.com/ipld/go-ipld-prime/datamodel" ) // ExploreRange traverses a list, and for each element in the range specified, // will apply a next selector to those reached nodes. type ExploreRange struct { next Selector // selector for element we're interested in start int64 end int64 interest []datamodel.PathSegment // index of element we're interested in } // Interests for ExploreRange are all path segments within the iteration range func (s ExploreRange) Interests() []datamodel.PathSegment { return s.interest } // Explore returns the node's selector if // the path matches an index in the range of this selector func (s ExploreRange) Explore(n datamodel.Node, p datamodel.PathSegment) (Selector, error) { if n.Kind() != datamodel.Kind_List { return nil, nil } index, err := p.Index() if err != nil { return nil, nil } if index < s.start || index >= s.end { return nil, nil } return s.next, nil } // Decide always returns false because this is not a matcher func (s ExploreRange) Decide(n datamodel.Node) bool { return false } // Match always returns false because this is not a matcher func (s ExploreRange) Match(node datamodel.Node) (datamodel.Node, error) { return nil, nil } // ParseExploreRange assembles a Selector // from a ExploreRange selector node func (pc ParseContext) ParseExploreRange(n datamodel.Node) (Selector, error) { if n.Kind() != datamodel.Kind_Map { return nil, fmt.Errorf("selector spec parse rejected: selector body must be a map") } startNode, err := n.LookupByString(SelectorKey_Start) if err != nil { return nil, fmt.Errorf("selector spec parse rejected: start field must be present in ExploreRange selector") } startValue, err := startNode.AsInt() if err != nil { return nil, fmt.Errorf("selector spec parse rejected: start field must be a number in ExploreRange selector") } endNode, err := n.LookupByString(SelectorKey_End) if err != nil { return nil, fmt.Errorf("selector spec parse rejected: end field must be present in ExploreRange selector") } endValue, err := endNode.AsInt() if err != nil { return nil, fmt.Errorf("selector spec parse rejected: end field must be a number in ExploreRange selector") } if startValue >= endValue { return nil, fmt.Errorf("selector spec parse rejected: end field must be greater than start field in ExploreRange selector") } next, err := n.LookupByString(SelectorKey_Next) if err != nil { return nil, fmt.Errorf("selector spec parse rejected: next field must be present in ExploreRange selector") } selector, err := pc.ParseSelector(next) if err != nil { return nil, err } x := ExploreRange{ selector, startValue, endValue, make([]datamodel.PathSegment, 0, endValue-startValue), } for i := startValue; i < endValue; i++ { x.interest = append(x.interest, datamodel.PathSegmentOfInt(i)) } return x, nil } ================================================ FILE: traversal/selector/exploreRange_test.go ================================================ package selector import ( "testing" qt "github.com/frankban/quicktest" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/fluent" "github.com/ipld/go-ipld-prime/node/basicnode" ) func TestParseExploreRange(t *testing.T) { t.Run("parsing non map node should error", func(t *testing.T) { sn := basicnode.NewInt(0) _, err := ParseContext{}.ParseExploreRange(sn) qt.Check(t, err, qt.ErrorMatches, "selector spec parse rejected: selector body must be a map") }) t.Run("parsing map node without next field should error", func(t *testing.T) { sn := fluent.MustBuildMap(basicnode.Prototype__Map{}, 2, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Start).AssignInt(2) na.AssembleEntry(SelectorKey_End).AssignInt(3) }) _, err := ParseContext{}.ParseExploreRange(sn) qt.Check(t, err, qt.ErrorMatches, "selector spec parse rejected: next field must be present in ExploreRange selector") }) t.Run("parsing map node without start field should error", func(t *testing.T) { sn := fluent.MustBuildMap(basicnode.Prototype__Map{}, 2, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_End).AssignInt(3) na.AssembleEntry(SelectorKey_Next).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Matcher).CreateMap(0, func(na fluent.MapAssembler) {}) }) }) _, err := ParseContext{}.ParseExploreRange(sn) qt.Check(t, err, qt.ErrorMatches, "selector spec parse rejected: start field must be present in ExploreRange selector") }) t.Run("parsing map node with start field that is not an int should error", func(t *testing.T) { sn := fluent.MustBuildMap(basicnode.Prototype__Map{}, 3, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Start).AssignString("cheese") na.AssembleEntry(SelectorKey_End).AssignInt(3) na.AssembleEntry(SelectorKey_Next).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Matcher).CreateMap(0, func(na fluent.MapAssembler) {}) }) }) _, err := ParseContext{}.ParseExploreRange(sn) qt.Check(t, err, qt.ErrorMatches, "selector spec parse rejected: start field must be a number in ExploreRange selector") }) t.Run("parsing map node without end field should error", func(t *testing.T) { sn := fluent.MustBuildMap(basicnode.Prototype__Map{}, 2, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Start).AssignInt(2) na.AssembleEntry(SelectorKey_Next).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Matcher).CreateMap(0, func(na fluent.MapAssembler) {}) }) }) _, err := ParseContext{}.ParseExploreRange(sn) qt.Check(t, err, qt.ErrorMatches, "selector spec parse rejected: end field must be present in ExploreRange selector") }) t.Run("parsing map node with end field that is not an int should error", func(t *testing.T) { sn := fluent.MustBuildMap(basicnode.Prototype__Map{}, 3, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Start).AssignInt(2) na.AssembleEntry(SelectorKey_End).AssignString("cheese") na.AssembleEntry(SelectorKey_Next).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Matcher).CreateMap(0, func(na fluent.MapAssembler) {}) }) }) _, err := ParseContext{}.ParseExploreRange(sn) qt.Check(t, err, qt.ErrorMatches, "selector spec parse rejected: end field must be a number in ExploreRange selector") }) t.Run("parsing map node where end is not greater than start should error", func(t *testing.T) { sn := fluent.MustBuildMap(basicnode.Prototype__Map{}, 3, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Start).AssignInt(3) na.AssembleEntry(SelectorKey_End).AssignInt(2) na.AssembleEntry(SelectorKey_Next).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Matcher).CreateMap(0, func(na fluent.MapAssembler) {}) }) }) _, err := ParseContext{}.ParseExploreRange(sn) qt.Check(t, err, qt.ErrorMatches, "selector spec parse rejected: end field must be greater than start field in ExploreRange selector") }) t.Run("parsing map node with next field with invalid selector node should return child's error", func(t *testing.T) { sn := fluent.MustBuildMap(basicnode.Prototype__Map{}, 3, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Start).AssignInt(2) na.AssembleEntry(SelectorKey_End).AssignInt(3) na.AssembleEntry(SelectorKey_Next).AssignInt(0) }) _, err := ParseContext{}.ParseExploreRange(sn) qt.Check(t, err, qt.ErrorMatches, "selector spec parse rejected: selector is a keyed union and thus must be a map") }) t.Run("parsing map node with next field with valid selector node should parse", func(t *testing.T) { sn := fluent.MustBuildMap(basicnode.Prototype__Map{}, 3, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Start).AssignInt(2) na.AssembleEntry(SelectorKey_End).AssignInt(3) na.AssembleEntry(SelectorKey_Next).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Matcher).CreateMap(0, func(na fluent.MapAssembler) {}) }) }) s, err := ParseContext{}.ParseExploreRange(sn) qt.Check(t, err, qt.IsNil) qt.Check(t, s, deepEqualsAllowAllUnexported, ExploreRange{Matcher{}, 2, 3, []datamodel.PathSegment{datamodel.PathSegmentOfInt(2)}}) }) } func TestExploreRangeExplore(t *testing.T) { s := ExploreRange{Matcher{}, 3, 4, []datamodel.PathSegment{datamodel.PathSegmentOfInt(3)}} t.Run("exploring should return nil unless node is a list", func(t *testing.T) { n := fluent.MustBuildMap(basicnode.Prototype__Map{}, 0, func(na fluent.MapAssembler) {}) returnedSelector, _ := s.Explore(n, datamodel.PathSegmentOfInt(3)) qt.Check(t, returnedSelector, qt.IsNil) }) n := fluent.MustBuildList(basicnode.Prototype__List{}, 4, func(na fluent.ListAssembler) { na.AssembleValue().AssignInt(0) na.AssembleValue().AssignInt(1) na.AssembleValue().AssignInt(2) na.AssembleValue().AssignInt(3) }) t.Run("exploring should return nil when given a path segment out of range", func(t *testing.T) { returnedSelector, _ := s.Explore(n, datamodel.PathSegmentOfInt(2)) qt.Check(t, returnedSelector, qt.IsNil) }) t.Run("exploring should return nil when given a path segment that isn't an index", func(t *testing.T) { returnedSelector, _ := s.Explore(n, datamodel.PathSegmentOfString("cheese")) qt.Check(t, returnedSelector, qt.IsNil) }) t.Run("exploring should return the next selector when given a path segment with index in range", func(t *testing.T) { returnedSelector, _ := s.Explore(n, datamodel.PathSegmentOfInt(3)) qt.Check(t, returnedSelector, qt.Equals, Matcher{}) }) } ================================================ FILE: traversal/selector/exploreRecursive.go ================================================ package selector import ( "fmt" "github.com/ipld/go-ipld-prime/datamodel" ) // ExploreRecursive traverses some structure recursively. // To guide this exploration, it uses a "sequence", which is another Selector // tree; some leaf node in this sequence should contain an ExploreRecursiveEdge // selector, which denotes the place recursion should occur. // // In implementation, whenever evaluation reaches an ExploreRecursiveEdge marker // in the recursion sequence's Selector tree, the implementation logically // produces another new Selector which is a copy of the original // ExploreRecursive selector, but with a decremented maxDepth parameter, and // continues evaluation thusly. // // It is not valid for an ExploreRecursive selector's sequence to contain // no instances of ExploreRecursiveEdge; it *is* valid for it to contain // more than one ExploreRecursiveEdge. // // ExploreRecursive can contain a nested ExploreRecursive! // This is comparable to a nested for-loop. // In these cases, any ExploreRecursiveEdge instance always refers to the // nearest parent ExploreRecursive (in other words, ExploreRecursiveEdge can // be thought of like the 'continue' statement, or end of a for-loop body; // it is *not* a 'goto' statement). // // Be careful when using ExploreRecursive with a large maxDepth parameter; // it can easily cause very large traversals (especially if used in combination // with selectors like ExploreAll inside the sequence). type ExploreRecursive struct { sequence Selector // selector for element we're interested in current Selector // selector to apply to the current node limit RecursionLimit // the limit for this recursive selector stopAt *Condition // a condition for not exploring the node or children } // RecursionLimit_Mode is an enum that represents the type of a recursion limit // -- either "depth" or "none" for now type RecursionLimit_Mode uint8 const ( // RecursionLimit_None means there is no recursion limit RecursionLimit_None RecursionLimit_Mode = 0 // RecursionLimit_Depth mean recursion stops after the recursive selector // is copied to a given depth RecursionLimit_Depth RecursionLimit_Mode = 1 ) // RecursionLimit is a union type that captures all data about the recursion // limit (both its type and data specific to the type) type RecursionLimit struct { mode RecursionLimit_Mode depth int64 } // Mode returns the type for this recursion limit func (rl RecursionLimit) Mode() RecursionLimit_Mode { return rl.mode } // Depth returns the depth for a depth recursion limit, or 0 otherwise func (rl RecursionLimit) Depth() int64 { if rl.mode != RecursionLimit_Depth { return 0 } return rl.depth } // RecursionLimitDepth returns a depth limited recursion to the given depth func RecursionLimitDepth(depth int64) RecursionLimit { return RecursionLimit{RecursionLimit_Depth, depth} } // RecursionLimitNone return recursion with no limit func RecursionLimitNone() RecursionLimit { return RecursionLimit{RecursionLimit_None, 0} } // Interests for ExploreRecursive is empty (meaning traverse everything) func (s ExploreRecursive) Interests() []datamodel.PathSegment { return s.current.Interests() } // Explore returns the node's selector for all fields func (s ExploreRecursive) Explore(n datamodel.Node, p datamodel.PathSegment) (Selector, error) { // Check any stopAt conditions right away. if s.stopAt != nil { target, err := n.LookupBySegment(p) if err != nil { return nil, err } if s.stopAt.Match(target) { return nil, nil } } // Fence against edge case: if the next selector is a recursion edge, nope, we're out. // (This is only reachable if a recursion contains nothing but an edge -- which is probably somewhat rare, // because it's certainly rather useless -- but it's not explicitly rejected as a malformed selector during compile, either, so it must be handled.) if _, ok := s.current.(ExploreRecursiveEdge); ok { return nil, nil } // Apply the current selector clause. (This could be midway through something resembling the initially specified sequence.) nextSelector, _ := s.current.Explore(n, p) // We have to wrap the nextSelector yielded by the current clause in recursion information before returning it, // so that future levels of recursion (as well as their limits) can continue to operate correctly. if nextSelector == nil { return nil, nil } limit := s.limit if !s.hasRecursiveEdge(nextSelector) { return ExploreRecursive{s.sequence, nextSelector, limit, s.stopAt}, nil } switch limit.mode { case RecursionLimit_Depth: if limit.depth < 2 { return s.replaceRecursiveEdge(nextSelector, nil), nil } return ExploreRecursive{s.sequence, s.replaceRecursiveEdge(nextSelector, s.sequence), RecursionLimit{RecursionLimit_Depth, limit.depth - 1}, s.stopAt}, nil case RecursionLimit_None: return ExploreRecursive{s.sequence, s.replaceRecursiveEdge(nextSelector, s.sequence), limit, s.stopAt}, nil default: panic("Unsupported recursion limit type") } } func (s ExploreRecursive) hasRecursiveEdge(nextSelector Selector) bool { _, isRecursiveEdge := nextSelector.(ExploreRecursiveEdge) if isRecursiveEdge { return true } exploreUnion, isUnion := nextSelector.(ExploreUnion) if isUnion { for _, selector := range exploreUnion.Members { if s.hasRecursiveEdge(selector) { return true } } } return false } func (s ExploreRecursive) replaceRecursiveEdge(nextSelector Selector, replacement Selector) Selector { _, isRecursiveEdge := nextSelector.(ExploreRecursiveEdge) if isRecursiveEdge { return replacement } exploreUnion, isUnion := nextSelector.(ExploreUnion) if isUnion { replacementMembers := make([]Selector, 0, len(exploreUnion.Members)) for _, selector := range exploreUnion.Members { newSelector := s.replaceRecursiveEdge(selector, replacement) if newSelector != nil { replacementMembers = append(replacementMembers, newSelector) } } if len(replacementMembers) == 0 { return nil } if len(replacementMembers) == 1 { return replacementMembers[0] } return ExploreUnion{replacementMembers} } return nextSelector } // Decide if a node directly matches func (s ExploreRecursive) Decide(n datamodel.Node) bool { return s.current.Decide(n) } // Match always returns false because this is not a matcher func (s ExploreRecursive) Match(node datamodel.Node) (datamodel.Node, error) { return s.current.Match(node) } type exploreRecursiveContext struct { edgesFound int } func (erc *exploreRecursiveContext) Link(s Selector) bool { _, ok := s.(ExploreRecursiveEdge) if ok { erc.edgesFound++ } return ok } // ParseExploreRecursive assembles a Selector from a ExploreRecursive selector node func (pc ParseContext) ParseExploreRecursive(n datamodel.Node) (Selector, error) { if n.Kind() != datamodel.Kind_Map { return nil, fmt.Errorf("selector spec parse rejected: selector body must be a map") } limitNode, err := n.LookupByString(SelectorKey_Limit) if err != nil { return nil, fmt.Errorf("selector spec parse rejected: limit field must be present in ExploreRecursive selector") } limit, err := parseLimit(limitNode) if err != nil { return nil, err } sequence, err := n.LookupByString(SelectorKey_Sequence) if err != nil { return nil, fmt.Errorf("selector spec parse rejected: sequence field must be present in ExploreRecursive selector") } erc := &exploreRecursiveContext{} selector, err := pc.PushParent(erc).ParseSelector(sequence) if err != nil { return nil, err } if erc.edgesFound == 0 { return nil, fmt.Errorf("selector spec parse rejected: ExploreRecursive must have at least one ExploreRecursiveEdge") } var stopCondition *Condition stop, err := n.LookupByString(SelectorKey_StopAt) if err == nil { condition, err := pc.ParseCondition(stop) if err != nil { return nil, err } stopCondition = &condition } return ExploreRecursive{selector, selector, limit, stopCondition}, nil } func parseLimit(n datamodel.Node) (RecursionLimit, error) { if n.Kind() != datamodel.Kind_Map { return RecursionLimit{}, fmt.Errorf("selector spec parse rejected: limit in ExploreRecursive is a keyed union and thus must be a map") } if n.Length() != 1 { return RecursionLimit{}, fmt.Errorf("selector spec parse rejected: limit in ExploreRecursive is a keyed union and thus must be a single-entry map") } kn, v, _ := n.MapIterator().Next() kstr, _ := kn.AsString() switch kstr { case SelectorKey_LimitDepth: maxDepthValue, err := v.AsInt() if err != nil { return RecursionLimit{}, fmt.Errorf("selector spec parse rejected: limit field of type depth must be a number in ExploreRecursive selector") } return RecursionLimit{RecursionLimit_Depth, maxDepthValue}, nil case SelectorKey_LimitNone: return RecursionLimit{RecursionLimit_None, 0}, nil default: return RecursionLimit{}, fmt.Errorf("selector spec parse rejected: %q is not a known member of the limit union in ExploreRecursive", kstr) } } ================================================ FILE: traversal/selector/exploreRecursiveEdge.go ================================================ package selector import ( "fmt" "github.com/ipld/go-ipld-prime/datamodel" ) // ExploreRecursiveEdge is a special sentinel value which is used to mark // the end of a sequence started by an ExploreRecursive selector: the recursion // goes back to the initial state of the earlier ExploreRecursive selector, // and proceeds again (with a decremented maxDepth value). // // An ExploreRecursive selector that doesn't contain an ExploreRecursiveEdge // is nonsensical. Containing more than one ExploreRecursiveEdge is valid. // An ExploreRecursiveEdge without an enclosing ExploreRecursive is an error. type ExploreRecursiveEdge struct{} // Interests should almost never get called for an ExploreRecursiveEdge selector func (s ExploreRecursiveEdge) Interests() []datamodel.PathSegment { return []datamodel.PathSegment{} } // Explore should ultimately never get called for an ExploreRecursiveEdge selector func (s ExploreRecursiveEdge) Explore(n datamodel.Node, p datamodel.PathSegment) (Selector, error) { panic("Traversed Explore Recursive Edge Node With No Parent") } // Decide should almost never get called for an ExploreRecursiveEdge selector func (s ExploreRecursiveEdge) Decide(n datamodel.Node) bool { return false } // Match always returns false because this is not a matcher func (s ExploreRecursiveEdge) Match(node datamodel.Node) (datamodel.Node, error) { return nil, nil } // ParseExploreRecursiveEdge assembles a Selector // from a exploreRecursiveEdge selector node func (pc ParseContext) ParseExploreRecursiveEdge(n datamodel.Node) (Selector, error) { if n.Kind() != datamodel.Kind_Map { return nil, fmt.Errorf("selector spec parse rejected: selector body must be a map") } s := ExploreRecursiveEdge{} for _, parent := range pc.parentStack { if parent.Link(s) { return s, nil } } return nil, fmt.Errorf("selector spec parse rejected: ExploreRecursiveEdge must be beneath ExploreRecursive") } ================================================ FILE: traversal/selector/exploreRecursive_test.go ================================================ package selector import ( "strings" "testing" qt "github.com/frankban/quicktest" "github.com/ipld/go-ipld-prime/codec/dagjson" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/fluent" "github.com/ipld/go-ipld-prime/node/basicnode" ) func TestParseExploreRecursive(t *testing.T) { t.Run("parsing non map node should error", func(t *testing.T) { sn := basicnode.NewInt(0) _, err := ParseContext{}.ParseExploreRecursive(sn) qt.Check(t, err, qt.ErrorMatches, "selector spec parse rejected: selector body must be a map") }) t.Run("parsing map node without sequence field should error", func(t *testing.T) { sn := fluent.MustBuildMap(basicnode.Prototype__Map{}, 1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Limit).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_LimitDepth).AssignInt(2) }) }) _, err := ParseContext{}.ParseExploreRecursive(sn) qt.Check(t, err, qt.ErrorMatches, "selector spec parse rejected: sequence field must be present in ExploreRecursive selector") }) t.Run("parsing map node without limit field should error", func(t *testing.T) { sn := fluent.MustBuildMap(basicnode.Prototype__Map{}, 1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Sequence).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Matcher).CreateMap(0, func(na fluent.MapAssembler) {}) }) }) _, err := ParseContext{}.ParseExploreRecursive(sn) qt.Check(t, err, qt.ErrorMatches, "selector spec parse rejected: limit field must be present in ExploreRecursive selector") }) t.Run("parsing map node with limit field that is not a map should fail", func(t *testing.T) { sn := fluent.MustBuildMap(basicnode.Prototype__Map{}, 2, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Limit).AssignString("cheese") na.AssembleEntry(SelectorKey_Sequence).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Matcher).CreateMap(0, func(na fluent.MapAssembler) {}) }) }) _, err := ParseContext{}.ParseExploreRecursive(sn) qt.Check(t, err, qt.ErrorMatches, "selector spec parse rejected: limit in ExploreRecursive is a keyed union and thus must be a map") }) t.Run("parsing map node with limit field that is not a single entry map should fail", func(t *testing.T) { sn := fluent.MustBuildMap(basicnode.Prototype__Map{}, 2, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Limit).CreateMap(2, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_LimitDepth).AssignInt(2) na.AssembleEntry(SelectorKey_LimitNone).CreateMap(0, func(na fluent.MapAssembler) {}) }) na.AssembleEntry(SelectorKey_Sequence).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Matcher).CreateMap(0, func(na fluent.MapAssembler) {}) }) }) _, err := ParseContext{}.ParseExploreRecursive(sn) qt.Check(t, err, qt.ErrorMatches, "selector spec parse rejected: limit in ExploreRecursive is a keyed union and thus must be a single-entry map") }) t.Run("parsing map node with limit field that does not have a known key should fail", func(t *testing.T) { sn := fluent.MustBuildMap(basicnode.Prototype__Map{}, 2, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Limit).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry("applesauce").AssignInt(2) }) na.AssembleEntry(SelectorKey_Sequence).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Matcher).CreateMap(0, func(na fluent.MapAssembler) {}) }) }) _, err := ParseContext{}.ParseExploreRecursive(sn) qt.Check(t, err, qt.ErrorMatches, "selector spec parse rejected: \"applesauce\" is not a known member of the limit union in ExploreRecursive") }) t.Run("parsing map node with limit field of type depth that is not an int should error", func(t *testing.T) { sn := fluent.MustBuildMap(basicnode.Prototype__Map{}, 2, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Limit).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_LimitDepth).AssignString("cheese") }) na.AssembleEntry(SelectorKey_Sequence).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Matcher).CreateMap(0, func(na fluent.MapAssembler) {}) }) }) _, err := ParseContext{}.ParseExploreRecursive(sn) qt.Check(t, err, qt.ErrorMatches, "selector spec parse rejected: limit field of type depth must be a number in ExploreRecursive selector") }) t.Run("parsing map node with sequence field with invalid selector node should return child's error", func(t *testing.T) { sn := fluent.MustBuildMap(basicnode.Prototype__Map{}, 2, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Limit).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_LimitDepth).AssignInt(2) }) na.AssembleEntry(SelectorKey_Sequence).AssignInt(0) }) _, err := ParseContext{}.ParseExploreRecursive(sn) qt.Check(t, err, qt.ErrorMatches, "selector spec parse rejected: selector is a keyed union and thus must be a map") }) t.Run("parsing map node with sequence field with valid selector w/o ExploreRecursiveEdge should not parse", func(t *testing.T) { sn := fluent.MustBuildMap(basicnode.Prototype__Map{}, 2, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Limit).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_LimitDepth).AssignInt(2) }) na.AssembleEntry(SelectorKey_Sequence).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_ExploreAll).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Next).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Matcher).CreateMap(0, func(na fluent.MapAssembler) {}) }) }) }) }) _, err := ParseContext{}.ParseExploreRecursive(sn) qt.Check(t, err, qt.ErrorMatches, "selector spec parse rejected: ExploreRecursive must have at least one ExploreRecursiveEdge") }) t.Run("parsing map node that is ExploreRecursiveEdge without ExploreRecursive parent should not parse", func(t *testing.T) { sn := fluent.MustBuildMap(basicnode.Prototype__Map{}, 0, func(na fluent.MapAssembler) {}) _, err := ParseContext{}.ParseExploreRecursiveEdge(sn) qt.Check(t, err, qt.ErrorMatches, "selector spec parse rejected: ExploreRecursiveEdge must be beneath ExploreRecursive") }) t.Run("parsing map node with sequence field with valid selector node should parse", func(t *testing.T) { sn := fluent.MustBuildMap(basicnode.Prototype__Map{}, 2, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Limit).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_LimitDepth).AssignInt(2) }) na.AssembleEntry(SelectorKey_Sequence).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_ExploreAll).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Next).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_ExploreRecursiveEdge).CreateMap(0, func(na fluent.MapAssembler) {}) }) }) }) }) s, err := ParseContext{}.ParseExploreRecursive(sn) qt.Check(t, err, qt.IsNil) qt.Check(t, s, qt.Equals, ExploreRecursive{ExploreAll{ExploreRecursiveEdge{}}, ExploreAll{ExploreRecursiveEdge{}}, RecursionLimit{RecursionLimit_Depth, 2}, nil}) }) t.Run("parsing map node with sequence field with valid selector node and limit type none should parse", func(t *testing.T) { sn := fluent.MustBuildMap(basicnode.Prototype__Map{}, 2, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Limit).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_LimitNone).CreateMap(0, func(na fluent.MapAssembler) {}) }) na.AssembleEntry(SelectorKey_Sequence).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_ExploreAll).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Next).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_ExploreRecursiveEdge).CreateMap(0, func(na fluent.MapAssembler) {}) }) }) }) }) s, err := ParseContext{}.ParseExploreRecursive(sn) qt.Check(t, err, qt.IsNil) qt.Check(t, s, qt.Equals, ExploreRecursive{ExploreAll{ExploreRecursiveEdge{}}, ExploreAll{ExploreRecursiveEdge{}}, RecursionLimit{RecursionLimit_None, 0}, nil}) }) } /* { exploreRecursive: { maxDepth: 3 sequence: { exploreFields: { fields: { Parents: { exploreAll: { exploreRecursiveEdge: {} } } } } } } } */ func TestExploreRecursiveExplore(t *testing.T) { recursiveEdge := ExploreRecursiveEdge{} maxDepth := int64(3) var rs Selector t.Run("exploring should traverse until we get to maxDepth", func(t *testing.T) { parentsSelector := ExploreAll{recursiveEdge} subTree := ExploreFields{map[string]Selector{"Parents": parentsSelector}, []datamodel.PathSegment{datamodel.PathSegmentOfString("Parents")}} rs = ExploreRecursive{subTree, subTree, RecursionLimit{RecursionLimit_Depth, maxDepth}, nil} nodeString := `{ "Parents": [ { "Parents": [ { "Parents": [ { "Parents": [] } ] } ] } ] } ` nb := basicnode.Prototype__Any{}.NewBuilder() err := dagjson.Decode(nb, strings.NewReader(nodeString)) qt.Check(t, err, qt.IsNil) rn := nb.Build() rs, _ = rs.Explore(rn, datamodel.PathSegmentOfString("Parents")) rn, err = rn.LookupByString("Parents") qt.Check(t, rs, deepEqualsAllowAllUnexported, ExploreRecursive{subTree, parentsSelector, RecursionLimit{RecursionLimit_Depth, maxDepth}, nil}) qt.Check(t, err, qt.IsNil) rs, _ = rs.Explore(rn, datamodel.PathSegmentOfInt(0)) rn, err = rn.LookupByIndex(0) qt.Check(t, rs, deepEqualsAllowAllUnexported, ExploreRecursive{subTree, subTree, RecursionLimit{RecursionLimit_Depth, maxDepth - 1}, nil}) qt.Check(t, err, qt.IsNil) rs, _ = rs.Explore(rn, datamodel.PathSegmentOfString("Parents")) rn, err = rn.LookupByString("Parents") qt.Check(t, rs, deepEqualsAllowAllUnexported, ExploreRecursive{subTree, parentsSelector, RecursionLimit{RecursionLimit_Depth, maxDepth - 1}, nil}) qt.Check(t, err, qt.IsNil) rs, _ = rs.Explore(rn, datamodel.PathSegmentOfInt(0)) rn, err = rn.LookupByIndex(0) qt.Check(t, rs, deepEqualsAllowAllUnexported, ExploreRecursive{subTree, subTree, RecursionLimit{RecursionLimit_Depth, maxDepth - 2}, nil}) qt.Check(t, err, qt.IsNil) rs, _ = rs.Explore(rn, datamodel.PathSegmentOfString("Parents")) rn, err = rn.LookupByString("Parents") qt.Check(t, rs, deepEqualsAllowAllUnexported, ExploreRecursive{subTree, parentsSelector, RecursionLimit{RecursionLimit_Depth, maxDepth - 2}, nil}) qt.Check(t, err, qt.IsNil) rs, _ = rs.Explore(rn, datamodel.PathSegmentOfInt(0)) _, err = rn.LookupByIndex(0) qt.Check(t, rs, qt.IsNil) qt.Check(t, err, qt.IsNil) }) t.Run("exploring should traverse indefinitely if no depth specified", func(t *testing.T) { parentsSelector := ExploreAll{recursiveEdge} subTree := ExploreFields{map[string]Selector{"Parents": parentsSelector}, []datamodel.PathSegment{datamodel.PathSegmentOfString("Parents")}} rs = ExploreRecursive{subTree, subTree, RecursionLimit{RecursionLimit_None, 0}, nil} nodeString := `{ "Parents": [ { "Parents": [ { "Parents": [ { "Parents": [] } ] } ] } ] } ` nb := basicnode.Prototype__Any{}.NewBuilder() err := dagjson.Decode(nb, strings.NewReader(nodeString)) qt.Check(t, err, qt.IsNil) rn := nb.Build() rs, _ = rs.Explore(rn, datamodel.PathSegmentOfString("Parents")) rn, err = rn.LookupByString("Parents") qt.Check(t, rs, deepEqualsAllowAllUnexported, ExploreRecursive{subTree, parentsSelector, RecursionLimit{RecursionLimit_None, 0}, nil}) qt.Check(t, err, qt.IsNil) rs, _ = rs.Explore(rn, datamodel.PathSegmentOfInt(0)) rn, err = rn.LookupByIndex(0) qt.Check(t, rs, deepEqualsAllowAllUnexported, ExploreRecursive{subTree, subTree, RecursionLimit{RecursionLimit_None, 0}, nil}) qt.Check(t, err, qt.IsNil) rs, _ = rs.Explore(rn, datamodel.PathSegmentOfString("Parents")) rn, err = rn.LookupByString("Parents") qt.Check(t, rs, deepEqualsAllowAllUnexported, ExploreRecursive{subTree, parentsSelector, RecursionLimit{RecursionLimit_None, 0}, nil}) qt.Check(t, err, qt.IsNil) rs, _ = rs.Explore(rn, datamodel.PathSegmentOfInt(0)) rn, err = rn.LookupByIndex(0) qt.Check(t, rs, deepEqualsAllowAllUnexported, ExploreRecursive{subTree, subTree, RecursionLimit{RecursionLimit_None, 0}, nil}) qt.Check(t, err, qt.IsNil) rs, _ = rs.Explore(rn, datamodel.PathSegmentOfString("Parents")) rn, err = rn.LookupByString("Parents") qt.Check(t, rs, deepEqualsAllowAllUnexported, ExploreRecursive{subTree, parentsSelector, RecursionLimit{RecursionLimit_None, 0}, nil}) qt.Check(t, err, qt.IsNil) rs, _ = rs.Explore(rn, datamodel.PathSegmentOfInt(0)) rn, err = rn.LookupByIndex(0) qt.Check(t, rs, deepEqualsAllowAllUnexported, ExploreRecursive{subTree, subTree, RecursionLimit{RecursionLimit_None, 0}, nil}) qt.Check(t, err, qt.IsNil) rs, _ = rs.Explore(rn, datamodel.PathSegmentOfString("Parents")) _, err = rn.LookupByString("Parents") qt.Check(t, rs, deepEqualsAllowAllUnexported, ExploreRecursive{subTree, parentsSelector, RecursionLimit{RecursionLimit_None, 0}, nil}) qt.Check(t, err, qt.IsNil) }) t.Run("exploring should continue till we get to selector that returns nil on explore", func(t *testing.T) { parentsSelector := ExploreIndex{recursiveEdge, [1]datamodel.PathSegment{datamodel.PathSegmentOfInt(1)}} subTree := ExploreFields{map[string]Selector{"Parents": parentsSelector}, []datamodel.PathSegment{datamodel.PathSegmentOfString("Parents")}} rs = ExploreRecursive{subTree, subTree, RecursionLimit{RecursionLimit_Depth, maxDepth}, nil} nodeString := `{ "Parents": { } } ` nb := basicnode.Prototype__Any{}.NewBuilder() err := dagjson.Decode(nb, strings.NewReader(nodeString)) qt.Check(t, err, qt.IsNil) rn := nb.Build() rs, _ = rs.Explore(rn, datamodel.PathSegmentOfString("Parents")) rn, err = rn.LookupByString("Parents") qt.Check(t, rs, deepEqualsAllowAllUnexported, ExploreRecursive{subTree, parentsSelector, RecursionLimit{RecursionLimit_Depth, maxDepth}, nil}) qt.Check(t, err, qt.IsNil) rs, _ = rs.Explore(rn, datamodel.PathSegmentOfInt(0)) qt.Check(t, rs, qt.IsNil) }) t.Run("exploring should work when there is nested recursion", func(t *testing.T) { parentsSelector := ExploreAll{recursiveEdge} sideSelector := ExploreAll{recursiveEdge} subTree := ExploreFields{map[string]Selector{ "Parents": parentsSelector, "Side": ExploreRecursive{sideSelector, sideSelector, RecursionLimit{RecursionLimit_Depth, maxDepth}, nil}, }, []datamodel.PathSegment{ datamodel.PathSegmentOfString("Parents"), datamodel.PathSegmentOfString("Side"), }, } s := ExploreRecursive{subTree, subTree, RecursionLimit{RecursionLimit_Depth, maxDepth}, nil} nodeString := `{ "Parents": [ { "Parents": [], "Side": { "cheese": { "whiz": { } } } } ], "Side": { "real": { "apple": { "sauce": { } } } } } ` nb := basicnode.Prototype__Any{}.NewBuilder() err := dagjson.Decode(nb, strings.NewReader(nodeString)) qt.Check(t, err, qt.IsNil) n := nb.Build() // traverse down Parent nodes rn := n rs = s rs, _ = rs.Explore(rn, datamodel.PathSegmentOfString("Parents")) rn, err = rn.LookupByString("Parents") qt.Check(t, rs, deepEqualsAllowAllUnexported, ExploreRecursive{subTree, parentsSelector, RecursionLimit{RecursionLimit_Depth, maxDepth}, nil}) qt.Check(t, err, qt.IsNil) rs, _ = rs.Explore(rn, datamodel.PathSegmentOfInt(0)) rn, err = rn.LookupByIndex(0) qt.Check(t, rs, deepEqualsAllowAllUnexported, ExploreRecursive{subTree, subTree, RecursionLimit{RecursionLimit_Depth, maxDepth - 1}, nil}) qt.Check(t, err, qt.IsNil) rs, _ = rs.Explore(rn, datamodel.PathSegmentOfString("Parents")) _, err = rn.LookupByString("Parents") qt.Check(t, rs, deepEqualsAllowAllUnexported, ExploreRecursive{subTree, parentsSelector, RecursionLimit{RecursionLimit_Depth, maxDepth - 1}, nil}) qt.Check(t, err, qt.IsNil) // traverse down top level Side tree (nested recursion) rn = n rs = s rs, _ = rs.Explore(rn, datamodel.PathSegmentOfString("Side")) rn, err = rn.LookupByString("Side") qt.Check(t, rs, deepEqualsAllowAllUnexported, ExploreRecursive{subTree, ExploreRecursive{sideSelector, sideSelector, RecursionLimit{RecursionLimit_Depth, maxDepth}, nil}, RecursionLimit{RecursionLimit_Depth, maxDepth}, nil}) qt.Check(t, err, qt.IsNil) rs, _ = rs.Explore(rn, datamodel.PathSegmentOfString("real")) rn, err = rn.LookupByString("real") qt.Check(t, rs, deepEqualsAllowAllUnexported, ExploreRecursive{subTree, ExploreRecursive{sideSelector, sideSelector, RecursionLimit{RecursionLimit_Depth, maxDepth - 1}, nil}, RecursionLimit{RecursionLimit_Depth, maxDepth}, nil}) qt.Check(t, err, qt.IsNil) rs, _ = rs.Explore(rn, datamodel.PathSegmentOfString("apple")) rn, err = rn.LookupByString("apple") qt.Check(t, rs, deepEqualsAllowAllUnexported, ExploreRecursive{subTree, ExploreRecursive{sideSelector, sideSelector, RecursionLimit{RecursionLimit_Depth, maxDepth - 2}, nil}, RecursionLimit{RecursionLimit_Depth, maxDepth}, nil}) qt.Check(t, err, qt.IsNil) rs, _ = rs.Explore(rn, datamodel.PathSegmentOfString("sauce")) _, err = rn.LookupByString("sauce") qt.Check(t, rs, qt.IsNil) qt.Check(t, err, qt.IsNil) // traverse once down Parent (top level recursion) then down Side tree (nested recursion) rn = n rs = s rs, _ = rs.Explore(rn, datamodel.PathSegmentOfString("Parents")) rn, err = rn.LookupByString("Parents") qt.Check(t, rs, deepEqualsAllowAllUnexported, ExploreRecursive{subTree, parentsSelector, RecursionLimit{RecursionLimit_Depth, maxDepth}, nil}) qt.Check(t, err, qt.IsNil) rs, _ = rs.Explore(rn, datamodel.PathSegmentOfInt(0)) rn, err = rn.LookupByIndex(0) qt.Check(t, rs, deepEqualsAllowAllUnexported, ExploreRecursive{subTree, subTree, RecursionLimit{RecursionLimit_Depth, maxDepth - 1}, nil}) qt.Check(t, err, qt.IsNil) rs, _ = rs.Explore(rn, datamodel.PathSegmentOfString("Side")) rn, err = rn.LookupByString("Side") qt.Check(t, rs, deepEqualsAllowAllUnexported, ExploreRecursive{subTree, ExploreRecursive{sideSelector, sideSelector, RecursionLimit{RecursionLimit_Depth, maxDepth}, nil}, RecursionLimit{RecursionLimit_Depth, maxDepth - 1}, nil}) qt.Check(t, err, qt.IsNil) rs, _ = rs.Explore(rn, datamodel.PathSegmentOfString("cheese")) rn, err = rn.LookupByString("cheese") qt.Check(t, rs, deepEqualsAllowAllUnexported, ExploreRecursive{subTree, ExploreRecursive{sideSelector, sideSelector, RecursionLimit{RecursionLimit_Depth, maxDepth - 1}, nil}, RecursionLimit{RecursionLimit_Depth, maxDepth - 1}, nil}) qt.Check(t, err, qt.IsNil) rs, _ = rs.Explore(rn, datamodel.PathSegmentOfString("whiz")) _, err = rn.LookupByString("whiz") qt.Check(t, rs, deepEqualsAllowAllUnexported, ExploreRecursive{subTree, ExploreRecursive{sideSelector, sideSelector, RecursionLimit{RecursionLimit_Depth, maxDepth - 2}, nil}, RecursionLimit{RecursionLimit_Depth, maxDepth - 1}, nil}) qt.Check(t, err, qt.IsNil) }) t.Run("exploring should work with explore union and recursion", func(t *testing.T) { parentsSelector := ExploreUnion{[]Selector{ExploreAll{Matcher{}}, ExploreIndex{recursiveEdge, [1]datamodel.PathSegment{datamodel.PathSegmentOfInt(0)}}}} subTree := ExploreFields{map[string]Selector{"Parents": parentsSelector}, []datamodel.PathSegment{datamodel.PathSegmentOfString("Parents")}} rs = ExploreRecursive{subTree, subTree, RecursionLimit{RecursionLimit_Depth, maxDepth}, nil} nodeString := `{ "Parents": [ { "Parents": [ { "Parents": [ { "Parents": [] } ] } ] } ] } ` nb := basicnode.Prototype__Any{}.NewBuilder() err := dagjson.Decode(nb, strings.NewReader(nodeString)) qt.Check(t, err, qt.IsNil) rn := nb.Build() rs, _ = rs.Explore(rn, datamodel.PathSegmentOfString("Parents")) rn, err = rn.LookupByString("Parents") qt.Check(t, rs, deepEqualsAllowAllUnexported, ExploreRecursive{subTree, parentsSelector, RecursionLimit{RecursionLimit_Depth, maxDepth}, nil}) qt.Check(t, err, qt.IsNil) rs, _ = rs.Explore(rn, datamodel.PathSegmentOfInt(0)) rn, err = rn.LookupByIndex(0) qt.Check(t, rs, deepEqualsAllowAllUnexported, ExploreRecursive{subTree, ExploreUnion{[]Selector{Matcher{}, subTree}}, RecursionLimit{RecursionLimit_Depth, maxDepth - 1}, nil}) qt.Check(t, err, qt.IsNil) rs, _ = rs.Explore(rn, datamodel.PathSegmentOfString("Parents")) rn, err = rn.LookupByString("Parents") qt.Check(t, rs, deepEqualsAllowAllUnexported, ExploreRecursive{subTree, parentsSelector, RecursionLimit{RecursionLimit_Depth, maxDepth - 1}, nil}) qt.Check(t, err, qt.IsNil) rs, _ = rs.Explore(rn, datamodel.PathSegmentOfInt(0)) _, err = rn.LookupByIndex(0) qt.Check(t, rs, deepEqualsAllowAllUnexported, ExploreRecursive{subTree, ExploreUnion{[]Selector{Matcher{}, subTree}}, RecursionLimit{RecursionLimit_Depth, maxDepth - 2}, nil}) qt.Check(t, err, qt.IsNil) }) } ================================================ FILE: traversal/selector/exploreUnion.go ================================================ package selector import ( "fmt" "github.com/ipld/go-ipld-prime/datamodel" ) // ExploreUnion allows selection to continue with two or more distinct selectors // while exploring the same tree of data. // // ExploreUnion can be used to apply a Matcher on one node (causing it to // be considered part of a (possibly labelled) result set), while simultaneously // continuing to explore deeper parts of the tree with another selector, // for example. type ExploreUnion struct { Members []Selector } // Interests for ExploreUnion is: // - nil (aka all) if any member selector has nil interests // - the union of values returned by all member selectors otherwise func (s ExploreUnion) Interests() []datamodel.PathSegment { // Check for any high-cardinality selectors first; if so, shortcircuit. // (n.b. we're assuming the 'Interests' method is cheap here.) for _, m := range s.Members { if m.Interests() == nil { return nil } } // Accumulate the whitelist of interesting path segments. // TODO: Dedup? v := []datamodel.PathSegment{} for _, m := range s.Members { v = append(v, m.Interests()...) } return v } // Explore for a Union selector calls explore for each member selector // and returns: // - a new union selector if more than one member returns a selector // - if exactly one member returns a selector, that selector // - nil if no members return a selector func (s ExploreUnion) Explore(n datamodel.Node, p datamodel.PathSegment) (Selector, error) { // TODO: memory efficient? nonNilResults := make([]Selector, 0, len(s.Members)) for _, member := range s.Members { resultSelector, err := member.Explore(n, p) if err != nil { return nil, err } if resultSelector != nil { nonNilResults = append(nonNilResults, resultSelector) } } if len(nonNilResults) == 0 { return nil, nil } if len(nonNilResults) == 1 { return nonNilResults[0], nil } return ExploreUnion{nonNilResults}, nil } // Decide returns true for a Union selector if any of the member selectors // return true func (s ExploreUnion) Decide(n datamodel.Node) bool { for _, m := range s.Members { if m.Decide(n) { return true } } return false } // Match returns true for a Union selector based on the matched union. func (s ExploreUnion) Match(n datamodel.Node) (datamodel.Node, error) { for _, m := range s.Members { if mn, err := m.Match(n); mn != nil { return mn, nil } else if err != nil { return nil, err } } return nil, nil } // ParseExploreUnion assembles a Selector // from an ExploreUnion selector node func (pc ParseContext) ParseExploreUnion(n datamodel.Node) (Selector, error) { if n.Kind() != datamodel.Kind_List { return nil, fmt.Errorf("selector spec parse rejected: explore union selector must be a list") } x := ExploreUnion{ make([]Selector, 0, n.Length()), } for itr := n.ListIterator(); !itr.Done(); { _, v, err := itr.Next() if err != nil { return nil, fmt.Errorf("error during selector spec parse: %w", err) } member, err := pc.ParseSelector(v) if err != nil { return nil, err } x.Members = append(x.Members, member) } return x, nil } ================================================ FILE: traversal/selector/exploreUnion_test.go ================================================ package selector import ( "testing" qt "github.com/frankban/quicktest" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/fluent" "github.com/ipld/go-ipld-prime/node/basicnode" ) func TestParseExploreUnion(t *testing.T) { t.Run("parsing non list node should error", func(t *testing.T) { sn := fluent.MustBuildMap(basicnode.Prototype__Map{}, 0, func(na fluent.MapAssembler) {}) _, err := ParseContext{}.ParseExploreUnion(sn) qt.Check(t, err, qt.ErrorMatches, "selector spec parse rejected: explore union selector must be a list") }) t.Run("parsing list node where one node is invalid should return child's error", func(t *testing.T) { sn := fluent.MustBuildList(basicnode.Prototype__List{}, 2, func(na fluent.ListAssembler) { na.AssembleValue().CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Matcher).CreateMap(0, func(na fluent.MapAssembler) {}) }) na.AssembleValue().AssignInt(2) }) _, err := ParseContext{}.ParseExploreUnion(sn) qt.Check(t, err, qt.ErrorMatches, "selector spec parse rejected: selector is a keyed union and thus must be a map") }) t.Run("parsing map node with next field with valid selector node should parse", func(t *testing.T) { sn := fluent.MustBuildList(basicnode.Prototype__List{}, 2, func(na fluent.ListAssembler) { na.AssembleValue().CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Matcher).CreateMap(0, func(na fluent.MapAssembler) {}) }) na.AssembleValue().CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_ExploreIndex).CreateMap(2, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Index).AssignInt(2) na.AssembleEntry(SelectorKey_Next).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(SelectorKey_Matcher).CreateMap(0, func(na fluent.MapAssembler) {}) }) }) }) }) s, err := ParseContext{}.ParseExploreUnion(sn) qt.Check(t, err, qt.IsNil) qt.Check(t, s, deepEqualsAllowAllUnexported, ExploreUnion{[]Selector{Matcher{}, ExploreIndex{Matcher{}, [1]datamodel.PathSegment{datamodel.PathSegmentOfInt(2)}}}}) }) } func TestExploreUnionExplore(t *testing.T) { n := fluent.MustBuildList(basicnode.Prototype__List{}, 4, func(na fluent.ListAssembler) { na.AssembleValue().AssignInt(0) na.AssembleValue().AssignInt(1) na.AssembleValue().AssignInt(2) na.AssembleValue().AssignInt(3) }) t.Run("exploring should return nil if all member selectors return nil when explored", func(t *testing.T) { s := ExploreUnion{[]Selector{Matcher{}, ExploreIndex{Matcher{}, [1]datamodel.PathSegment{datamodel.PathSegmentOfInt(2)}}}} returnedSelector, _ := s.Explore(n, datamodel.PathSegmentOfInt(3)) qt.Check(t, returnedSelector, qt.IsNil) }) t.Run("if exactly one member selector returns a non-nil selector when explored, exploring should return that value", func(t *testing.T) { s := ExploreUnion{[]Selector{Matcher{}, ExploreIndex{Matcher{}, [1]datamodel.PathSegment{datamodel.PathSegmentOfInt(2)}}}} returnedSelector, _ := s.Explore(n, datamodel.PathSegmentOfInt(2)) qt.Check(t, returnedSelector, qt.Equals, Matcher{}) }) t.Run("exploring should return a new union selector if more than one member selector returns a non nil selector when explored", func(t *testing.T) { s := ExploreUnion{[]Selector{ Matcher{}, ExploreIndex{Matcher{}, [1]datamodel.PathSegment{datamodel.PathSegmentOfInt(2)}}, ExploreRange{Matcher{}, 2, 3, []datamodel.PathSegment{datamodel.PathSegmentOfInt(2)}}, ExploreFields{map[string]Selector{"applesauce": Matcher{}}, []datamodel.PathSegment{datamodel.PathSegmentOfString("applesauce")}}, }} returnedSelector, _ := s.Explore(n, datamodel.PathSegmentOfInt(2)) qt.Check(t, returnedSelector, deepEqualsAllowAllUnexported, ExploreUnion{[]Selector{Matcher{}, Matcher{}}}) }) } func TestExploreUnionInterests(t *testing.T) { t.Run("if any member selector is high-cardinality, interests should be high-cardinality", func(t *testing.T) { s := ExploreUnion{[]Selector{ ExploreAll{Matcher{}}, Matcher{}, ExploreIndex{Matcher{}, [1]datamodel.PathSegment{datamodel.PathSegmentOfInt(2)}}, }} qt.Check(t, s.Interests(), deepEqualsAllowAllUnexported, []datamodel.PathSegment(nil)) }) t.Run("if no member selector is high-cardinality, interests should be combination of member selectors interests", func(t *testing.T) { s := ExploreUnion{[]Selector{ ExploreFields{map[string]Selector{"applesauce": Matcher{}}, []datamodel.PathSegment{datamodel.PathSegmentOfString("applesauce")}}, Matcher{}, ExploreIndex{Matcher{}, [1]datamodel.PathSegment{datamodel.PathSegmentOfInt(2)}}, }} qt.Check(t, s.Interests(), deepEqualsAllowAllUnexported, []datamodel.PathSegment{datamodel.PathSegmentOfString("applesauce"), datamodel.PathSegmentOfInt(2)}) }) } func TestExploreUnionDecide(t *testing.T) { n := basicnode.NewInt(2) t.Run("if any member selector returns true, decide should be true", func(t *testing.T) { s := ExploreUnion{[]Selector{ ExploreAll{Matcher{}}, Matcher{}, ExploreIndex{Matcher{}, [1]datamodel.PathSegment{datamodel.PathSegmentOfInt(2)}}, }} qt.Check(t, s.Decide(n), qt.IsTrue) }) t.Run("if no member selector returns true, decide should be false", func(t *testing.T) { s := ExploreUnion{[]Selector{ ExploreFields{map[string]Selector{"applesauce": Matcher{}}, []datamodel.PathSegment{datamodel.PathSegmentOfString("applesauce")}}, ExploreAll{Matcher{}}, ExploreIndex{Matcher{}, [1]datamodel.PathSegment{datamodel.PathSegmentOfInt(2)}}, }} qt.Check(t, s.Decide(n), qt.IsFalse) }) } ================================================ FILE: traversal/selector/fieldKeys.go ================================================ package selector const ( SelectorKey_Matcher = "." SelectorKey_ExploreAll = "a" SelectorKey_ExploreFields = "f" SelectorKey_ExploreIndex = "i" SelectorKey_ExploreRange = "r" SelectorKey_ExploreRecursive = "R" SelectorKey_ExploreUnion = "|" SelectorKey_ExploreConditional = "&" SelectorKey_ExploreRecursiveEdge = "@" SelectorKey_ExploreInterpretAs = "~" SelectorKey_Next = ">" SelectorKey_Fields = "f>" SelectorKey_Index = "i" SelectorKey_Start = "^" SelectorKey_End = "$" SelectorKey_Sequence = ":>" SelectorKey_Limit = "l" SelectorKey_LimitDepth = "depth" SelectorKey_LimitNone = "none" SelectorKey_StopAt = "!" SelectorKey_Condition = "&" SelectorKey_As = "as" SelectorKey_Subset = "subset" SelectorKey_From = "[" SelectorKey_To = "]" // not filling conditional keys since it's not complete ) ================================================ FILE: traversal/selector/matcher.go ================================================ package selector import ( "fmt" "io" "math" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/node/basicnode" ) // Matcher marks a node to be included in the "result" set. // (All nodes traversed by a selector are in the "covered" set (which is a.k.a. // "the merkle proof"); the "result" set is a subset of the "covered" set.) // // In libraries using selectors, the "result" set is typically provided to // some user-specified callback. // // A selector tree with only "explore*"-type selectors and no Matcher selectors // is valid; it will just generate a "covered" set of nodes and no "result" set. // TODO: From spec: implement conditions and labels type Matcher struct { *Slice } // Slice limits a result node to a subset of the node. // The returned node will be limited based on slicing the specified range of the // node into a new node, or making use of the `AsLargeBytes` io.ReadSeeker to // restrict response with a SectionReader. // // Slice supports [From,To) ranges, where From is inclusive and To is exclusive. // Negative values for From and To are interpreted as offsets from the end of // the node. If To is greater than the node length, it will be truncated to the // node length. If From is greater than the node length or greater than To, the // result will be a non-match. type Slice struct { From int64 To int64 } func sliceBounds(from, to, length int64) (bool, int64, int64) { if to < 0 { to = length + to } else if length < to { to = length } if from < 0 { from = length + from if from < 0 { from = 0 } } if from > to || from >= length { return false, 0, 0 } return true, from, to } func (s Slice) Slice(n datamodel.Node) (datamodel.Node, error) { var from, to int64 switch n.Kind() { case datamodel.Kind_String: str, err := n.AsString() if err != nil { return nil, err } var match bool match, from, to = sliceBounds(s.From, s.To, int64(len(str))) if !match { return nil, nil } return basicnode.NewString(str[from:to]), nil case datamodel.Kind_Bytes: to = s.To from = s.From var length int64 = math.MaxInt64 var rdr io.ReadSeeker var bytes []byte var err error if lbn, ok := n.(datamodel.LargeBytesNode); ok { rdr, err = lbn.AsLargeBytes() if err != nil { return nil, err } // calculate length from seeker length, err = rdr.Seek(0, io.SeekEnd) if err != nil { return nil, err } // reset _, err = rdr.Seek(0, io.SeekStart) if err != nil { return nil, err } } else { bytes, err = n.AsBytes() if err != nil { return nil, err } length = int64(len(bytes)) } var match bool match, from, to = sliceBounds(from, to, length) if !match { return nil, nil } if rdr != nil { sr := io.NewSectionReader(&readerat{rdr, 0}, from, to-from) return basicnode.NewBytesFromReader(sr), nil } return basicnode.NewBytes(bytes[from:to]), nil default: return nil, nil } } // Interests are empty for a matcher (for now) because // It is always just there to match, not explore further func (s Matcher) Interests() []datamodel.PathSegment { return []datamodel.PathSegment{} } // Explore will return nil because a matcher is a terminal selector func (s Matcher) Explore(n datamodel.Node, p datamodel.PathSegment) (Selector, error) { return nil, nil } // Decide is always true for a match cause it's in the result set // Deprecated: use Match instead func (s Matcher) Decide(n datamodel.Node) bool { return true } // Match is always true for a match cause it's in the result set func (s Matcher) Match(node datamodel.Node) (datamodel.Node, error) { if s.Slice != nil { return s.Slice.Slice(node) } return node, nil } // ParseMatcher assembles a Selector // from a matcher selector node // TODO: Parse labels and conditions func (pc ParseContext) ParseMatcher(n datamodel.Node) (Selector, error) { if n.Kind() != datamodel.Kind_Map { return nil, fmt.Errorf("selector spec parse rejected: selector body must be a map") } // check if a slice is specified if subset, err := n.LookupByString("subset"); err == nil { if subset.Kind() != datamodel.Kind_Map { return nil, fmt.Errorf("selector spec parse rejected: subset body must be a map") } from, err := subset.LookupByString("[") if err != nil { return nil, fmt.Errorf("selector spec parse rejected: selector body must be a map with a from '[' key") } fromN, err := from.AsInt() if err != nil { return nil, fmt.Errorf("selector spec parse rejected: selector body must be a map with a 'from' key that is a number") } to, err := subset.LookupByString("]") if err != nil { return nil, fmt.Errorf("selector spec parse rejected: selector body must be a map with a to ']' key") } toN, err := to.AsInt() if err != nil { return nil, fmt.Errorf("selector spec parse rejected: selector body must be a map with a 'to' key that is a number") } if toN >= 0 && fromN > toN { return nil, fmt.Errorf("selector spec parse rejected: selector body must be a map with a 'from' key that is less than or equal to the 'to' key") } return Matcher{&Slice{ From: fromN, To: toN, }}, nil } return Matcher{}, nil } ================================================ FILE: traversal/selector/matcher_test.go ================================================ package selector_test import ( "fmt" "math" "regexp" "testing" qt "github.com/frankban/quicktest" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/fluent/qp" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/testutil" "github.com/ipld/go-ipld-prime/traversal" "github.com/ipld/go-ipld-prime/traversal/selector" ) func TestSubsetMatch(t *testing.T) { expectedString := "foobarbaz!" nodes := []struct { name string node datamodel.Node }{ {"stringNode", basicnode.NewString(expectedString)}, {"bytesNode", testutil.NewSimpleBytes([]byte(expectedString))}, {"largeBytesNode", testutil.NewMultiByteNode( []byte("foo"), []byte("bar"), []byte("baz"), []byte("!"), )}, } // selector for a slice of the value of the "bipbop" field within a map mkRangeSelector := func(from int64, to int64) (datamodel.Node, error) { return qp.BuildMap(basicnode.Prototype.Map, 1, func(na datamodel.MapAssembler) { qp.MapEntry(na, selector.SelectorKey_ExploreFields, qp.Map(1, func(na datamodel.MapAssembler) { qp.MapEntry(na, selector.SelectorKey_Fields, qp.Map(1, func(na datamodel.MapAssembler) { qp.MapEntry(na, "bipbop", qp.Map(1, func(na datamodel.MapAssembler) { qp.MapEntry(na, selector.SelectorKey_Matcher, qp.Map(1, func(na datamodel.MapAssembler) { qp.MapEntry(na, selector.SelectorKey_Subset, qp.Map(1, func(na datamodel.MapAssembler) { qp.MapEntry(na, selector.SelectorKey_From, qp.Int(from)) qp.MapEntry(na, selector.SelectorKey_To, qp.Int(to)) })) })) })) })) })) }) } for _, tc := range []struct { from int64 to int64 exp string match bool }{ {0, math.MaxInt64, expectedString, true}, {0, int64(len(expectedString)), expectedString, true}, {0, 0, "", true}, {0, 1, "f", true}, {0, 2, "fo", true}, {0, 3, "foo", true}, {0, 4, "foob", true}, {1, 4, "oob", true}, {2, 4, "ob", true}, {3, 4, "b", true}, {4, 4, "", true}, {4, math.MaxInt64, "arbaz!", true}, {4, int64(len(expectedString)), "arbaz!", true}, {4, int64(len(expectedString) - 1), "arbaz", true}, {0, int64(len(expectedString) - 1), expectedString[0 : len(expectedString)-1], true}, {0, int64(len(expectedString) - 2), expectedString[0 : len(expectedString)-2], true}, {0, -1, expectedString[0 : len(expectedString)-1], true}, {0, -2, expectedString[0 : len(expectedString)-2], true}, {-2, -1, "z", true}, {-1, math.MaxInt64, "!", true}, {-int64(len(expectedString)), math.MaxInt64, expectedString, true}, {math.MaxInt64 - 1, math.MaxInt64, "", false}, {int64(len(expectedString)), math.MaxInt64, "", false}, {-1, -2, "", false}, // To < From, no match {-1, -1, "", true}, // To==From, match zero bytes {-1000, -100, "", false}, // From undeflow, adjusted to 0, To underflow, not adjusted, To < From, no match {-100, -1000, "", false}, // From undeflow, adjusted to 0, To underflow, adjusted to 0, To < From, no match {-1000, 1000, expectedString, true}, // From undeflow, adjusted to 0, To overflow, adjusted to len, match all } { for _, variant := range nodes { t.Run(fmt.Sprintf("%s[%d:%d]", variant.name, tc.from, tc.to), func(t *testing.T) { selNode, err := mkRangeSelector(tc.from, tc.to) qt.Assert(t, err, qt.IsNil) ss, err := selector.ParseSelector(selNode) qt.Assert(t, err, qt.IsNil) // node that the selector will match, with our variant node embedded in the "bipbop" field n, err := qp.BuildMap(basicnode.Prototype.Map, 1, func(na datamodel.MapAssembler) { qp.MapEntry(na, "bipbop", qp.Node(variant.node)) }) var got datamodel.Node qt.Assert(t, err, qt.IsNil) err = traversal.WalkMatching(n, ss, func(prog traversal.Progress, n datamodel.Node) error { qt.Assert(t, got, qt.IsNil) got = n return nil }) qt.Assert(t, err, qt.IsNil) if tc.match { qt.Assert(t, got, qt.IsNotNil) qt.Assert(t, got.Kind(), qt.Equals, variant.node.Kind()) var gotString string switch got.Kind() { case datamodel.Kind_String: gotString, err = got.AsString() qt.Assert(t, err, qt.IsNil) case datamodel.Kind_Bytes: byts, err := got.AsBytes() qt.Assert(t, err, qt.IsNil) gotString = string(byts) } qt.Assert(t, gotString, qt.DeepEquals, tc.exp) } else { qt.Assert(t, got, qt.IsNil) } }) } } // when both are positive, we can validate ranges up-front t.Run("invalid range", func(t *testing.T) { selNode, err := mkRangeSelector(1000, 100) qt.Assert(t, err, qt.IsNil) re, err := regexp.Compile("from.*less than or equal to.*to") qt.Assert(t, err, qt.IsNil) ss, err := selector.ParseSelector(selNode) qt.Assert(t, ss, qt.IsNil) qt.Assert(t, err, qt.ErrorMatches, re) }) } ================================================ FILE: traversal/selector/matcher_util.go ================================================ package selector import ( "io" ) type readerat struct { rs io.ReadSeeker off int64 } // ReadAt provides the io.ReadAt method over a ReadSeeker. It will track the // current offset and seek if necessary. func (r *readerat) ReadAt(p []byte, off int64) (n int, err error) { if off != r.off { if _, err = r.rs.Seek(off, io.SeekStart); err != nil { return 0, err } r.off = off } c, err := r.rs.Read(p) if err != nil { return c, err } r.off += int64(c) return c, nil } ================================================ FILE: traversal/selector/parse/selector_parse.go ================================================ /* selectorparse package contains some helpful functions for parsing the serial form of Selectors. Some common selectors are also exported as pre-compiled variables, both for convenience of use and to be readable as examples. */ package selectorparse import ( "strings" "github.com/ipld/go-ipld-prime/codec/dagjson" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/traversal/selector" ) // ParseJSONSelector accepts a string of json which will be parsed as a selector, // and returns a datamodel.Node of the parsed Data Model. // The returned datamodel.Node is suitable to hand to `selector.CompileSelector`, // or, could be composed programmatically with other Data Model selector clauses // and then compiled later. // // The selector will be checked for compileability, and an error returned if it is not. func ParseJSONSelector(jsonStr string) (datamodel.Node, error) { nb := basicnode.Prototype.Any.NewBuilder() if err := dagjson.Decode(nb, strings.NewReader(jsonStr)); err != nil { return nil, err } // Compile it, because that's where all of our error checking is right now. // ... but throw that result away, because the point of this method is to return nodes that you can compose further. // Ideally, we'd have just used Schemas for this check, // which would be cheaper than running the full compile, // and also more correct (because it would let us parse incomplete phrases that won't compile alone), // but that's not currently how the Selectors code is implemented. Future work! n := nb.Build() if _, err := selector.CompileSelector(n); err != nil { return nil, err } return n, nil } // ParseJSONSelector accepts a string of json which will be parsed as a selector, // and returns a compiled and ready-to-run Selector. // // ParseJSONSelector is functionally equivalent to combining ParseJSONSelector and CompileSelector into one step. func ParseAndCompileJSONSelector(jsonStr string) (selector.Selector, error) { nb := basicnode.Prototype.Any.NewBuilder() if err := dagjson.Decode(nb, strings.NewReader(jsonStr)); err != nil { return nil, err } if s, err := selector.CompileSelector(nb.Build()); err != nil { return nil, err } else { return s, nil } } func must(s datamodel.Node, e error) datamodel.Node { if e != nil { panic(e) } return s } // CommonSelector_MatchPoint is a selector that matches exactly one thing: the first node it touches. // It doesn't walk anywhere at all. // // This is not a very useful selector, but is an example of how selectors can be written. var CommonSelector_MatchPoint = must(ParseJSONSelector(`{".":{}}`)) // CommonSelector_MatchChildren will examine the node it is applied to, // walk to each of its children, and match the children. // It does not recurse. // Note that the root node itself is visited (necessarily!) but it is not "matched". var CommonSelector_MatchChildren = must(ParseJSONSelector(`{"a":{">":{".":{}}}}`)) // CommonSelector_ExploreAllRecursively is a selector that walks over a graph of data, // recursively, without limit (!) until it reaches every part of the graph. // (This is safe to assume will halt eventually, because in IPLD, we work with DAGs -- // although it still may be a bad idea to do this in practice, // because you could accidentally do this on terabytes of linked data, and that would still take a while!) // // It does not actually _match_ anything at all. // That means if you're intercepting block loads (e.g. you're looking at calls to LinkSystem.StorageReadOpener), you'll see them; // and if you're using `traversal.AdvVisitFn`, you'll still hear about nodes visited during the exploration; // however, if you're using just `traversal.VisitFn`, nothing is considered "matched", so that callback will never be called. var CommonSelector_ExploreAllRecursively = must(ParseJSONSelector(`{"R":{"l":{"none":{}},":>":{"a":{">":{"@":{}}}}}}`)) // CommonSelector_MatchAllRecursively is like CommonSelector_ExploreAllRecursively, but also matching everything it touches. // The first thing inside the recursion is an ExploreUnion clause (which means the selection continues with multiple logical paths); // the first thing inside that union clause is a Matcher clause; // the second thing inside that union is the ExploreAll clause, which gets us deeper, and then that contains the ExploreRecursiveEdge. var CommonSelector_MatchAllRecursively = must(ParseJSONSelector(`{"R":{"l":{"none":{}},":>":{"|":[{".":{}},{"a":{">":{"@":{}}}}]}}}`)) ================================================ FILE: traversal/selector/parse/selector_parse_test.go ================================================ package selectorparse // nothing -- this file just makes sure the vars get initialized, which is a defacto test. ================================================ FILE: traversal/selector/selector.go ================================================ package selector import ( "fmt" "github.com/ipld/go-ipld-prime/datamodel" ) // Selector is a "compiled" and executable IPLD Selector. // It can be put to work with functions like traversal.Walk, // which will use the Selector's guidance to decide how to traverse an IPLD data graph. // A user will not generally call any of the methods of Selector themselves, nor implement the interface; // it is produced by "compile" functions in this package, and used by functions in the `traversal` package. // // A Selector is created by parsing an IPLD Data Model document that declares a Selector // (this is accomplished with functions like CompileSelector). // To make this even easier, there is a `parse` subpackage, // which contains helper methods for parsing direction from a JSON Selector document to a compiled Selector value. // Alternatively, there is a `builder` subpackage, // which may be useful if you would rather create the Selector declaration programmatically in golang // (however, we recommend using this sparingly, because part of what makes Selectors cool is their language-agnostic declarative nature). // // There is no way to go backwards from this "compiled" Selector type into the declarative IPLD data model information that produced it. // That declaration information is discarded after compilation in order to limit the amount of memory held. // Therefore, if you're building APIs about Selector composition, keep in mind that // you'll probably want to approach this be composing the Data Model declaration documents, // and you should *not* attempt to be composing this type, which is only for the "compiled" result. type Selector interface { // Notes for you who implements a Selector: // this type holds the state describing what we will do at one step in a traversal. // The actual traversal stepping is applied *from the outside* (and this is implemented mostly in the `traversal` package; // this type just gives it instructions on how to step. // Each of the functions on this type should be pure; they can can read the Selector's fields, but should treat them as config, not as state -- the Selector should never mutate. // // The traversal process will ask things of a Selector in three phases, // and control flow will bounce back and forth between traversal logic and selector evaluation -- // traversal owns the actual walking (and any data loading), and just briefly dips down into the Selector so it can answer questions: // T1. Traversal starts at some Node with some Selector. // S1. First, the traversal asks the Selector what its "interests" are. // This lets the Selector hint to the traversal process what it should load, // which can be important for performance if not all of the next data elements are in memory already. // (This is applicable to ADLs which contain large sharded data, for example.) // (The "interests" phase should be _fast_; more complicated checks, and anything that actually looks at the children, should wait until the "explore" phase; // in fact, for this reason, the `Interests` function doesn't even get to look at the data at all yet.) // T2. The traversal looks at the Node and its actual fields, and what the Selector just said are interesting, // and between the two of them figures out what's actually here to act on. // (Note that the Selector can say that certain paths are interesting, and that path can then not be there.) // S2. Second, the code driving the traversal will ask us to "explore", **stepwise**. // The "explore" step is applied **repeatedly**: once per pathSegment that identifies a child in the Node. // (If `Interests()` returned a list, `Explore` will be called for each element in the list (as long as that pathSegment actually existed in the Node, of course); // or if `Interest()` returned no guidance, `Explore` will be called for everything in the object.) // S2.a. The "explore" step returns a new Selector object, with instructions about how to continue the walk for the reached object and beneath. // (Note that the "explore" step can also return `nil` here to say "actually, don't look any further", // and it may do so even if the "interests" phase suggested there might be something to follow up on here. (Remember "interests" had to be fast, and was a first pass only.)) // T2.a. ***Recursion time!*** // The traversal now takes that pathSegment and that subsequent Selector produced by `Explore`, // gets the child Node at that pathSegment, and recurses into traversing on that Node with that Selector! // It is also possibly ***link load time***, right before recursing: // if the child node is a Link, the traversal may choose to load it now, // and then do the recursion on the loaded Node (instead of on the actual direct child Node, which was a Link) with the next Selector. // T2.b. When the recursion is done, the traversal goes on to repeat S2, with the next pathSegment, // until it runs out of things to do. // T3. The traversal asks the Selector to "decide" if this current Node is one that is "matched or not. // See the Selector specs for discussion on "matched" vs "reached"/"visited" nodes. // (Long story short: the traversal probably fires off callbacks for "matched" nodes, aka if `Decide` says `true`.) // S3. The selector does so. // T4. The traversal for this node is done. // // Phase T3+S3 can also be T0+S0, which makes for a pre-order traversal instead of a post-order traversal. // The Selector doesn't know the difference. // (In particular, a Selector implementation absolutely may **not** assume `Decide` will be called before `Interests`, and may **not** hold onto a Node statefully, etc.) // // Note that it's not until phase T2.a that the traversal actually loads child Nodes. // This is interesting because it's *after* when the Selector is asked to `Explore` and yield a subsequent Selector to use on that upcoming Node. // // Can `Explore` and `Decide` do Link loading on their own? Do they need to? // Right now, no, they can't. (Sort of.) They don't have access to a LinkLoader; the traversal would have to give them one. // This might be needed in the future, e.g. if the Selector has a Condition clause that requires looking deeper; so far, we don't have those features, so it hasn't been needed. // The "sort of" is for ADLs. ADLs that work with large sharded data sometimes hold onto their own LinkLoader and apply it transparently. // In that case, of course, `Explore` and `Decide` can just interrogate the Node they've been given, and that may cause link loading. // (If that happens, we're currently assuming the ADL has a reasonable caching behavior. It's very likely that the traversal will look up the same paths that Explore just looked up (assuming the Condition told exploration to continue).) // // Interests should return either a list of PathSegment we're likely interested in, // **or nil**, which indicates we're a high-cardinality or expression-based selection clause and thus we'll need all segments proposed to us. // Note that a non-nil zero length list of PathSegment is distinguished from nil: this would mean this selector is interested absolutely nothing. // // Traversal will call this before calling Explore, and use it to try to call Explore less often (or even avoid iterating on the data node at all). Interests() []datamodel.PathSegment // Explore is told about the node we're at, and the pathSegment inside it to consider, // and returns either nil, if we shouldn't explore that path any further, // or returns a Selector, which should then be used to explore the child at that path. // // Note that the node parameter is not the child, it's the node we're currently at. // (Often, this is sufficient information: consider ExploreFields, // which only even needs to regard the pathSegment, and not the node at all.) // // Remember that Explore does **not** iterate `node` itself; the visits to any children of `node` will be driven from the outside, by the traversal function. // (The Selector's job is just guiding that process by returning information.) // The architecture works this way so that a sufficiently clever traversal function could consider several reasons for exploring a node before deciding whether to do so. Explore(node datamodel.Node, child datamodel.PathSegment) (subsequent Selector, err error) // Decide returns true if the subject node is "matched". // // Only "Matcher" clauses actually implement this in a way that ever returns "true". // See the Selector specs for discussion on "matched" vs "reached"/"visited" nodes. Decide(node datamodel.Node) bool // Match is an extension to Decide allowing the matcher to `decide` a transformation of // the matched node. This is used for `Subset` match behavior. If the node is matched, // the first argument will be the matched node. If it is not matched, the first argument // will be null. If there is an error, the first argument will be null. Match(node datamodel.Node) (datamodel.Node, error) } // REVIEW: do ParsedParent and ParseContext need to be exported? They're mostly used during the compilation process. // ParsedParent is created whenever you are parsing a selector node that may have // child selectors nodes that need to know it type ParsedParent interface { Link(s Selector) bool } // ParseContext tracks the progress when parsing a selector type ParseContext struct { parentStack []ParsedParent } // CompileSelector accepts a datamodel.Node which should contain data that declares a Selector. // The data layout expected for this declaration is documented in https://datamodel.io/specs/selectors/ . // // If the Selector is compiled successfully, it is returned. // Otherwise, if the given data Node doesn't match the expected shape for a Selector declaration, // or there are any other problems compiling the selector // (such as a recursion edge with no enclosing recursion declaration, etc), // then nil and an error will be returned. func CompileSelector(dmt datamodel.Node) (Selector, error) { return ParseContext{}.ParseSelector(dmt) } // ParseSelector is an alias for CompileSelector, and is deprecated. // Prefer CompileSelector. func ParseSelector(dmt datamodel.Node) (Selector, error) { return CompileSelector(dmt) } // ParseSelector creates a Selector from an IPLD Selector Node with the given context func (pc ParseContext) ParseSelector(n datamodel.Node) (Selector, error) { if n.Kind() != datamodel.Kind_Map { return nil, fmt.Errorf("selector spec parse rejected: selector is a keyed union and thus must be a map") } if n.Length() != 1 { return nil, fmt.Errorf("selector spec parse rejected: selector is a keyed union and thus must be single-entry map") } kn, v, _ := n.MapIterator().Next() kstr, _ := kn.AsString() // Switch over the single key to determine which selector body comes next. // (This switch is where the keyed union discriminators concretely happen.) switch kstr { case SelectorKey_ExploreFields: return pc.ParseExploreFields(v) case SelectorKey_ExploreAll: return pc.ParseExploreAll(v) case SelectorKey_ExploreIndex: return pc.ParseExploreIndex(v) case SelectorKey_ExploreRange: return pc.ParseExploreRange(v) case SelectorKey_ExploreUnion: return pc.ParseExploreUnion(v) case SelectorKey_ExploreRecursive: return pc.ParseExploreRecursive(v) case SelectorKey_ExploreRecursiveEdge: return pc.ParseExploreRecursiveEdge(v) case SelectorKey_ExploreInterpretAs: return pc.ParseExploreInterpretAs(v) case SelectorKey_Matcher: return pc.ParseMatcher(v) default: return nil, fmt.Errorf("selector spec parse rejected: %q is not a known member of the selector union", kstr) } } // PushParent puts a parent onto the stack of parents for a parse context func (pc ParseContext) PushParent(parent ParsedParent) ParseContext { l := len(pc.parentStack) parents := make([]ParsedParent, 0, l+1) parents = append(parents, parent) parents = append(parents, pc.parentStack...) return ParseContext{parents} } // SegmentIterator iterates either a list or a map, generating PathSegments // instead of indexes or keys type SegmentIterator interface { Next() (pathSegment datamodel.PathSegment, value datamodel.Node, err error) Done() bool } // NewSegmentIterator generates a new iterator based on the node type func NewSegmentIterator(n datamodel.Node) SegmentIterator { if n.Kind() == datamodel.Kind_List { return listSegmentIterator{n.ListIterator()} } return mapSegmentIterator{n.MapIterator()} } type listSegmentIterator struct { datamodel.ListIterator } func (lsi listSegmentIterator) Next() (pathSegment datamodel.PathSegment, value datamodel.Node, err error) { i, v, err := lsi.ListIterator.Next() return datamodel.PathSegmentOfInt(i), v, err } func (lsi listSegmentIterator) Done() bool { return lsi.ListIterator.Done() } type mapSegmentIterator struct { datamodel.MapIterator } func (msi mapSegmentIterator) Next() (pathSegment datamodel.PathSegment, value datamodel.Node, err error) { k, v, err := msi.MapIterator.Next() if err != nil { return datamodel.PathSegment{}, v, err } kstr, _ := k.AsString() return datamodel.PathSegmentOfString(kstr), v, err } func (msi mapSegmentIterator) Done() bool { return msi.MapIterator.Done() } ================================================ FILE: traversal/selector/spec_test.go ================================================ package selector_test import ( "bytes" "io" "os" "regexp" "testing" qt "github.com/frankban/quicktest" "github.com/warpfork/go-testmark" "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/codec/dagjson" "github.com/ipld/go-ipld-prime/codec/json" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/fluent/qp" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/traversal" selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" ) func TestSpecFixtures(t *testing.T) { dir := "../../.ipld/specs/selectors/fixtures/" testOneSpecFixtureFile(t, dir+"selector-fixtures-1.md") testOneSpecFixtureFile(t, dir+"selector-fixtures-recursion.md") } func testOneSpecFixtureFile(t *testing.T, filename string) { data, err := os.ReadFile(filename) if os.IsNotExist(err) { t.Skipf("not running spec suite: %s (did you clone the submodule with the data?)", err) } crre := regexp.MustCompile(`\r?\n`) data = []byte(crre.ReplaceAllString(string(data), "\n")) // fix windows carriage-return doc, err := testmark.Parse(data) qt.Assert(t, err, qt.IsNil) // Data hunk in this spec file are in "directories" of a test scenario each. doc.BuildDirIndex() for _, dir := range doc.DirEnt.ChildrenList { t.Run(dir.Name, func(t *testing.T) { // Each "directory" contains three piece of data: // - `data` -- this is the "block". It's arbitrary example data. They're all in json (or dag-json) format, for simplicity. // - `selector` -- this is the selector. Again, as json. // - `expect-visit` -- these are json lines (one json object on each line) containing description of each node that should be visited, in order. fixtureData := dir.Children["data"].Hunk.Body fixtureSelector := dir.Children["selector"].Hunk.Body fixtureExpect := dir.Children["expect-visit"].Hunk.Body // Parse data into DMT form. dataDmt, err := ipld.Decode(fixtureData, dagjson.Decode) qt.Assert(t, err, qt.IsNil) // Parse and compile Selector. // (This is already arguably a test event on its own. selector, err := selectorparse.ParseAndCompileJSONSelector(string(fixtureSelector)) qt.Assert(t, err, qt.IsNil) // Go! // We'll store the logs of our visit events as... ipld Nodes, actually. // This will make them easy to serialize, which is good for two reasons: // at the end, we're actually going to... do that, and use string diffs for the final assertion // (because string diffing is actually really nice for aggregate feedback in a system like this); // and also that means we're ready to save updated serial data into the fixture files, if we did want to patch them. var visitLogs []datamodel.Node traversal.WalkAdv(dataDmt, selector, func(prog traversal.Progress, n datamodel.Node, reason traversal.VisitReason) error { // Munge info about where we are into DMT shaped like the expectation records in the fixture. visitEventDescr, err := qp.BuildMap(basicnode.Prototype.Any, 3, func(ma datamodel.MapAssembler) { qp.MapEntry(ma, "path", qp.String(prog.Path.String())) qp.MapEntry(ma, "node", qp.Map(1, func(ma datamodel.MapAssembler) { qp.MapEntry(ma, n.Kind().String(), func(na datamodel.NodeAssembler) { switch n.Kind() { case datamodel.Kind_Map, datamodel.Kind_List: na.AssignNull() default: na.AssignNode(n) } }) })) qp.MapEntry(ma, "matched", qp.Bool(reason == traversal.VisitReason_SelectionMatch)) }) if reason == traversal.VisitReason_SelectionMatch && n.Kind() == datamodel.Kind_Bytes { if lbn, ok := n.(datamodel.LargeBytesNode); ok { rdr, err := lbn.AsLargeBytes() if err == nil { io.Copy(io.Discard, rdr) } } _, err := n.AsBytes() if err != nil { panic("insanity at a deeper level than this test's target") } } if err != nil { panic("insanity at a deeper level than this test's target") } visitLogs = append(visitLogs, visitEventDescr) return nil }) // Brief detour -- we're going to bounce the fixture data through our own deserialize and serialize. // Just to normalize the heck out of it. I'm not really interested in if the fixture files have non-normative whitespace in them. var fixtureExpectNormBuf bytes.Buffer for _, line := range bytes.Split(fixtureExpect, []byte{'\n'}) { if len(line) == 0 { continue } exp, err := ipld.Decode(line, json.Decode) qt.Assert(t, err, qt.IsNil) qt.Assert(t, ipld.EncodeStreaming(&fixtureExpectNormBuf, exp, json.Encode), qt.IsNil) fixtureExpectNormBuf.WriteByte('\n') } // Serialize our own visit logs now too. var visitLogString bytes.Buffer for _, logEnt := range visitLogs { qt.Assert(t, ipld.EncodeStreaming(&visitLogString, logEnt, json.Encode), qt.IsNil) visitLogString.WriteByte('\n') } // DIFF TIME. qt.Assert(t, visitLogString.String(), qt.CmpEquals(), fixtureExpectNormBuf.String()) }) } } ================================================ FILE: traversal/walk.go ================================================ package traversal import ( "errors" "fmt" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/linking" "github.com/ipld/go-ipld-prime/linking/preload" "github.com/ipld/go-ipld-prime/traversal/selector" ) // phase is an internal enum used to track the current phase of a walk. It's // used to control for a preload pass over a block if one is required. type phase int const ( phasePreload phase = iota phaseTraverse phase = iota ) // WalkLocal walks a tree of Nodes, visiting each of them, // and calling the given VisitFn on all of them; // it does not traverse any links. // // WalkLocal can skip subtrees if the VisitFn returns SkipMe, // but lacks any other options for controlling or directing the visit; // consider using some of the various Walk functions with Selector parameters if you want more control. func WalkLocal(n datamodel.Node, fn VisitFn) error { return Progress{}.WalkLocal(n, fn) } // WalkMatching walks a graph of Nodes, deciding which to visit by applying a Selector, // and calling the given VisitFn on those that the Selector deems a match. // // This function is a helper function which starts a new walk with default configuration. // It cannot cross links automatically (since this requires configuration). // Use the equivalent WalkMatching function on the Progress structure // for more advanced and configurable walks. func WalkMatching(n datamodel.Node, s selector.Selector, fn VisitFn) error { return Progress{}.WalkMatching(n, s, fn) } // WalkAdv is identical to WalkMatching, except it is called for *all* nodes // visited (not just matching nodes), together with the reason for the visit. // An AdvVisitFn is used instead of a VisitFn, so that the reason can be provided. // // This function is a helper function which starts a new walk with default configuration. // It cannot cross links automatically (since this requires configuration). // Use the equivalent WalkAdv function on the Progress structure // for more advanced and configurable walks. func WalkAdv(n datamodel.Node, s selector.Selector, fn AdvVisitFn) error { return Progress{}.WalkAdv(n, s, fn) } // WalkTransforming walks a graph of Nodes, deciding which to alter by applying a Selector, // and calls the given TransformFn to decide what new node to replace the visited node with. // A new Node tree will be returned (the original is unchanged). // // This function is a helper function which starts a new walk with default configuration. // It cannot cross links automatically (since this requires configuration). // Use the equivalent WalkTransforming function on the Progress structure // for more advanced and configurable walks. func WalkTransforming(n datamodel.Node, s selector.Selector, fn TransformFn) (datamodel.Node, error) { return Progress{}.WalkTransforming(n, s, fn) } // WalkMatching walks a graph of Nodes, deciding which to visit by applying a Selector, // and calling the given VisitFn on those that the Selector deems a match. // // WalkMatching is a read-only traversal. // See WalkTransforming if looking for a way to do "updates" to a tree of nodes. // // Provide configuration to this process using the Config field in the Progress object. // // This walk will automatically cross links, but requires some configuration // with link loading functions to do so. // // Traversals are defined as visiting a (node,path) tuple. // This is important to note because when walking DAGs with Links, // it means you may visit the same node multiple times // due to having reached it via a different path. // (You can prevent this by using a LinkLoader function which memoizes a set of // already-visited Links, and returns a SkipMe when encountering them again.) // // WalkMatching (and the other traversal functions) can be used again again inside the VisitFn! // By using the traversal.Progress handed to the VisitFn, // the Path recorded of the traversal so far will continue to be extended, // and thus continued nested uses of Walk and Focus will see the fully contextualized Path. // // WalkMatching can be configured to run with a Preloader. // When a Preloader is configured, the walk will first do a "preload" pass over the initial, // root tree up to link boundaries and report any links encountered to the preloader. // It will then perform a second pass over the tree, calling the VisitFn where necessary as per normal WalkMatching behavior. // This two-pass operation will continue for each block loaded, allowing the preloader to // potentially asynchronously preload any blocks that are going to be encountered at a future point in the walk. func (prog Progress) WalkMatching(n datamodel.Node, s selector.Selector, fn VisitFn) error { prog.init() return prog.walkBlock(n, s, func(prog Progress, n datamodel.Node, tr VisitReason) error { if tr != VisitReason_SelectionMatch { return nil } return fn(prog, n) }) } // WalkLocal is the same as the package-scope function of the same name, // but considers an existing Progress state (and any config it might reference). func (prog Progress) WalkLocal(n datamodel.Node, fn VisitFn) error { if err := prog.checkNodeBudget(); err != nil { return err } // Visit the current node. if err := fn(prog, n); err != nil { if _, ok := err.(SkipMe); ok { return nil } return err } // Recurse on nodes with a recursive kind; otherwise just return. switch n.Kind() { case datamodel.Kind_Map: for itr := n.MapIterator(); !itr.Done(); { k, v, err := itr.Next() if err != nil { return err } ks, _ := k.AsString() progNext := prog progNext.Path = prog.Path.AppendSegmentString(ks) if err := progNext.WalkLocal(v, fn); err != nil { return err } } return nil case datamodel.Kind_List: for itr := n.ListIterator(); !itr.Done(); { idx, v, err := itr.Next() if err != nil { return err } progNext := prog progNext.Path = prog.Path.AppendSegmentInt(idx) if err := progNext.WalkLocal(v, fn); err != nil { return err } } return nil default: return nil } } // WalkAdv is identical to WalkMatching, except it is called for *all* nodes // visited (not just matching nodes), together with the reason for the visit. // An AdvVisitFn is used instead of a VisitFn, so that the reason can be provided. func (prog Progress) WalkAdv(n datamodel.Node, s selector.Selector, fn AdvVisitFn) error { prog.init() return prog.walkBlock(n, s, fn) } // walkBlock anchors a walk at the beginning of the traversal and at the // beginning of each new link traversed. This allows us to do a preload phase if // we have a preloader configured. func (prog Progress) walkBlock(n datamodel.Node, s selector.Selector, visitFn AdvVisitFn) error { ph := phaseTraverse var budget *Budget if prog.Cfg.Preloader != nil { ph = phasePreload // preserve the budget so we can reset it for the second pass; it will // likely not correctly apply during the preload phase because it // doesn't descend into links first. But we'll use it anyway as a // best-guess because we have nothing better budget = prog.Budget.Clone() } // First pass. err := prog.walkAdv(ph, n, s, visitFn) if err != nil && (ph != phasePreload || !errors.Is(&ErrBudgetExceeded{}, err)) { return err } if ph == phasePreload { // First past was a preload; now do the _real_ pass. prog.Budget = budget // reset return prog.walkAdv(phaseTraverse, n, s, visitFn) } return nil } // walkAdv is the main recursive walk function, called to iterate through // recursive nodes (root node, maps, lists and new link root nodes). func (prog Progress) walkAdv(ph phase, n datamodel.Node, s selector.Selector, visitFn AdvVisitFn) error { if err := prog.checkNodeBudget(); err != nil { return err } // If we need to interpret this node in an alternative form, reify and replace. if rn, rs, err := prog.reify(n, s); err != nil { return err } else if rn != nil { n = rn s = rs } // Call the visit function if necessary. if err := prog.visit(ph, n, s, visitFn); err != nil { return err } // If we're handling scalars (e.g. not maps and lists) we can return now. switch n.Kind() { case datamodel.Kind_Map, datamodel.Kind_List: // continue default: return nil } // For maps and lists: recurse (in one of two ways, depending on if the selector also states specific interests). haveStartAtPath := prog.Cfg.StartAtPath.Len() > 0 var reachedStartAtPath bool recurse := func(v datamodel.Node, ps datamodel.PathSegment) error { // First, make sure we're past the start path; if one is specified. if haveStartAtPath { if reachedStartAtPath { prog.PastStartAtPath = reachedStartAtPath } else if !prog.PastStartAtPath && prog.Path.Len() < prog.Cfg.StartAtPath.Len() { if ps.Equals(prog.Cfg.StartAtPath.Segments()[prog.Path.Len()]) { reachedStartAtPath = true } if !reachedStartAtPath { return nil } } } if err := prog.explore(ph, s, n, visitFn, v, ps); err != nil { return err } return nil } attn := s.Interests() if attn == nil { // no specific interests; recurse on all children. for itr := selector.NewSegmentIterator(n); !itr.Done(); { ps, v, err := itr.Next() if err != nil { return err } if err := recurse(v, ps); err != nil { return err } } return nil } if len(attn) == 0 { // nothing to see here return nil } // specific interests, recurse on those. for _, ps := range attn { if v, err := n.LookupBySegment(ps); err != nil { continue } else if err := recurse(v, ps); err != nil { return err } } return nil } func (prog Progress) checkNodeBudget() error { if prog.Budget != nil { if prog.Budget.NodeBudget <= 0 { return &ErrBudgetExceeded{BudgetKind: "node", Path: prog.Path} } prog.Budget.NodeBudget-- } return nil } func (prog Progress) checkLinkBudget(lnk datamodel.Link) error { if prog.Budget != nil { if prog.Budget.LinkBudget <= 0 { return &ErrBudgetExceeded{BudgetKind: "link", Path: prog.Path, Link: lnk} } prog.Budget.LinkBudget-- } return nil } func (prog Progress) reify(n datamodel.Node, s selector.Selector) (datamodel.Node, selector.Selector, error) { // refiy the node if advised. if rs, ok := s.(selector.Reifiable); ok { adl := rs.NamedReifier() if prog.Cfg.LinkSystem.KnownReifiers == nil { return nil, nil, fmt.Errorf("adl requested but not supported by link system: %q", adl) } reifier, ok := prog.Cfg.LinkSystem.KnownReifiers[adl] if !ok { return nil, nil, fmt.Errorf("unregistered adl requested: %q", adl) } rn, err := reifier(linking.LinkContext{ Ctx: prog.Cfg.Ctx, LinkPath: prog.Path, }, n, &prog.Cfg.LinkSystem) if err != nil { return nil, nil, fmt.Errorf("failed to reify node as %q: %w", adl, err) } // explore into the `InterpretAs` clause to the child selector. s, err = s.Explore(n, datamodel.EmptyPathSegment) if err != nil { return nil, nil, err } return rn, s, nil } return nil, nil, nil } // visit calls the visitor if required func (prog Progress) visit(ph phase, n datamodel.Node, s selector.Selector, visitFn AdvVisitFn) error { if ph != phaseTraverse { return nil } if !prog.PastStartAtPath && prog.Path.Len() < prog.Cfg.StartAtPath.Len() { return nil } // Decide if this node is matched -- do callbacks as appropriate. match, err := s.Match(n) if err != nil { return err } if match != nil { return visitFn(prog, match, VisitReason_SelectionMatch) } return visitFn(prog, n, VisitReason_SelectionCandidate) } // explore is called to explore a single node, and recurse into it if necessary, // including loading and recursing into links if the node is a link. func (prog Progress) explore( ph phase, s selector.Selector, n datamodel.Node, visitFn AdvVisitFn, v datamodel.Node, ps datamodel.PathSegment, ) error { sNext, err := s.Explore(n, ps) if err != nil { return err } if sNext == nil { return nil } progNext := prog progNext.Path = prog.Path.AppendSegment(ps) if v.Kind() != datamodel.Kind_Link { return progNext.walkAdv(ph, v, sNext, visitFn) } lnk, _ := v.AsLink() if prog.Cfg.LinkVisitOnlyOnce { if _, seen := prog.SeenLinks[lnk]; seen { return nil } if ph == phaseTraverse { prog.SeenLinks[lnk] = struct{}{} } } if ph == phasePreload { if err := prog.checkLinkBudget(lnk); err != nil { return err } pctx := preload.PreloadContext{ Ctx: prog.Cfg.Ctx, BasePath: prog.Path, ParentNode: n, } pl := preload.Link{ Segment: ps, LinkNode: v, Link: lnk, } prog.Cfg.Preloader(pctx, pl) return nil } progNext.LastBlock.Path = progNext.Path progNext.LastBlock.Link = lnk v, err = progNext.loadLink(lnk, v, n) if err != nil { if _, ok := err.(SkipMe); ok { return nil } return err } return progNext.walkBlock(v, sNext, visitFn) } // loadLink is called to load a link from the configured LinkSystem with the // appropriate prototype. func (prog Progress) loadLink(lnk datamodel.Link, v datamodel.Node, parent datamodel.Node) (datamodel.Node, error) { if err := prog.checkLinkBudget(lnk); err != nil { return nil, err } // Put together the context info we'll offer to the loader and prototypeChooser. lnkCtx := linking.LinkContext{ Ctx: prog.Cfg.Ctx, LinkPath: prog.Path, LinkNode: v, ParentNode: parent, } // Pick what in-memory format we will build. np, err := prog.Cfg.LinkTargetNodePrototypeChooser(lnk, lnkCtx) if err != nil { return nil, fmt.Errorf("error traversing node at %q: could not load link %q: %w", prog.Path, lnk, err) } // Load link! n, err := prog.Cfg.LinkSystem.Load(lnkCtx, lnk, np) if err != nil { if _, ok := err.(SkipMe); ok { return nil, err } return nil, fmt.Errorf("error traversing node at %q: could not load link %q: %w", prog.Path, lnk, err) } return n, nil } // WalkTransforming walks a graph of Nodes, deciding which to alter by applying a Selector, // and calls the given TransformFn to decide what new node to replace the visited node with. // A new Node tree will be returned (the original is unchanged). // // If the TransformFn returns the same Node which it was called with, // then the transform is a no-op; if every visited node is a no-op, // then the root node returned from the walk as a whole will also be // the same as its starting Node (no new memory will be used). // // When a Node is replaced, no further recursion of this walk will occur on its contents. // (You can certainly do a additional traversals, including transforms, // from inside the TransformFn while building the replacement node.) // // The prototype (that is, implementation) of Node returned will be the same as the // prototype of the Nodes at the same positions in the existing tree // (literally, builders used to construct any new needed intermediate nodes // are chosen by asking the existing nodes about their prototype). func (prog Progress) WalkTransforming(n datamodel.Node, s selector.Selector, fn TransformFn) (datamodel.Node, error) { prog.init() return prog.walkTransforming(n, s, fn) } func (prog Progress) walkTransforming(n datamodel.Node, s selector.Selector, fn TransformFn) (datamodel.Node, error) { if err := prog.checkNodeBudget(); err != nil { return nil, err } if rn, rs, err := prog.reify(n, s); err != nil { return nil, err } else if rn != nil { n = rn s = rs } // Decide if this node is matched -- do callbacks as appropriate. if s.Decide(n) { new_n, err := fn(prog, n) if err != nil { return nil, err } if new_n != n { // don't continue on transformed subtrees return new_n, nil } } // If we're handling scalars (e.g. not maps and lists) we can return now. nk := n.Kind() switch nk { case datamodel.Kind_List: return prog.walk_transform_iterateList(n, s, fn, s.Interests()) case datamodel.Kind_Map: return prog.walk_transform_iterateMap(n, s, fn, s.Interests()) default: return n, nil } } func contains(interest []datamodel.PathSegment, candidate datamodel.PathSegment) bool { for _, i := range interest { if i == candidate { return true } } return false } func (prog Progress) walk_transform_iterateList(n datamodel.Node, s selector.Selector, fn TransformFn, attn []datamodel.PathSegment) (datamodel.Node, error) { bldr := n.Prototype().NewBuilder() lstBldr, err := bldr.BeginList(n.Length()) if err != nil { return nil, err } for itr := selector.NewSegmentIterator(n); !itr.Done(); { ps, v, err := itr.Next() if err != nil { return nil, err } if attn == nil || contains(attn, ps) { sNext, err := s.Explore(n, ps) if err != nil { return nil, err } if sNext != nil { progNext := prog progNext.Path = prog.Path.AppendSegment(ps) if v.Kind() == datamodel.Kind_Link { lnk, _ := v.AsLink() if prog.Cfg.LinkVisitOnlyOnce { if _, seen := prog.SeenLinks[lnk]; seen { continue } prog.SeenLinks[lnk] = struct{}{} } progNext.LastBlock.Path = progNext.Path progNext.LastBlock.Link = lnk v, err = progNext.loadLink(lnk, v, n) if err != nil { if _, ok := err.(SkipMe); ok { continue } return nil, err } } next, err := progNext.WalkTransforming(v, sNext, fn) if err != nil { return nil, err } if err := lstBldr.AssembleValue().AssignNode(next); err != nil { return nil, err } } else { if err := lstBldr.AssembleValue().AssignNode(v); err != nil { return nil, err } } } else { if err := lstBldr.AssembleValue().AssignNode(v); err != nil { return nil, err } } } if err := lstBldr.Finish(); err != nil { return nil, err } return bldr.Build(), nil } func (prog Progress) walk_transform_iterateMap(n datamodel.Node, s selector.Selector, fn TransformFn, attn []datamodel.PathSegment) (datamodel.Node, error) { bldr := n.Prototype().NewBuilder() mapBldr, err := bldr.BeginMap(n.Length()) if err != nil { return nil, err } for itr := selector.NewSegmentIterator(n); !itr.Done(); { ps, v, err := itr.Next() if err != nil { return nil, err } if err := mapBldr.AssembleKey().AssignString(ps.String()); err != nil { return nil, err } if attn == nil || contains(attn, ps) { sNext, err := s.Explore(n, ps) if err != nil { return nil, err } if sNext != nil { progNext := prog progNext.Path = prog.Path.AppendSegment(ps) if v.Kind() == datamodel.Kind_Link { lnk, _ := v.AsLink() if prog.Cfg.LinkVisitOnlyOnce { if _, seen := prog.SeenLinks[lnk]; seen { continue } prog.SeenLinks[lnk] = struct{}{} } progNext.LastBlock.Path = progNext.Path progNext.LastBlock.Link = lnk v, err = progNext.loadLink(lnk, v, n) if err != nil { if _, ok := err.(SkipMe); ok { continue } return nil, err } } next, err := progNext.WalkTransforming(v, sNext, fn) if err != nil { return nil, err } if err := mapBldr.AssembleValue().AssignNode(next); err != nil { return nil, err } } else { if err := mapBldr.AssembleValue().AssignNode(v); err != nil { return nil, err } } } else { if err := mapBldr.AssembleValue().AssignNode(v); err != nil { return nil, err } } } if err := mapBldr.Finish(); err != nil { return nil, err } return bldr.Build(), nil } ================================================ FILE: traversal/walk_test.go ================================================ package traversal_test import ( "fmt" "io" "testing" qt "github.com/frankban/quicktest" "github.com/ipld/go-ipld-prime" _ "github.com/ipld/go-ipld-prime/codec/dagjson" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/fluent" "github.com/ipld/go-ipld-prime/linking" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipld/go-ipld-prime/linking/preload" "github.com/ipld/go-ipld-prime/node/basicnode" nodetests "github.com/ipld/go-ipld-prime/node/tests" "github.com/ipld/go-ipld-prime/storage" "github.com/ipld/go-ipld-prime/traversal" "github.com/ipld/go-ipld-prime/traversal/selector" "github.com/ipld/go-ipld-prime/traversal/selector/builder" selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" ) /* Remember, we've got the following fixtures in scope: var ( // baguqeeyexkjwnfy leafAlpha, leafAlphaLnk = encode(basicnode.NewString("alpha")) // baguqeeyeqvc7t3a leafBeta, leafBetaLnk = encode(basicnode.NewString("beta")) // baguqeeyezhlahvq middleMapNode, middleMapNodeLnk = encode(fluent.MustBuildMap(basicnode.Prototype.Map, 3, func(na fluent.MapAssembler) { na.AssembleEntry("foo").AssignBool(true) na.AssembleEntry("bar").AssignBool(false) na.AssembleEntry("nested").CreateMap(2, func(na fluent.MapAssembler) { na.AssembleEntry("alink").AssignLink(leafAlphaLnk) na.AssembleEntry("nonlink").AssignString("zoo") }) })) // baguqeeyehfkkfwa middleListNode, middleListNodeLnk = encode(fluent.MustBuildList(basicnode.Prototype.List, 4, func(na fluent.ListAssembler) { na.AssembleValue().AssignLink(leafAlphaLnk) na.AssembleValue().AssignLink(leafAlphaLnk) na.AssembleValue().AssignLink(leafBetaLnk) na.AssembleValue().AssignLink(leafAlphaLnk) })) // baguqeeyeie4ajfy rootNode, rootNodeLnk = encode(fluent.MustBuildMap(basicnode.Prototype.Map, 4, func(na fluent.MapAssembler) { na.AssembleEntry("plain").AssignString("olde string") na.AssembleEntry("linkedString").AssignLink(leafAlphaLnk) na.AssembleEntry("linkedMap").AssignLink(middleMapNodeLnk) na.AssembleEntry("linkedList").AssignLink(middleListNodeLnk) }))) */ // covers traverse using a variety of selectors. // all cases here use one already-loaded Node; no link-loading exercised. func TestWalkMatching(t *testing.T) { ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) t.Run("traverse selecting true should visit the root", func(t *testing.T) { err := traversal.WalkMatching(basicnode.NewString("x"), selector.Matcher{}, func(prog traversal.Progress, n datamodel.Node) error { qt.Check(t, n, nodetests.NodeContentEquals, basicnode.NewString("x")) qt.Check(t, prog.Path.String(), qt.Equals, datamodel.Path{}.String()) return nil }) qt.Check(t, err, qt.IsNil) }) t.Run("traverse selecting true should visit only the root and no deeper", func(t *testing.T) { err := traversal.WalkMatching(middleMapNode, selector.Matcher{}, func(prog traversal.Progress, n datamodel.Node) error { qt.Check(t, n, qt.Equals, middleMapNode) qt.Check(t, prog.Path.String(), qt.Equals, datamodel.Path{}.String()) return nil }) qt.Check(t, err, qt.IsNil) }) t.Run("traverse selecting fields should work", func(t *testing.T) { ss := ssb.ExploreFields(func(efsb builder.ExploreFieldsSpecBuilder) { efsb.Insert("foo", ssb.Matcher()) efsb.Insert("bar", ssb.Matcher()) }) s, err := ss.Selector() qt.Assert(t, err, qt.IsNil) var order int err = traversal.WalkMatching(middleMapNode, s, func(prog traversal.Progress, n datamodel.Node) error { switch order { case 0: qt.Check(t, n, nodetests.NodeContentEquals, basicnode.NewBool(true)) qt.Check(t, prog.Path.String(), qt.Equals, "foo") case 1: qt.Check(t, n, nodetests.NodeContentEquals, basicnode.NewBool(false)) qt.Check(t, prog.Path.String(), qt.Equals, "bar") } order++ return nil }) qt.Check(t, err, qt.IsNil) qt.Check(t, order, qt.Equals, 2) }) t.Run("traverse selecting fields recursively should work", func(t *testing.T) { ss := ssb.ExploreFields(func(efsb builder.ExploreFieldsSpecBuilder) { efsb.Insert("foo", ssb.Matcher()) efsb.Insert("nested", ssb.ExploreFields(func(efsb builder.ExploreFieldsSpecBuilder) { efsb.Insert("nonlink", ssb.Matcher()) })) }) s, err := ss.Selector() qt.Assert(t, err, qt.IsNil) var order int err = traversal.WalkMatching(middleMapNode, s, func(prog traversal.Progress, n datamodel.Node) error { switch order { case 0: qt.Check(t, n, nodetests.NodeContentEquals, basicnode.NewBool(true)) qt.Check(t, prog.Path.String(), qt.Equals, "foo") case 1: qt.Check(t, n, nodetests.NodeContentEquals, basicnode.NewString("zoo")) qt.Check(t, prog.Path.String(), qt.Equals, "nested/nonlink") } order++ return nil }) qt.Check(t, err, qt.IsNil) qt.Check(t, order, qt.Equals, 2) }) t.Run("traversing across nodes should work", func(t *testing.T) { ss := ssb.ExploreRecursive(selector.RecursionLimitDepth(3), ssb.ExploreUnion( ssb.Matcher(), ssb.ExploreAll(ssb.ExploreRecursiveEdge()), )) s, err := ss.Selector() qt.Check(t, err, qt.IsNil) var order int lsys := cidlink.DefaultLinkSystem() lsys.SetReadStorage(&store) err = traversal.Progress{ Cfg: &traversal.Config{ LinkSystem: lsys, LinkTargetNodePrototypeChooser: basicnode.Chooser, }, }.WalkMatching(middleMapNode, s, func(prog traversal.Progress, n datamodel.Node) error { switch order { case 0: qt.Check(t, n, qt.Equals, middleMapNode) qt.Check(t, prog.Path.String(), qt.Equals, "") case 1: qt.Check(t, n, nodetests.NodeContentEquals, basicnode.NewBool(true)) qt.Check(t, prog.Path.String(), qt.Equals, "foo") case 2: qt.Check(t, n, nodetests.NodeContentEquals, basicnode.NewBool(false)) qt.Check(t, prog.Path.String(), qt.Equals, "bar") case 3: qt.Check(t, n, nodetests.NodeContentEquals, fluent.MustBuildMap(basicnode.Prototype.Map, 2, func(na fluent.MapAssembler) { na.AssembleEntry("alink").AssignLink(leafAlphaLnk) na.AssembleEntry("nonlink").AssignString("zoo") })) qt.Check(t, prog.Path.String(), qt.Equals, "nested") case 4: qt.Check(t, n, nodetests.NodeContentEquals, basicnode.NewString("alpha")) qt.Check(t, prog.Path.String(), qt.Equals, "nested/alink") qt.Check(t, prog.LastBlock.Path.String(), qt.Equals, "nested/alink") qt.Check(t, prog.LastBlock.Link.String(), qt.Equals, leafAlphaLnk.String()) case 5: qt.Check(t, n, nodetests.NodeContentEquals, basicnode.NewString("zoo")) qt.Check(t, prog.Path.String(), qt.Equals, "nested/nonlink") } order++ return nil }) qt.Check(t, err, qt.IsNil) qt.Check(t, order, qt.Equals, 6) }) t.Run("traversing lists should work", func(t *testing.T) { ss := ssb.ExploreRange(0, 3, ssb.Matcher()) s, err := ss.Selector() qt.Check(t, err, qt.IsNil) var order int lsys := cidlink.DefaultLinkSystem() lsys.SetReadStorage(&store) err = traversal.Progress{ Cfg: &traversal.Config{ LinkSystem: lsys, LinkTargetNodePrototypeChooser: basicnode.Chooser, }, }.WalkMatching(middleListNode, s, func(prog traversal.Progress, n datamodel.Node) error { switch order { case 0: qt.Check(t, n, nodetests.NodeContentEquals, basicnode.NewString("alpha")) qt.Check(t, prog.Path.String(), qt.Equals, "0") qt.Check(t, prog.LastBlock.Path.String(), qt.Equals, "0") qt.Check(t, prog.LastBlock.Link.String(), qt.Equals, leafAlphaLnk.String()) case 1: qt.Check(t, n, nodetests.NodeContentEquals, basicnode.NewString("alpha")) qt.Check(t, prog.Path.String(), qt.Equals, "1") qt.Check(t, prog.LastBlock.Path.String(), qt.Equals, "1") qt.Check(t, prog.LastBlock.Link.String(), qt.Equals, leafAlphaLnk.String()) case 2: qt.Check(t, n, nodetests.NodeContentEquals, basicnode.NewString("beta")) qt.Check(t, prog.Path.String(), qt.Equals, "2") qt.Check(t, prog.LastBlock.Path.String(), qt.Equals, "2") qt.Check(t, prog.LastBlock.Link.String(), qt.Equals, leafBetaLnk.String()) } order++ return nil }) qt.Check(t, err, qt.IsNil) qt.Check(t, order, qt.Equals, 3) }) t.Run("multiple layers of link traversal should work", func(t *testing.T) { ss := ssb.ExploreFields(func(efsb builder.ExploreFieldsSpecBuilder) { efsb.Insert("linkedList", ssb.ExploreAll(ssb.Matcher())) efsb.Insert("linkedMap", ssb.ExploreRecursive(selector.RecursionLimitDepth(3), ssb.ExploreFields(func(efsb builder.ExploreFieldsSpecBuilder) { efsb.Insert("foo", ssb.Matcher()) efsb.Insert("nonlink", ssb.Matcher()) efsb.Insert("alink", ssb.Matcher()) efsb.Insert("nested", ssb.ExploreRecursiveEdge()) }))) }) s, err := ss.Selector() qt.Check(t, err, qt.IsNil) var order int lsys := cidlink.DefaultLinkSystem() lsys.SetReadStorage(&store) err = traversal.Progress{ Cfg: &traversal.Config{ LinkSystem: lsys, LinkTargetNodePrototypeChooser: basicnode.Chooser, }, }.WalkMatching(rootNode, s, func(prog traversal.Progress, n datamodel.Node) error { switch order { case 0: qt.Check(t, n, nodetests.NodeContentEquals, basicnode.NewString("alpha")) qt.Check(t, prog.Path.String(), qt.Equals, "linkedList/0") qt.Check(t, prog.LastBlock.Path.String(), qt.Equals, "linkedList/0") qt.Check(t, prog.LastBlock.Link.String(), qt.Equals, leafAlphaLnk.String()) case 1: qt.Check(t, n, nodetests.NodeContentEquals, basicnode.NewString("alpha")) qt.Check(t, prog.Path.String(), qt.Equals, "linkedList/1") qt.Check(t, prog.LastBlock.Path.String(), qt.Equals, "linkedList/1") qt.Check(t, prog.LastBlock.Link.String(), qt.Equals, leafAlphaLnk.String()) case 2: qt.Check(t, n, nodetests.NodeContentEquals, basicnode.NewString("beta")) qt.Check(t, prog.Path.String(), qt.Equals, "linkedList/2") qt.Check(t, prog.LastBlock.Path.String(), qt.Equals, "linkedList/2") qt.Check(t, prog.LastBlock.Link.String(), qt.Equals, leafBetaLnk.String()) case 3: qt.Check(t, n, nodetests.NodeContentEquals, basicnode.NewString("alpha")) qt.Check(t, prog.Path.String(), qt.Equals, "linkedList/3") qt.Check(t, prog.LastBlock.Path.String(), qt.Equals, "linkedList/3") qt.Check(t, prog.LastBlock.Link.String(), qt.Equals, leafAlphaLnk.String()) case 4: qt.Check(t, n, nodetests.NodeContentEquals, basicnode.NewBool(true)) qt.Check(t, prog.Path.String(), qt.Equals, "linkedMap/foo") qt.Check(t, prog.LastBlock.Path.String(), qt.Equals, "linkedMap") qt.Check(t, prog.LastBlock.Link.String(), qt.Equals, middleMapNodeLnk.String()) case 5: qt.Check(t, n, nodetests.NodeContentEquals, basicnode.NewString("zoo")) qt.Check(t, prog.Path.String(), qt.Equals, "linkedMap/nested/nonlink") qt.Check(t, prog.LastBlock.Path.String(), qt.Equals, "linkedMap") qt.Check(t, prog.LastBlock.Link.String(), qt.Equals, middleMapNodeLnk.String()) case 6: qt.Check(t, n, nodetests.NodeContentEquals, basicnode.NewString("alpha")) qt.Check(t, prog.Path.String(), qt.Equals, "linkedMap/nested/alink") qt.Check(t, prog.LastBlock.Path.String(), qt.Equals, "linkedMap/nested/alink") qt.Check(t, prog.LastBlock.Link.String(), qt.Equals, leafAlphaLnk.String()) } order++ return nil }) qt.Check(t, err, qt.IsNil) qt.Check(t, order, qt.Equals, 7) }) t.Run("no visiting of nodes before start path", func(t *testing.T) { ss := ssb.ExploreFields(func(efsb builder.ExploreFieldsSpecBuilder) { efsb.Insert("linkedList", ssb.ExploreRecursive( selector.RecursionLimitNone(), ssb.ExploreUnion(ssb.Matcher(), ssb.ExploreAll(ssb.ExploreRecursiveEdge())))) efsb.Insert("plain", ssb.ExploreAll(ssb.Matcher())) efsb.Insert("linkedString", ssb.ExploreAll(ssb.Matcher())) efsb.Insert("linkedMap", ssb.ExploreUnion(ssb.Matcher(), ssb.ExploreRecursive(selector.RecursionLimitDepth(3), ssb.ExploreFields(func(efsb builder.ExploreFieldsSpecBuilder) { efsb.Insert("foo", ssb.Matcher()) efsb.Insert("nonlink", ssb.Matcher()) efsb.Insert("alink", ssb.Matcher()) efsb.Insert("nested", ssb.ExploreRecursiveEdge()) })))) }) s, err := ss.Selector() qt.Check(t, err, qt.IsNil) var order int lsys := cidlink.DefaultLinkSystem() lsys.SetReadStorage(&store) visitedCids := make([]string, 0) lsys.StorageReadOpener = func(lnkCtx ipld.LinkContext, lnk ipld.Link) (io.Reader, error) { visitedCids = append(visitedCids, lnk.(cidlink.Link).Cid.String()) return store.GetStream(lnkCtx.Ctx, lnk.(cidlink.Link).Cid.KeyString()) } err = traversal.Progress{ Cfg: &traversal.Config{ LinkSystem: lsys, LinkTargetNodePrototypeChooser: basicnode.Chooser, StartAtPath: datamodel.ParsePath("linkedMap/nested/nonlink"), }, }.WalkMatching(rootNode, s, func(prog traversal.Progress, n datamodel.Node) error { switch order { case 0: qt.Check(t, n, nodetests.NodeContentEquals, basicnode.NewString("zoo")) qt.Check(t, prog.Path.String(), qt.Equals, "linkedMap/nested/nonlink") qt.Check(t, prog.LastBlock.Path.String(), qt.Equals, "linkedMap") qt.Check(t, prog.LastBlock.Link.String(), qt.Equals, middleMapNodeLnk.String()) case 1: qt.Check(t, n, nodetests.NodeContentEquals, basicnode.NewString("alpha")) qt.Check(t, prog.Path.String(), qt.Equals, "linkedMap/nested/alink") qt.Check(t, prog.LastBlock.Path.String(), qt.Equals, "linkedMap/nested/alink") qt.Check(t, prog.LastBlock.Link.String(), qt.Equals, leafAlphaLnk.String()) } order++ return nil }) qt.Check(t, err, qt.IsNil) qt.Check(t, order, qt.Equals, 2) // linkedMap=baguqeeyezhlahvq, alink=baguqeeyexkjwnfy qt.Check(t, visitedCids, qt.DeepEquals, []string{"baguqeeyezhlahvq", "baguqeeyexkjwnfy"}) }) t.Run("no loading of unnecessary nodes before start path", func(t *testing.T) { ss := ssb.ExploreFields(func(efsb builder.ExploreFieldsSpecBuilder) { efsb.Insert("linkedList", ssb.ExploreRecursive(selector.RecursionLimitNone(), ssb.ExploreAll(ssb.ExploreRecursiveEdge()))) efsb.Insert("plain", ssb.ExploreRecursive(selector.RecursionLimitNone(), ssb.ExploreAll(ssb.ExploreRecursiveEdge()))) efsb.Insert("linkedString", ssb.ExploreRecursive(selector.RecursionLimitNone(), ssb.ExploreAll(ssb.ExploreRecursiveEdge()))) efsb.Insert("linkedMap", ssb.ExploreRecursive(selector.RecursionLimitDepth(3), ssb.ExploreFields(func(efsb builder.ExploreFieldsSpecBuilder) { efsb.Insert("foo", ssb.ExploreRecursive(selector.RecursionLimitNone(), ssb.ExploreAll(ssb.ExploreRecursiveEdge()))) efsb.Insert("nonlink", ssb.ExploreRecursive(selector.RecursionLimitNone(), ssb.ExploreAll(ssb.ExploreRecursiveEdge()))) efsb.Insert("alink", ssb.ExploreRecursive(selector.RecursionLimitNone(), ssb.ExploreAll(ssb.ExploreRecursiveEdge()))) efsb.Insert("nested", ssb.ExploreRecursiveEdge()) }))) }) s, err := ss.Selector() qt.Check(t, err, qt.IsNil) lsys := cidlink.DefaultLinkSystem() lsys.SetReadStorage(&store) visitedCids := make([]string, 0) lsys.StorageReadOpener = func(lnkCtx ipld.LinkContext, lnk ipld.Link) (io.Reader, error) { visitedCids = append(visitedCids, lnk.(cidlink.Link).Cid.String()) return store.GetStream(lnkCtx.Ctx, lnk.(cidlink.Link).Cid.KeyString()) } err = traversal.Progress{ Cfg: &traversal.Config{ LinkSystem: lsys, LinkTargetNodePrototypeChooser: basicnode.Chooser, StartAtPath: datamodel.ParsePath("linkedMap/nested/nonlink"), }, }.WalkMatching(rootNode, s, func(prog traversal.Progress, n datamodel.Node) error { return nil }) qt.Check(t, err, qt.IsNil) // linkedMap=baguqeeyezhlahvq, alink=baguqeeyexkjwnfy qt.Check(t, visitedCids, qt.DeepEquals, []string{"baguqeeyezhlahvq", "baguqeeyexkjwnfy"}) }) } func TestWalkBudgets(t *testing.T) { for _, preloader := range []bool{false, true} { t.Run(fmt.Sprintf("preloader=%v", preloader), func(t *testing.T) { t.Run("node-budget-halts", func(t *testing.T) { ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) ss := ssb.ExploreFields(func(efsb builder.ExploreFieldsSpecBuilder) { efsb.Insert("foo", ssb.Matcher()) efsb.Insert("bar", ssb.Matcher()) }) s, err := ss.Selector() qt.Assert(t, err, qt.Equals, nil) var order int prog := traversal.Progress{} prog.Budget = &traversal.Budget{ NodeBudget: 2, // should reach root, then "foo", then stop. } preloadLinks := make([]preload.Link, 0) if preloader { // having a preloader shouldn't change budgeting prog.Cfg = &traversal.Config{ Preloader: func(_ preload.PreloadContext, link preload.Link) { preloadLinks = append(preloadLinks, link) }, } } err = prog.WalkMatching(middleMapNode, s, func(prog traversal.Progress, n datamodel.Node) error { switch order { case 0: qt.Assert(t, n, nodetests.NodeContentEquals, basicnode.NewBool(true)) qt.Assert(t, prog.Path.String(), qt.Equals, "foo") } order++ return nil }) if preloader { qt.Assert(t, preloadLinks, qt.HasLen, 0) } qt.Check(t, order, qt.Equals, 1) // because it should've stopped early qt.Assert(t, err, qt.Not(qt.Equals), nil) qt.Check(t, err.Error(), qt.Equals, `traversal budget exceeded: budget for nodes reached zero while on path "bar"`) }) t.Run("link-budget-halts", func(t *testing.T) { ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) ss := ssb.ExploreAll(ssb.Matcher()) s, err := ss.Selector() qt.Assert(t, err, qt.Equals, nil) var order int lsys := cidlink.DefaultLinkSystem() lsys.SetReadStorage(&store) prog := traversal.Progress{ Cfg: &traversal.Config{ LinkSystem: lsys, LinkTargetNodePrototypeChooser: basicnode.Chooser, }, Budget: &traversal.Budget{ NodeBudget: 9000, LinkBudget: 3, }, } preloadLinks := make([]preload.Link, 0) if preloader { // having a preloader shouldn't change budgeting prog.Cfg.Preloader = func(_ preload.PreloadContext, link preload.Link) { preloadLinks = append(preloadLinks, link) } } err = prog.WalkMatching(middleListNode, s, func(prog traversal.Progress, n datamodel.Node) error { switch order { case 0: qt.Assert(t, n, nodetests.NodeContentEquals, basicnode.NewString("alpha")) qt.Assert(t, prog.Path.String(), qt.Equals, "0") case 1: qt.Assert(t, n, nodetests.NodeContentEquals, basicnode.NewString("alpha")) qt.Assert(t, prog.Path.String(), qt.Equals, "1") case 2: qt.Assert(t, n, nodetests.NodeContentEquals, basicnode.NewString("beta")) qt.Assert(t, prog.Path.String(), qt.Equals, "2") } order++ return nil }) qt.Check(t, order, qt.Equals, 3) qt.Assert(t, err, qt.Not(qt.Equals), nil) qt.Check(t, err.Error(), qt.Equals, `traversal budget exceeded: budget for links reached zero while on path "3" (link: "baguqeeyexkjwnfy")`) if preloader { qt.Assert(t, preloadLinks, qt.HasLen, 3) qt.Check(t, preloadLinks[0].Link, qt.Equals, leafAlphaLnk) qt.Check(t, preloadLinks[1].Link, qt.Equals, leafAlphaLnk) qt.Check(t, preloadLinks[2].Link, qt.Equals, leafBetaLnk) } }) }) } } func TestWalkBlockLoadOrder(t *testing.T) { // a more nested root that we can use to test SkipMe as well // note that in using `rootNodeLnk` here rather than `rootNode` we're using the // dag-json round-trip version which will have different field ordering newRootNode, newRootLink := encode(fluent.MustBuildList(basicnode.Prototype.List, 6, func(na fluent.ListAssembler) { na.AssembleValue().AssignLink(rootNodeLnk) na.AssembleValue().AssignLink(middleListNodeLnk) na.AssembleValue().AssignLink(rootNodeLnk) na.AssembleValue().AssignLink(middleListNodeLnk) na.AssembleValue().AssignLink(rootNodeLnk) na.AssembleValue().AssignLink(middleListNodeLnk) })) linkNames := make(map[datamodel.Link]string) linkNames[newRootLink] = "newRootLink" linkNames[rootNodeLnk] = "rootNodeLnk" linkNames[leafAlphaLnk] = "leafAlphaLnk" linkNames[middleMapNodeLnk] = "middleMapNodeLnk" linkNames[leafAlphaLnk] = "leafAlphaLnk" linkNames[middleListNodeLnk] = "middleListNodeLnk" linkNames[leafAlphaLnk] = "leafAlphaLnk" linkNames[leafBetaLnk] = "leafBetaLnk" /* useful to know CIDs for these when debugging for v, n := range linkNames { t.Logf("n:%v:%v\n", n, v) } */ // the links that we expect from the root node, starting _at_ the root node itself rootNodeExpectedLinks := []datamodel.Link{ rootNodeLnk, middleListNodeLnk, leafAlphaLnk, leafAlphaLnk, leafBetaLnk, leafAlphaLnk, middleMapNodeLnk, leafAlphaLnk, leafAlphaLnk, } // same thing but just for middleListNode middleListNodeLinks := []datamodel.Link{ middleListNodeLnk, leafAlphaLnk, leafAlphaLnk, leafBetaLnk, leafAlphaLnk, } // our newRootNode is a list that contains 3 consecutive links to rootNode expectedAllBlocks := make([]datamodel.Link, 3*(len(rootNodeExpectedLinks)+len(middleListNodeLinks))) for i := 0; i < 3; i++ { copy(expectedAllBlocks[i*len(rootNodeExpectedLinks)+i*len(middleListNodeLinks):], rootNodeExpectedLinks[:]) copy(expectedAllBlocks[(i+1)*len(rootNodeExpectedLinks)+i*len(middleListNodeLinks):], middleListNodeLinks[:]) } verifySelectorLoads := func( t *testing.T, rootNode datamodel.Node, expected []datamodel.Link, s datamodel.Node, linkVisitOnce bool, startAtPath datamodel.Path, preloader preload.Loader, readFn func(lc linking.LinkContext, l datamodel.Link) (io.Reader, error)) { var count int lsys := cidlink.DefaultLinkSystem() lsys.StorageReadOpener = func(lc linking.LinkContext, l datamodel.Link) (io.Reader, error) { // t.Logf("load %d: %v (%s) <> %v (%s) - %s", count, expected[count].String(), linkNames[expected[count]], l.String(), linkNames[l], lc.LinkPath) // t.Logf("%v (%v) %s<> %v (%v)\n", l, linkNames[l], strings.Repeat(" ", 17-len(linkNames[l])), expected[count], linkNames[expected[count]]) qt.Check(t, l.String(), qt.Equals, expected[count].String()) count++ return readFn(lc, l) } sel, err := selector.CompileSelector(s) qt.Check(t, err, qt.IsNil) err = traversal.Progress{ Cfg: &traversal.Config{ LinkSystem: lsys, LinkTargetNodePrototypeChooser: basicnode.Chooser, LinkVisitOnlyOnce: linkVisitOnce, StartAtPath: startAtPath, Preloader: preloader, }, }.WalkMatching(rootNode, sel, func(prog traversal.Progress, n datamodel.Node) error { return nil }) qt.Check(t, err, qt.IsNil) qt.Check(t, count, qt.Equals, len(expected)) } t.Run("CommonSelector_MatchAllRecursively", func(t *testing.T) { s := selectorparse.CommonSelector_MatchAllRecursively verifySelectorLoads(t, newRootNode, expectedAllBlocks, s, false, datamodel.NewPath(nil), nil, func(lctx linking.LinkContext, lnk datamodel.Link) (io.Reader, error) { return storage.GetStream(lctx.Ctx, &store, lnk.Binary()) }) }) t.Run("CommonSelector_ExploreAllRecursively", func(t *testing.T) { s := selectorparse.CommonSelector_ExploreAllRecursively verifySelectorLoads(t, newRootNode, expectedAllBlocks, s, false, datamodel.NewPath(nil), nil, func(lctx linking.LinkContext, lnk datamodel.Link) (io.Reader, error) { return storage.GetStream(lctx.Ctx, &store, lnk.Binary()) }) }) t.Run("explore all with preload", func(t *testing.T) { s := selectorparse.CommonSelector_ExploreAllRecursively newNestedRootNode, _ := encode(fluent.MustBuildList(basicnode.Prototype.List, 2, func(na fluent.ListAssembler) { na.AssembleValue().CreateMap(3, func(ma fluent.MapAssembler) { ma.AssembleEntry("a").AssignLink(rootNodeLnk) ma.AssembleEntry("b").AssignLink(middleListNodeLnk) ma.AssembleEntry("c").AssignLink(rootNodeLnk) }) na.AssembleValue().CreateMap(3, func(ma fluent.MapAssembler) { ma.AssembleEntry("d").AssignLink(middleListNodeLnk) ma.AssembleEntry("e").AssignLink(rootNodeLnk) ma.AssembleEntry("f").AssignLink(middleListNodeLnk) }) })) rootNodePreloads := []datamodel.Link{middleListNodeLnk, middleMapNodeLnk, leafAlphaLnk} middleListNodePreloads := []datamodel.Link{leafAlphaLnk, leafAlphaLnk, leafBetaLnk, leafAlphaLnk} middleMapNodePreloads := []datamodel.Link{leafAlphaLnk} rootNodePreloadsRecursive := [][]datamodel.Link{rootNodePreloads, middleListNodePreloads, middleMapNodePreloads} el := [][]datamodel.Link{ {rootNodeLnk, middleListNodeLnk, rootNodeLnk, middleListNodeLnk, rootNodeLnk, middleListNodeLnk}, } for i := 0; i < 3; i++ { el = append(el, rootNodePreloadsRecursive...) el = append(el, middleListNodePreloads) } expectedLinks := make([]datamodel.Link, 0) for _, l := range el { expectedLinks = append(expectedLinks, l...) } preloadIndex := 0 preloader := func(_ preload.PreloadContext, link preload.Link) { if preloadIndex >= len(expectedLinks) { t.Fatal("too many preloads") } qt.Check(t, link.Link, qt.Equals, expectedLinks[preloadIndex]) preloadIndex++ } verifySelectorLoads(t, newNestedRootNode, expectedAllBlocks, s, false, datamodel.NewPath(nil), preloader, func(lctx linking.LinkContext, lnk datamodel.Link) (io.Reader, error) { return storage.GetStream(lctx.Ctx, &store, lnk.Binary()) }) qt.Check(t, preloadIndex, qt.Equals, len(expectedLinks)) }) t.Run("constructed explore-all selector", func(t *testing.T) { // used commonly in Filecoin and other places to "visit all blocks in stable order" ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) s := ssb.ExploreRecursive(selector.RecursionLimitNone(), ssb.ExploreAll(ssb.ExploreRecursiveEdge())). Node() verifySelectorLoads(t, newRootNode, expectedAllBlocks, s, false, datamodel.NewPath(nil), nil, func(lctx linking.LinkContext, lnk datamodel.Link) (io.Reader, error) { return storage.GetStream(lctx.Ctx, &store, lnk.Binary()) }) }) t.Run("explore-all with duplicate load skips via SkipMe", func(t *testing.T) { // when we use SkipMe to skip loading of already visited blocks we expect to // see the links show up in Loads but the lack of the links inside rootNode // and middleListNode in this list beyond the first set of loads show that // the block is not traversed when the SkipMe is received expectedSkipMeBlocks := []datamodel.Link{ rootNodeLnk, middleListNodeLnk, leafAlphaLnk, leafAlphaLnk, leafBetaLnk, leafAlphaLnk, middleMapNodeLnk, leafAlphaLnk, leafAlphaLnk, middleListNodeLnk, rootNodeLnk, middleListNodeLnk, rootNodeLnk, middleListNodeLnk, } s := selectorparse.CommonSelector_ExploreAllRecursively visited := make(map[datamodel.Link]bool) verifySelectorLoads(t, newRootNode, expectedSkipMeBlocks, s, false, datamodel.NewPath(nil), nil, func(lc linking.LinkContext, l datamodel.Link) (io.Reader, error) { // t.Logf("load %v [%v]\n", l, visited[l]) if visited[l] { return nil, traversal.SkipMe{} } visited[l] = true return storage.GetStream(lc.Ctx, &store, l.Binary()) }) }) t.Run("explore-all with duplicate load skips via LinkVisitOnlyOnce:true", func(t *testing.T) { // when using LinkRevisit:false to skip duplicate block loads, our loader // doesn't even get to see the load attempts (unlike SkipMe, where the // loader signals the skips) expectedLinkRevisitBlocks := []datamodel.Link{ rootNodeLnk, middleListNodeLnk, leafAlphaLnk, leafBetaLnk, middleMapNodeLnk, } s := selectorparse.CommonSelector_ExploreAllRecursively verifySelectorLoads(t, newRootNode, expectedLinkRevisitBlocks, s, true, datamodel.NewPath(nil), nil, func(lctx linking.LinkContext, lnk datamodel.Link) (io.Reader, error) { return storage.GetStream(lctx.Ctx, &store, lnk.Binary()) }) }) t.Run("explore-all with duplicate load and preloader skips via LinkVisitOnlyOnce:true", func(t *testing.T) { // same as above but make sure the preloader doesn't get in the way expectedLinkRevisitBlocks := []datamodel.Link{ rootNodeLnk, middleListNodeLnk, leafAlphaLnk, leafBetaLnk, middleMapNodeLnk, } s := selectorparse.CommonSelector_ExploreAllRecursively preloadLinks := make(map[datamodel.Link]struct{}) preloader := func(_ preload.PreloadContext, link preload.Link) { preloadLinks[link.Link] = struct{}{} } verifySelectorLoads(t, newRootNode, expectedLinkRevisitBlocks, s, true, datamodel.NewPath(nil), preloader, func(lctx linking.LinkContext, lnk datamodel.Link) (io.Reader, error) { return storage.GetStream(lctx.Ctx, &store, lnk.Binary()) }) for _, l := range expectedLinkRevisitBlocks { qt.Check(t, preloadLinks[l], qt.IsNotNil) } }) t.Run("explore-all with duplicate traversal skip via load at path", func(t *testing.T) { // when using LinkRevisit:false to skip duplicate block loads, our loader // doesn't even get to see the load attempts (unlike SkipMe, where the // loader signals the skips) testPathsToBlocksSkipped := []struct { path string expectedLinkVisits []datamodel.Link }{ // 5th node in load sequence for rootNode {"0/linkedList/2", append([]datamodel.Link{rootNodeLnk, middleListNodeLnk}, expectedAllBlocks[4:]...)}, // LinkedMap is 7th no, foo doesn't affect loading {"0/linkedMap/foo", append([]datamodel.Link{rootNodeLnk}, expectedAllBlocks[6:]...)}, // 8th node in load sequence for rootNode {"0/linkedMap/nested/alink", append([]datamodel.Link{rootNodeLnk, middleMapNodeLnk}, expectedAllBlocks[7:]...)}, {"0/linkedString", append([]datamodel.Link{rootNodeLnk}, expectedAllBlocks[8:]...)}, // pash through all nodes first root block, then go load middle list block {"1/2", append([]datamodel.Link{middleListNodeLnk}, expectedAllBlocks[len(rootNodeExpectedLinks)+3:]...)}, {"3/1", append([]datamodel.Link{middleListNodeLnk}, expectedAllBlocks[2*len(rootNodeExpectedLinks)+len(middleListNodeLinks)+2:]...)}, } for _, testCase := range testPathsToBlocksSkipped { t.Run(testCase.path, func(t *testing.T) { startAtPath := datamodel.ParsePath(testCase.path) s := selectorparse.CommonSelector_ExploreAllRecursively verifySelectorLoads(t, newRootNode, testCase.expectedLinkVisits, s, false, startAtPath, nil, func(lctx linking.LinkContext, lnk datamodel.Link) (io.Reader, error) { return storage.GetStream(lctx.Ctx, &store, lnk.Binary()) }) }) } }) } func TestWalk_ADLs(t *testing.T) { // we'll make a reifier that when it sees a list returns a custom element instead. customReifier := func(_ linking.LinkContext, n datamodel.Node, _ *linking.LinkSystem) (datamodel.Node, error) { if n.Kind() == datamodel.Kind_List { return leafAlpha, nil } return n, nil } ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) ss := ssb.ExploreFields(func(efsb builder.ExploreFieldsSpecBuilder) { efsb.Insert("linkedList", ssb.ExploreInterpretAs("linkJumper", ssb.Matcher())) }) s, err := ss.Selector() qt.Check(t, err, qt.IsNil) lsys := cidlink.DefaultLinkSystem() lsys.KnownReifiers = map[string]linking.NodeReifier{"linkJumper": customReifier} lsys.SetReadStorage(&store) var order int err = traversal.Progress{ Cfg: &traversal.Config{ LinkSystem: lsys, LinkTargetNodePrototypeChooser: basicnode.Chooser, }, }.WalkMatching(rootNode, s, func(prog traversal.Progress, n datamodel.Node) error { switch order { case 0: qt.Check(t, n, nodetests.NodeContentEquals, basicnode.NewString("alpha")) qt.Check(t, prog.Path.String(), qt.Equals, "linkedList") } order++ return nil }) qt.Check(t, err, qt.IsNil) qt.Check(t, order, qt.Equals, 1) } func TestWalkTransforming(t *testing.T) { ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) t.Run("transform selecting true should transform the root", func(t *testing.T) { n, err := traversal.WalkTransforming(basicnode.NewString("x"), selector.Matcher{}, func(prog traversal.Progress, n datamodel.Node) (datamodel.Node, error) { qt.Check(t, n, nodetests.NodeContentEquals, basicnode.NewString("x")) qt.Check(t, prog.Path.String(), qt.Equals, datamodel.Path{}.String()) return basicnode.NewString("replaced"), nil }) qt.Check(t, err, qt.IsNil) qt.Check(t, n, nodetests.NodeContentEquals, basicnode.NewString("replaced")) }) t.Run("transforming selecting fields recursively should work", func(t *testing.T) { ss := ssb.ExploreFields(func(efsb builder.ExploreFieldsSpecBuilder) { efsb.Insert("foo", ssb.Matcher()) efsb.Insert("nested", ssb.ExploreFields(func(efsb builder.ExploreFieldsSpecBuilder) { efsb.Insert("nonlink", ssb.Matcher()) })) }) s, err := ss.Selector() qt.Assert(t, err, qt.IsNil) var order int n, err := traversal.WalkTransforming(middleMapNode, s, func(prog traversal.Progress, n datamodel.Node) (datamodel.Node, error) { switch order { case 0: qt.Check(t, n, nodetests.NodeContentEquals, basicnode.NewBool(true)) qt.Check(t, prog.Path.String(), qt.Equals, "foo") case 1: qt.Check(t, n, nodetests.NodeContentEquals, basicnode.NewString("zoo")) qt.Check(t, prog.Path.String(), qt.Equals, "nested/nonlink") } order++ return basicnode.NewString("replaced"), nil }) qt.Check(t, err, qt.IsNil) qt.Check(t, order, qt.Equals, 2) qt.Check(t, n, nodetests.NodeContentEquals, fluent.MustBuildMap(basicnode.Prototype.Map, 3, func(na fluent.MapAssembler) { na.AssembleEntry("foo").AssignString("replaced") na.AssembleEntry("bar").AssignBool(false) na.AssembleEntry("nested").CreateMap(2, func(na fluent.MapAssembler) { na.AssembleEntry("alink").AssignLink(leafAlphaLnk) na.AssembleEntry("nonlink").AssignString("replaced") }) })) }) } ================================================ FILE: traversal/walk_with_stop_test.go ================================================ package traversal_test import ( "fmt" "testing" qt "github.com/frankban/quicktest" _ "github.com/ipld/go-ipld-prime/codec/dagjson" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/fluent" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipld/go-ipld-prime/node/basicnode" nodetests "github.com/ipld/go-ipld-prime/node/tests" "github.com/ipld/go-ipld-prime/traversal" "github.com/ipld/go-ipld-prime/traversal/selector" "github.com/ipld/go-ipld-prime/traversal/selector/builder" ) /* Remember, we've got the following fixtures in scope: var ( leafAlpha, leafAlphaLnk = encode(basicnode.NewString("alpha")) leafBeta, leafBetaLnk = encode(basicnode.NewString("beta")) middleMapNode, middleMapNodeLnk = encode(fluent.MustBuildMap(basicnode.Prototype__Map{}, 3, func(na fluent.MapAssembler) { na.AssembleEntry("foo").AssignBool(true) na.AssembleEntry("bar").AssignBool(false) na.AssembleEntry("nested").CreateMap(2, func(na fluent.MapAssembler) { na.AssembleEntry("alink").AssignLink(leafAlphaLnk) na.AssembleEntry("nonlink").AssignString("zoo") }) })) middleListNode, middleListNodeLnk = encode(fluent.MustBuildList(basicnode.Prototype__List{}, 4, func(na fluent.ListAssembler) { na.AssembleValue().AssignLink(leafAlphaLnk) na.AssembleValue().AssignLink(leafAlphaLnk) na.AssembleValue().AssignLink(leafBetaLnk) na.AssembleValue().AssignLink(leafAlphaLnk) })) rootNode, rootNodeLnk = encode(fluent.MustBuildMap(basicnode.Prototype__Map{}, 4, func(na fluent.MapAssembler) { na.AssembleEntry("plain").AssignString("olde string") na.AssembleEntry("linkedString").AssignLink(leafAlphaLnk) na.AssembleEntry("linkedMap").AssignLink(middleMapNodeLnk) na.AssembleEntry("linkedList").AssignLink(middleListNodeLnk) })) ) */ // ExploreRecursiveWithStop builds a recursive selector node with a stop condition func ExploreRecursiveWithStop(limit selector.RecursionLimit, sequence builder.SelectorSpec, stopLnk datamodel.Link) datamodel.Node { np := basicnode.Prototype__Map{} return fluent.MustBuildMap(np, 1, func(na fluent.MapAssembler) { // RecursionLimit na.AssembleEntry(selector.SelectorKey_ExploreRecursive).CreateMap(3, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_Limit).CreateMap(1, func(na fluent.MapAssembler) { switch limit.Mode() { case selector.RecursionLimit_Depth: na.AssembleEntry(selector.SelectorKey_LimitDepth).AssignInt(limit.Depth()) case selector.RecursionLimit_None: na.AssembleEntry(selector.SelectorKey_LimitNone).CreateMap(0, func(na fluent.MapAssembler) {}) default: panic("Unsupported recursion limit type") } }) // Sequence na.AssembleEntry(selector.SelectorKey_Sequence).CreateMap(1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_ExploreUnion).CreateList(2, func(na fluent.ListAssembler) { na.AssembleValue().AssignNode(fluent.MustBuildMap(np, 1, func(na fluent.MapAssembler) { na.AssembleEntry(selector.SelectorKey_Matcher).CreateMap(0, func(na fluent.MapAssembler) {}) })) na.AssembleValue().AssignNode(sequence.Node()) }) }) // Stop condition if stopLnk != nil { cond := fluent.MustBuildMap(basicnode.Prototype__Map{}, 1, func(na fluent.MapAssembler) { na.AssembleEntry(string(selector.ConditionMode_Link)).AssignLink(stopLnk) }) na.AssembleEntry(selector.SelectorKey_StopAt).AssignNode(cond) } }) }) } func TestStopAtLink(t *testing.T) { ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype__Any{}) t.Run("test ExploreRecursive stopAt with simple node", func(t *testing.T) { // Selector that passes through the map s, err := selector.CompileSelector(ExploreRecursiveWithStop( selector.RecursionLimitNone(), ssb.ExploreAll(ssb.ExploreRecursiveEdge()), middleMapNodeLnk)) if err != nil { t.Fatal(err) } var order int lsys := cidlink.DefaultLinkSystem() lsys.SetReadStorage(&store) err = traversal.Progress{ Cfg: &traversal.Config{ LinkSystem: lsys, LinkTargetNodePrototypeChooser: basicnode.Chooser, }, }.WalkMatching(rootNode, s, func(prog traversal.Progress, n datamodel.Node) error { // fmt.Println("Order", order, prog.Path.String()) switch order { case 0: // Root qt.Check(t, prog.Path.String(), qt.Equals, "") case 1: qt.Check(t, prog.Path.String(), qt.Equals, "plain") qt.Check(t, n, nodetests.NodeContentEquals, basicnode.NewString("olde string")) case 2: qt.Check(t, prog.Path.String(), qt.Equals, "linkedString") case 3: qt.Check(t, prog.Path.String(), qt.Equals, "linkedList") // We are starting to traverse the linkedList, we passed through the map already case 4: qt.Check(t, prog.Path.String(), qt.Equals, "linkedList/0") } order++ return nil }) qt.Check(t, err, qt.IsNil) qt.Check(t, order, qt.Equals, 8) }) } // mkChain creates a DAG that represent a chain of subDAGs. // The stopAt condition is extremely appealing for these use cases, as we can // partially sync a chain using ExploreRecursive without having to sync the // chain from scratch if we are already partially synced func mkChain() (datamodel.Node, []datamodel.Link) { leafAlpha, leafAlphaLnk = encode(basicnode.NewString("alpha")) leafBeta, leafBetaLnk = encode(basicnode.NewString("beta")) middleMapNode, middleMapNodeLnk = encode(fluent.MustBuildMap(basicnode.Prototype__Map{}, 3, func(na fluent.MapAssembler) { na.AssembleEntry("foo").AssignBool(true) na.AssembleEntry("bar").AssignBool(false) na.AssembleEntry("nested").CreateMap(2, func(na fluent.MapAssembler) { na.AssembleEntry("alink").AssignLink(leafAlphaLnk) na.AssembleEntry("nonlink").AssignString("zoo") }) })) middleListNode, middleListNodeLnk = encode(fluent.MustBuildList(basicnode.Prototype__List{}, 4, func(na fluent.ListAssembler) { na.AssembleValue().AssignLink(leafAlphaLnk) na.AssembleValue().AssignLink(leafAlphaLnk) na.AssembleValue().AssignLink(leafBetaLnk) na.AssembleValue().AssignLink(leafAlphaLnk) })) _, ch1Lnk := encode(fluent.MustBuildMap(basicnode.Prototype__Map{}, 4, func(na fluent.MapAssembler) { na.AssembleEntry("linkedList").AssignLink(middleListNodeLnk) })) _, ch2Lnk := encode(fluent.MustBuildMap(basicnode.Prototype__Map{}, 4, func(na fluent.MapAssembler) { na.AssembleEntry("linkedMap").AssignLink(middleMapNodeLnk) na.AssembleEntry("ch1").AssignLink(ch1Lnk) })) _, ch3Lnk := encode(fluent.MustBuildMap(basicnode.Prototype__Map{}, 4, func(na fluent.MapAssembler) { na.AssembleEntry("linkedString").AssignLink(leafAlphaLnk) na.AssembleEntry("ch2").AssignLink(ch2Lnk) })) headNode, headLnk := encode(fluent.MustBuildMap(basicnode.Prototype__Map{}, 4, func(na fluent.MapAssembler) { na.AssembleEntry("plain").AssignString("olde string") na.AssembleEntry("ch3").AssignLink(ch3Lnk) })) return headNode, []datamodel.Link{headLnk, ch3Lnk, ch2Lnk, ch1Lnk} } func TestStopInChain(t *testing.T) { chainNode, chainLnks := mkChain() // Stay in head stopAtInChainTest(t, chainNode, chainLnks[1], []string{"", "plain"}) // Get head and following block stopAtInChainTest(t, chainNode, chainLnks[2], []string{"", "plain", "ch3", "ch3/linkedString"}) // One more stopAtInChainTest(t, chainNode, chainLnks[3], []string{ "", "plain", "ch3", "ch3/ch2", "ch3/ch2/linkedMap", "ch3/ch2/linkedMap/bar", "ch3/ch2/linkedMap/foo", "ch3/ch2/linkedMap/nested", "ch3/ch2/linkedMap/nested/alink", "ch3/ch2/linkedMap/nested/nonlink", "ch3/linkedString", }) // Get the full chain stopAtInChainTest(t, chainNode, nil, []string{ "", "plain", "ch3", "ch3/ch2", "ch3/ch2/ch1", "ch3/ch2/ch1/linkedList", "ch3/ch2/ch1/linkedList/0", "ch3/ch2/ch1/linkedList/1", "ch3/ch2/ch1/linkedList/2", "ch3/ch2/ch1/linkedList/3", "ch3/ch2/linkedMap", "ch3/ch2/linkedMap/bar", "ch3/ch2/linkedMap/foo", "ch3/ch2/linkedMap/nested", "ch3/ch2/linkedMap/nested/alink", "ch3/ch2/linkedMap/nested/nonlink", "ch3/linkedString", }) } func stopAtInChainTest(t *testing.T, chainNode datamodel.Node, stopLnk datamodel.Link, expectedPaths []string) { ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype__Any{}) t.Run(fmt.Sprintf("test ExploreRecursive stopAt in chain with stoplink: %s", stopLnk), func(t *testing.T) { s, err := selector.CompileSelector(ExploreRecursiveWithStop( selector.RecursionLimitNone(), ssb.ExploreAll(ssb.ExploreRecursiveEdge()), stopLnk)) if err != nil { t.Fatal(err) } var order int lsys := cidlink.DefaultLinkSystem() lsys.SetReadStorage(&store) err = traversal.Progress{ Cfg: &traversal.Config{ LinkSystem: lsys, LinkTargetNodePrototypeChooser: basicnode.Chooser, }, }.WalkMatching(chainNode, s, func(prog traversal.Progress, n datamodel.Node) error { //fmt.Println("Order", order, prog.Path.String()) qt.Check(t, order < len(expectedPaths), qt.IsTrue) qt.Check(t, prog.Path.String(), qt.Equals, expectedPaths[order]) order++ return nil }) qt.Check(t, err, qt.IsNil) qt.Check(t, order, qt.Equals, len(expectedPaths)) }) } ================================================ FILE: version.json ================================================ { "version": "v0.23.0" }